Commit a025f90d authored by Francesco Poldi's avatar Francesco Poldi Committed by GitHub

Csv json output fix (#307)

* Changed storing options for csv and json

* Reduced LOC
parent 6d026c2c
...@@ -3,6 +3,20 @@ import csv ...@@ -3,6 +3,20 @@ import csv
import json import json
import os import os
def outputExt(objType, fType):
if objType == "str":
objType = "username"
outExt = f"/{objType}s.{fType}"
return outExt
def addExt(base, objType, fType):
if len(base.split('.')) == 1:
createDirIfMissing(base)
base += outputExt(objType, fType)
return base
def Text(entry, f): def Text(entry, f):
print(entry, file=open(f, "a", encoding="utf-8")) print(entry, file=open(f, "a", encoding="utf-8"))
...@@ -34,35 +48,25 @@ def createDirIfMissing(dirname): ...@@ -34,35 +48,25 @@ def createDirIfMissing(dirname):
def Csv(obj, config): def Csv(obj, config):
_obj_type = obj.__class__.__name__ _obj_type = obj.__class__.__name__
if _obj_type == "str": _obj_type = "username"
Output_csv = {"tweet": config.Output.split(".")[0] + "/tweets.csv",
"user": config.Output.split(".")[0] + "/users.csv",
"username": config.Output.split(".")[0] + "/usernames.csv"}
fieldnames, row = struct(obj, config.Custom[_obj_type], _obj_type) fieldnames, row = struct(obj, config.Custom[_obj_type], _obj_type)
createDirIfMissing(config.Output.split(".")[0]) base = addExt(config.Output, _obj_type, "csv")
if not (os.path.exists(Output_csv[_obj_type])): if not (os.path.exists(base)):
with open(Output_csv[_obj_type], "w", newline='', encoding="utf-8") as csv_file: with open(base, "w", newline='', encoding="utf-8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader() writer.writeheader()
with open(Output_csv[_obj_type], "a", newline='', encoding="utf-8") as csv_file: with open(base, "a", newline='', encoding="utf-8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writerow(row) writer.writerow(row)
def Json(obj, config): def Json(obj, config):
_obj_type = obj.__class__.__name__ _obj_type = obj.__class__.__name__
if _obj_type == "str": _obj_type = "username"
Output_json = {"tweet": config.Output.split(".")[0] + "/tweets.json",
"user": config.Output.split(".")[0] + "/users.json",
"username": config.Output.split(".")[0] + "/usernames.json"}
null, data = struct(obj, config.Custom[_obj_type], _obj_type) null, data = struct(obj, config.Custom[_obj_type], _obj_type)
createDirIfMissing(config.Output.split(".")[0]) base = addExt(config.Output, _obj_type, "json")
with open(Output_json[_obj_type], "a", newline='', encoding="utf-8") as json_file: with open(base, "a", newline='', encoding="utf-8") as json_file:
json.dump(data, json_file, ensure_ascii=False) json.dump(data, json_file, ensure_ascii=False)
json_file.write("\n") json_file.write("\n")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment