Commit dd5aaaba authored by Francesco Poldi's avatar Francesco Poldi

Finished Pandas storing

parent b13299a0
import pandas as pd import warnings
from time import strftime, localtime from time import strftime, localtime
import pandas as pd
from .elasticsearch import * from .elasticsearch import *
_blocks = [] _blocks = []
def update(Tweet, session): def update(Tweet, session):
day = weekday(strftime("%A", localtime(Tweet.datetime))) day = weekday(strftime("%A", localtime(Tweet.datetime)))
dt = "{} {}".format(Tweet.datestamp, Tweet.timestamp) dt = "{} {}".format(Tweet.datestamp, Tweet.timestamp)
...@@ -32,27 +32,34 @@ def get(): ...@@ -32,27 +32,34 @@ def get():
df = pd.DataFrame(_blocks) df = pd.DataFrame(_blocks)
return df return df
def save(_dataframe, _dataname, _filename, _type): def save(_filename, _dataframe, **options):
if not _dataname: if options.get("dataname"):
_dataname = options.get("dataname")
else:
_dataname = "twint" _dataname = "twint"
if not _type or _type == "HDF5": if not options.get("type"):
_store = pd.HDFStore(_filename) with warnings.catch_warnings():
_store[_dataname] = _dataframe warnings.simplefilter("ignore")
elif _type == "Pickle": _store = pd.HDFStore(_filename)
_dataframe.to_pickle(_filename) _store[_dataname] = _dataframe
_store.close()
elif options.get("type") == "Pickle":
with warnings.catch_warnings():
warnings.simplefilter("ignore")
_dataframe.to_pickle(_filename)
else: else:
print("Please specify: DataFrame, DataFrame name, filename and type (HDF5, default, or Pickle") print("Please specify: filename, DataFrame, DataFrame name and type (HDF5, default, or Pickle")
def read(_dataframe, _dataname, _filename, _type): def read(_filename, **options):
if not _dataname: if not options.get("dataname"):
_dataname = "Twint" _dataname = "Twint"
if not _type or _type == "HDF5": if not options.get("type"):
_store = pd.HDFStore(_filename) _store = pd.HDFStore(_filename)
df = _store[_dataname] df = _store[_dataname]
return df return df
elif _type == "Pickle": elif options.get("type") == "Pickle":
df = pd.read_pickle(_filename) df = pd.read_pickle(_filename)
return df return df
else: else:
......
...@@ -29,7 +29,7 @@ def _output(obj, output, config): ...@@ -29,7 +29,7 @@ def _output(obj, output, config):
write.Json(obj, config) write.Json(obj, config)
else: else:
write.Text(output, config.Output) write.Text(output, config.Output)
if config.Pandas: if config.Pandas:
Pandas.update(obj, config.Essid) Pandas.update(obj, config.Essid)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment