Commit 74bb6407 authored by Francesco Poldi's avatar Francesco Poldi

Added timedelta

Added timedelta to choose the time interval for every request, this speed up the process
parent b7e67b52
......@@ -447,7 +447,7 @@ async def main():
init = -1
num = 0
while _since < _until:
arg.since = str(_until - datetime.timedelta(days=1))
arg.since = str(_until - datetime.timedelta(days=int(arg.timedelta)))
arg.until = str(_until)
'''
If our response from getFeed() has an exception,
......@@ -458,7 +458,7 @@ async def main():
feed, init, count = await getTweets(init)
num += count
else:
_until = _until - datetime.timedelta(days=1)
_until = _until - datetime.timedelta(days=int(arg.timedelta))
feed = [-1]
# Control when we want to stop scraping.
if arg.limit is not None and num >= int(arg.limit):
......@@ -509,6 +509,7 @@ if __name__ == "__main__":
ap.add_argument("-l", help="Serch for Tweets in a specific language")
ap.add_argument("-o", help="Save output to a file.")
ap.add_argument("-es", "--elasticsearch", help="Index to Elasticsearch")
ap.add_argument("-t", "--timedelta", help="Time intervall for every request")
ap.add_argument("--year", help="Filter Tweets before specified year.")
ap.add_argument("--since", help="Filter Tweets sent since date (Example: 2017-12-27).")
ap.add_argument("--until", help="Filter Tweets sent until date (Example: 2017-12-27).")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment