Commit 74bb6407 authored by Francesco Poldi's avatar Francesco Poldi

Added timedelta

Added timedelta to choose the time interval for every request, this speed up the process
parent b7e67b52
...@@ -447,7 +447,7 @@ async def main(): ...@@ -447,7 +447,7 @@ async def main():
init = -1 init = -1
num = 0 num = 0
while _since < _until: while _since < _until:
arg.since = str(_until - datetime.timedelta(days=1)) arg.since = str(_until - datetime.timedelta(days=int(arg.timedelta)))
arg.until = str(_until) arg.until = str(_until)
''' '''
If our response from getFeed() has an exception, If our response from getFeed() has an exception,
...@@ -458,7 +458,7 @@ async def main(): ...@@ -458,7 +458,7 @@ async def main():
feed, init, count = await getTweets(init) feed, init, count = await getTweets(init)
num += count num += count
else: else:
_until = _until - datetime.timedelta(days=1) _until = _until - datetime.timedelta(days=int(arg.timedelta))
feed = [-1] feed = [-1]
# Control when we want to stop scraping. # Control when we want to stop scraping.
if arg.limit is not None and num >= int(arg.limit): if arg.limit is not None and num >= int(arg.limit):
...@@ -509,6 +509,7 @@ if __name__ == "__main__": ...@@ -509,6 +509,7 @@ if __name__ == "__main__":
ap.add_argument("-l", help="Serch for Tweets in a specific language") ap.add_argument("-l", help="Serch for Tweets in a specific language")
ap.add_argument("-o", help="Save output to a file.") ap.add_argument("-o", help="Save output to a file.")
ap.add_argument("-es", "--elasticsearch", help="Index to Elasticsearch") ap.add_argument("-es", "--elasticsearch", help="Index to Elasticsearch")
ap.add_argument("-t", "--timedelta", help="Time intervall for every request")
ap.add_argument("--year", help="Filter Tweets before specified year.") ap.add_argument("--year", help="Filter Tweets before specified year.")
ap.add_argument("--since", help="Filter Tweets sent since date (Example: 2017-12-27).") ap.add_argument("--since", help="Filter Tweets sent since date (Example: 2017-12-27).")
ap.add_argument("--until", help="Filter Tweets sent until date (Example: 2017-12-27).") ap.add_argument("--until", help="Filter Tweets sent until date (Example: 2017-12-27).")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment