Commit 9bfd9f56 authored by Cody Zacharias's avatar Cody Zacharias Committed by GitHub

Fix Month Scrape Limit bug

parent 5c4e897e
...@@ -639,30 +639,38 @@ async def main(): ...@@ -639,30 +639,38 @@ async def main():
init = -1 init = -1
num = 0 num = 0
action = getAction() action = getAction()
while _since < _until: if action != "":
arg.since = str(_until - datetime.timedelta(days=int(arg.timedelta))) while True:
arg.until = str(_until) if len(feed) > 0:
'''
If our response from getFeed() has an exception,
it signifies there are no position IDs to continue
with, telling Twint it's finished scraping.
'''
if len(feed) > 0:
if action != "":
if arg.favorites: if arg.favorites:
feed, init, count = await getFavorites(init) feed, init, count = await getFavorites(init)
num += count
else: else:
feed, init = await getFollow(init) feed, init = await getFollow(init)
else: else:
break
if arg.limit is not None and num >= int(arg.limit):
break
else:
while _since < _until:
arg.since = str(_until - datetime.timedelta(days=int(arg.timedelta)))
arg.until = str(_until)
'''
If our response from getFeed() has an exception,
it signifies there are no position IDs to continue
with, telling Twint it's finished scraping.
'''
if len(feed) > 0:
feed, init, count = await getTweets(init) feed, init, count = await getTweets(init)
num += count num += count
else: else:
_until = _until - datetime.timedelta(days=int(arg.timedelta)) _until = _until - datetime.timedelta(days=int(arg.timedelta))
feed = [-1] feed = [-1]
break
# Control when we want to stop scraping. # Control when we want to stop scraping.
if arg.limit is not None and num >= int(arg.limit): if arg.limit is not None and num >= int(arg.limit):
break break
if arg.database: if arg.database:
now = str(datetime.datetime.now()) now = str(datetime.datetime.now())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment