Skip to content

T
Twint
Commit f9c76746 authored by minamotorin's avatar minamotorin Committed by GitHub
Browse files
Options

Related to twintproject/twint#1335 

This commit doesn't completely fix the problem, but `twint.Profile` works
parent 92239ae6
Hide whitespace changes
Inline Side-by-side
Showing with 53 additions and 42 deletions
twint/feed.py
View file @ f9c76746
...@@ -59,9 +59,12 @@ def _get_cursor(response): ...@@ -59,9 +59,12 @@ def _get_cursor(response):
next_cursor = response['timeline']['instructions'][0]['addEntries']['entries'][-1]['content'][ next_cursor = response['timeline']['instructions'][0]['addEntries']['entries'][-1]['content'][
'operation']['cursor']['value'] 'operation']['cursor']['value']
except KeyError: except KeyError:
# this is needed because after the first request location of cursor is changed try:
next_cursor = response['timeline']['instructions'][-1]['replaceEntry']['entry']['content']['operation'][ # this is needed because after the first request location of cursor is changed
'cursor']['value'] next_cursor = response['timeline']['instructions'][-1]['replaceEntry']['entry']['content']['operation'][
'cursor']['value']
except KeyError:
next_cursor = response['timeline']['instructions'][0]['entries'][-1]['content']['value']
return next_cursor return next_cursor
...@@ -77,46 +80,54 @@ def Json(response): ...@@ -77,46 +80,54 @@ def Json(response):
def parse_tweets(config, response): def parse_tweets(config, response):
logme.debug(__name__ + ':parse_tweets') logme.debug(__name__ + ':parse_tweets')
response = loads(response) response = loads(response)
if len(response['globalObjects']['tweets']) == 0:
msg = 'No more data!'
raise NoMoreTweetsException(msg)
feed = [] feed = []
for timeline_entry in response['timeline']['instructions'][0]['addEntries']['entries']: if 'globalObjects' in response:
# this will handle the cases when the timeline entry is a tweet if len(response['globalObjects']['tweets']) == 0:
if (config.TwitterSearch or config.Profile) and (timeline_entry['entryId'].startswith('sq-I-t-') or msg = 'No more data!'
timeline_entry['entryId'].startswith('tweet-')): raise NoMoreTweetsException(msg)
if 'tweet' in timeline_entry['content']['item']['content']: for timeline_entry in response['timeline']['instructions'][0]['addEntries']['entries']:
_id = timeline_entry['content']['item']['content']['tweet']['id'] # this will handle the cases when the timeline entry is a tweet
# skip the ads if (config.TwitterSearch or config.Profile) and (timeline_entry['entryId'].startswith('sq-I-t-') or
if 'promotedMetadata' in timeline_entry['content']['item']['content']['tweet']: timeline_entry['entryId'].startswith('tweet-')):
if 'tweet' in timeline_entry['content']['item']['content']:
_id = timeline_entry['content']['item']['content']['tweet']['id']
# skip the ads
if 'promotedMetadata' in timeline_entry['content']['item']['content']['tweet']:
continue
elif 'tombstone' in timeline_entry['content']['item']['content'] and 'tweet' in \
timeline_entry['content']['item']['content']['tombstone']:
_id = timeline_entry['content']['item']['content']['tombstone']['tweet']['id']
else:
_id = None
if _id is None:
raise ValueError('Unable to find ID of tweet in timeline.')
try:
temp_obj = response['globalObjects']['tweets'][_id]
except KeyError:
logme.info('encountered a deleted tweet with id {}'.format(_id))
config.deleted.append(_id)
continue continue
elif 'tombstone' in timeline_entry['content']['item']['content'] and 'tweet' in \ temp_obj['user_data'] = response['globalObjects']['users'][temp_obj['user_id_str']]
timeline_entry['content']['item']['content']['tombstone']: if 'retweeted_status_id_str' in temp_obj:
_id = timeline_entry['content']['item']['content']['tombstone']['tweet']['id'] rt_id = temp_obj['retweeted_status_id_str']
else: _dt = response['globalObjects']['tweets'][rt_id]['created_at']
_id = None _dt = datetime.strptime(_dt, '%a %b %d %H:%M:%S %z %Y')
if _id is None: _dt = utc_to_local(_dt)
raise ValueError('Unable to find ID of tweet in timeline.') _dt = str(_dt.strftime(Tweet_formats['datetime']))
try: temp_obj['retweet_data'] = {
temp_obj = response['globalObjects']['tweets'][_id] 'user_rt_id': response['globalObjects']['tweets'][rt_id]['user_id_str'],
except KeyError: 'user_rt': response['globalObjects']['tweets'][rt_id]['full_text'],
logme.info('encountered a deleted tweet with id {}'.format(_id)) 'retweet_id': rt_id,
'retweet_date': _dt,
config.deleted.append(_id) }
continue feed.append(temp_obj)
temp_obj['user_data'] = response['globalObjects']['users'][temp_obj['user_id_str']] else:
if 'retweeted_status_id_str' in temp_obj: response = response['data']['user']['result']['timeline']
rt_id = temp_obj['retweeted_status_id_str'] for timeline_entry in response['timeline']['instructions'][0]['entries']:
_dt = response['globalObjects']['tweets'][rt_id]['created_at'] if timeline_entry['content'].get('itemContent'):
_dt = datetime.strptime(_dt, '%a %b %d %H:%M:%S %z %Y') temp_obj = timeline_entry['content']['itemContent']['tweet_results']['result']['legacy']
_dt = utc_to_local(_dt) temp_obj['user_data'] = timeline_entry['content']['itemContent']['tweet_results']['result']['core']['user_results']['result']['legacy']
_dt = str(_dt.strftime(Tweet_formats['datetime'])) feed.append(temp_obj)
temp_obj['retweet_data'] = {
'user_rt_id': response['globalObjects']['tweets'][rt_id]['user_id_str'],
'user_rt': response['globalObjects']['tweets'][rt_id]['full_text'],
'retweet_id': rt_id,
'retweet_date': _dt,
}
feed.append(temp_obj)
next_cursor = _get_cursor(response) next_cursor = _get_cursor(response)
return feed, next_cursor return feed, next_cursor
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment