Commit 40c6fd0d authored by Yunus Emre Çatalçam's avatar Yunus Emre Çatalçam Committed by GitHub

fetching favorites and parsing in a new way since twitter returns a different...

fetching favorites and parsing in a new way since twitter returns a different page now. Output includes tweet text, tweet id, conversation id and date (#826)
Co-authored-by: default avataryunus.catalcam <yunus.catalcam@vircongroup.com>
parent e21e1acb
...@@ -28,6 +28,18 @@ def Mobile(response): ...@@ -28,6 +28,18 @@ def Mobile(response):
return tweets, max_id return tweets, max_id
def MobileFav(response):
soup = BeautifulSoup(response, "html.parser")
tweets = soup.find_all("table", "tweet")
max_id = soup.find_all("div", "w-button-more")
try:
max_id = findall(r'max_id=(.*?)">', str(max_id))[0]
except Exception as e:
print(str(e) + " [x] feed.MobileFav")
return tweets, max_id
def profile(response): def profile(response):
logme.debug(__name__+':profile') logme.debug(__name__+':profile')
json_response = loads(response) json_response = loads(response)
......
import sys, os, time import sys, os, time, datetime
from asyncio import get_event_loop, TimeoutError, ensure_future, new_event_loop, set_event_loop from asyncio import get_event_loop, TimeoutError, ensure_future, new_event_loop, set_event_loop
from . import datelock, feed, get, output, verbose, storage from . import datelock, feed, get, output, verbose, storage
...@@ -49,7 +49,18 @@ class Twint: ...@@ -49,7 +49,18 @@ class Twint:
self.feed = [] self.feed = []
try: try:
if self.config.Favorites: if self.config.Favorites:
self.feed, self.init = feed.Mobile(response) self.feed, self.init = feed.MobileFav(response)
favorite_err_cnt = 0
if len(self.feed) == 0 and len(self.init) == 0:
while ((len(self.feed) == 0 or len(self.init) == 0) and favorite_err_cnt < 5):
self.user_agent = await get.RandomUserAgent(wa=False)
response = await get.RequestUrl(self.config, self.init,
headers=[("User-Agent", self.user_agent)])
self.feed, self.init = feed.MobileFav(response)
favorite_err_cnt += 1
time.sleep(1)
if favorite_err_cnt == 5:
print("Favorite page could not be fetched")
if not self.count % 40: if not self.count % 40:
time.sleep(5) time.sleep(5)
elif self.config.Followers or self.config.Following: elif self.config.Followers or self.config.Following:
...@@ -120,7 +131,51 @@ class Twint: ...@@ -120,7 +131,51 @@ class Twint:
async def favorite(self): async def favorite(self):
logme.debug(__name__+':Twint:favorite') logme.debug(__name__+':Twint:favorite')
await self.Feed() await self.Feed()
self.count += await get.Multi(self.feed, self.config, self.conn) favorited_tweets_list = []
for tweet in self.feed:
tweet_dict = {}
self.count += 1
try:
tweet_dict['data-item-id'] = tweet.find("div", {"class": "tweet-text"})['data-id']
t_url = tweet.find("span", {"class": "metadata"}).find("a")["href"]
tweet_dict['data-conversation-id'] = t_url.split('?')[0].split('/')[-1]
tweet_dict['username'] = tweet.find("div", {"class": "username"}).text.replace('\n', '').replace(' ',
'')
tweet_dict['tweet'] = tweet.find("div", {"class": "tweet-text"}).find("div", {"class": "dir-ltr"}).text
date_str = tweet.find("td", {"class": "timestamp"}).find("a").text
# test_dates = ["1m", "2h", "Jun 21, 2019", "Mar 12", "28 Jun 19"]
# date_str = test_dates[3]
if len(date_str) <= 3 and (date_str[-1] == "m" or date_str[-1] == "h"): # 25m 1h
dateu = str(datetime.date.today())
tweet_dict['date'] = dateu
elif ',' in date_str: # Aug 21, 2019
sp = date_str.replace(',', '').split(' ')
date_str_formatted = sp[1] + ' ' + sp[0] + ' ' + sp[2]
dateu = datetime.datetime.strptime(date_str_formatted, "%d %b %Y").strftime("%Y-%m-%d")
tweet_dict['date'] = dateu
elif len(date_str.split(' ')) == 3: # 28 Jun 19
sp = date_str.split(' ')
if len(sp[2]) == 2:
sp[2] = '20' + sp[2]
date_str_formatted = sp[0] + ' ' + sp[1] + ' ' + sp[2]
dateu = datetime.datetime.strptime(date_str_formatted, "%d %b %Y").strftime("%Y-%m-%d")
tweet_dict['date'] = dateu
else: # Aug 21
sp = date_str.split(' ')
date_str_formatted = sp[1] + ' ' + sp[0] + ' ' + str(datetime.date.today().year)
dateu = datetime.datetime.strptime(date_str_formatted, "%d %b %Y").strftime("%Y-%m-%d")
tweet_dict['date'] = dateu
favorited_tweets_list.append(tweet_dict)
except Exception as e:
logme.critical(__name__ + ':Twint:favorite:favorite_field_lack')
print("shit: ", date_str, " ", str(e))
try:
self.config.favorited_tweets_list += favorited_tweets_list
except AttributeError:
self.config.favorited_tweets_list = favorited_tweets_list
async def profile(self): async def profile(self):
await self.Feed() await self.Feed()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment