fetching favorites and parsing in a new way since twitter returns a different...

fetching favorites and parsing in a new way since twitter returns a different page now. Output includes tweet text, tweet id, conversation id and date (#826)
Co-authored-by: yunus.catalcam <yunus.catalcam@vircongroup.com>

twint/feed.py

@@ -28,6 +28,18 @@ def Mobile(response):
 
     return tweets, max_id
 
+def MobileFav(response):
+
+    soup = BeautifulSoup(response, "html.parser")
+    tweets = soup.find_all("table", "tweet")
+    max_id = soup.find_all("div", "w-button-more")
+    try:
+        max_id = findall(r'max_id=(.*?)">', str(max_id))[0]
+    except Exception as e:
+        print(str(e) + " [x] feed.MobileFav")
+
+    return tweets, max_id
+
 def profile(response):
     logme.debug(__name__+':profile')
     json_response = loads(response)

twint/run.py

-import sys, os, time
+import sys, os, time, datetime
 from asyncio import get_event_loop, TimeoutError, ensure_future, new_event_loop, set_event_loop
 
 from . import datelock, feed, get, output, verbose, storage
@@ -49,7 +49,18 @@ class Twint:
             self.feed = []
             try:
                 if self.config.Favorites:
-                    self.feed, self.init = feed.Mobile(response)
+                    self.feed, self.init = feed.MobileFav(response)
+                    favorite_err_cnt = 0
+                    if len(self.feed) == 0 and len(self.init) == 0:
+                        while ((len(self.feed) == 0 or len(self.init) == 0) and favorite_err_cnt < 5):
+                            self.user_agent = await get.RandomUserAgent(wa=False)
+                            response = await get.RequestUrl(self.config, self.init,
+                                                            headers=[("User-Agent", self.user_agent)])
+                            self.feed, self.init = feed.MobileFav(response)
+                            favorite_err_cnt += 1
+                            time.sleep(1)
+                        if favorite_err_cnt == 5:
+                            print("Favorite page could not be fetched")
                     if not self.count % 40:
                         time.sleep(5)
                 elif self.config.Followers or self.config.Following:
@@ -120,7 +131,51 @@ class Twint:
     async def favorite(self):
         logme.debug(__name__+':Twint:favorite')
         await self.Feed()
-        self.count += await get.Multi(self.feed, self.config, self.conn)
+        favorited_tweets_list = []
+        for tweet in self.feed:
+            tweet_dict = {}
+            self.count += 1
+            try:
+                tweet_dict['data-item-id'] = tweet.find("div", {"class": "tweet-text"})['data-id']
+                t_url = tweet.find("span", {"class": "metadata"}).find("a")["href"]
+                tweet_dict['data-conversation-id'] = t_url.split('?')[0].split('/')[-1]
+                tweet_dict['username'] = tweet.find("div", {"class": "username"}).text.replace('\n', '').replace(' ',
+                                                                                                                 '')
+                tweet_dict['tweet'] = tweet.find("div", {"class": "tweet-text"}).find("div", {"class": "dir-ltr"}).text
+                date_str = tweet.find("td", {"class": "timestamp"}).find("a").text
+                # test_dates = ["1m", "2h", "Jun 21, 2019", "Mar 12", "28 Jun 19"]
+                # date_str = test_dates[3]
+                if len(date_str) <= 3 and (date_str[-1] == "m" or date_str[-1] == "h"):  # 25m 1h
+                    dateu = str(datetime.date.today())
+                    tweet_dict['date'] = dateu
+                elif ',' in date_str:  # Aug 21, 2019
+                    sp = date_str.replace(',', '').split(' ')
+                    date_str_formatted = sp[1] + ' ' + sp[0] + ' ' + sp[2]
+                    dateu = datetime.datetime.strptime(date_str_formatted, "%d %b %Y").strftime("%Y-%m-%d")
+                    tweet_dict['date'] = dateu
+                elif len(date_str.split(' ')) == 3:  # 28 Jun 19
+                    sp = date_str.split(' ')
+                    if len(sp[2]) == 2:
+                        sp[2] = '20' + sp[2]
+                    date_str_formatted = sp[0] + ' ' + sp[1] + ' ' + sp[2]
+                    dateu = datetime.datetime.strptime(date_str_formatted, "%d %b %Y").strftime("%Y-%m-%d")
+                    tweet_dict['date'] = dateu
+                else:  # Aug 21
+                    sp = date_str.split(' ')
+                    date_str_formatted = sp[1] + ' ' + sp[0] + ' ' + str(datetime.date.today().year)
+                    dateu = datetime.datetime.strptime(date_str_formatted, "%d %b %Y").strftime("%Y-%m-%d")
+                    tweet_dict['date'] = dateu
+
+                favorited_tweets_list.append(tweet_dict)
+
+            except Exception as e:
+                logme.critical(__name__ + ':Twint:favorite:favorite_field_lack')
+                print("shit: ", date_str, " ", str(e))
+
+        try:
+            self.config.favorited_tweets_list += favorited_tweets_list
+        except AttributeError:
+            self.config.favorited_tweets_list = favorited_tweets_list
 
     async def profile(self):
         await self.Feed()