Fix repeating set of tweets load #250 (#252)

* New twitter scheme

* Fix minor mistake

* Fix location issue

twint/feed.py

@@ -43,7 +43,4 @@ def Json(response):
     html = json_response["items_html"]
     soup = BeautifulSoup(html, "html.parser")
     feed = soup.find_all("li", "js-stream-item")
-    split = json_response["min_position"].split("-")
-    split[1] = feed[-1]["data-item-id"]
-    
-    return feed, "-".join(split)
+    return feed, json_response["min_position"]

twint/get.py

@@ -53,8 +53,8 @@ async def RequestUrl(config, init):
             _url = await url.Profile(config.Username, init)
             response = await Request(_url, connector=_connector)
     elif config.TwitterSearch:
-        _url = await url.Search(config, init)
-        response = await Request(_url, connector=_connector)
+        _url, params = await url.Search(config, init)
+        response = await Request(_url, params=params, connector=_connector)
     else:
         if config.Following:
             _url = await url.Following(config.Username, init)
@@ -71,27 +71,27 @@ async def RequestUrl(config, init):
 
 async def MobileRequest(url, **options):
     #loggin.info("[<] " + str(datetime.now()) + ':: get+MobileRequest')
-    ua = {'User-Agent': 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12'}
     connector = options.get("connector")
     if connector:
-        async with aiohttp.ClientSession(headers=ua, connector=connector) as session:
+        async with aiohttp.ClientSession(connector=connector) as session:
             return await Response(session, url)
-    async with aiohttp.ClientSession(headers=ua) as session:
+    async with aiohttp.ClientSession() as session:
         return await Response(session, url)
 
-async def Request(url, **options):
+
+async def Request(url, connector=None, params=[]):
     #loggin.info("[<] " + str(datetime.now()) + ':: get+Request')
-    connector = options.get("connector")
     if connector:
         async with aiohttp.ClientSession(connector=connector) as session:
-            return await Response(session, url)
+            return await Response(session, url, params)
     async with aiohttp.ClientSession() as session:
-        return await Response(session, url)
+        return await Response(session, url, params)
 
-async def Response(session, url):
+async def Response(session, url, params=[]):
     #loggin.info("[<] " + str(datetime.now()) + ':: get+Response')
+    headers = {'User-Agent': 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12' }
     with timeout(30):
-        async with session.get(url, ssl=False) as response:
+        async with session.get(url, ssl=False, headers=headers, params=params) as response:
             return await response.text()
 
 async def Username(_id):

twint/run.py

@@ -11,7 +11,7 @@ class Twint:
         if config.Resume is not None and config.TwitterSearch:
             self.init = f"TWEET-{config.Resume}-0"
         else:
-            self.init = -1
+            self.init = "-1"
         self.feed = [-1]
         self.count = 0
         self.config = config

twint/url.py

@@ -53,49 +53,54 @@ async def Profile(username, init):
 
 async def Search(config, init):
     #logging.info("[<] " + str(datetime.now()) + ':: url+Search')
-    url = f"{base}/search/timeline?f=tweets&vertical=default&lang=en"
-    url += "&include_available_features=1&include_entities=1&"
-    url += f"reset_error_state=false&src=typd&qf=off&max_position={init}&q="
-
+    url = f"{base}/search/timeline"
+    params = [
+        ('f', 'tweets'),
+        ('vertical', 'default'),
+        ('src', 'unkn'),
+        ('include_available_features', '1'),
+        ('include_entities', '1'),
+        ('max_position', init),
+        ('reset_error_state', 'false'),
+    ]
+    q = ""
     if config.Lang:
-        url = url.replace("lang=en", f"l={config.Lang}&lang=en")
+        params.append(("l", config.Lang))
+        params.append(("lang", "en"))
     if config.Username:
-        url += f"from%3A{config.Username}"
+        q += f"from:{config.Username}"
     if config.Geo:
         config.Geo = config.Geo.replace(" ", "")
-        url += f"geocode%3A{config.Geo}"
+        q += f"geocode:{config.Geo}"
     if config.Search:
-        config.Search = config.Search.replace(" ", "%20")
-        config.Search = config.Search.replace("#", "%23")
-        url += f"%20{config.Search}"
+        q += f"{config.Search}"
     if config.Year:
-        url += f"%20until%3A{config.Year}-1-1"
+        q += f" until:{config.Year}-1-1"
     if config.Since:
-        url += f"%20since%3A{config.Since}"
+        q += f" since:{config.Since}"
     if config.Until:
-        url += f"%20until%3A{config.Until}"
+        q += f" until:{config.Until}"
     if config.Fruit:
-        url += "%20%22myspace.com%22%20OR%20%22last.fm%22%20OR"
-        url += "%20%22mail%22%20OR%20%22email%22%20OR%20%22gmail%22%20OR%20%22e-mail%22"
-        url += "%20OR%20%22phone%22%20OR%20%22call%20me%22%20OR%20%22text%20me%22"
-        url += "%20OR%20%22keybase%22"
+        url += ' "myspace.com" OR "last.fm" OR'
+        url += ' "mail" OR "email" OR "gmail" OR "e-mail"'
+        # url += "%20OR%20%22phone%22%20OR%20%22call%20me%22%20OR%20%22text%20me%22"
+        # url += "%20OR%20%22keybase%22"
     if config.Verified:
-        url += "%20filter%3Averified"
+        q += " filter:verified"
     if config.To:
-        url += f"%20to%3A{config.To}"
+        q += f" to:{config.To}"
     if config.All:
-        url += f"%20to%3A{config.All}%20OR%20from%3A{config.All}%20OR%20@{config.All}"
+        q += f" to:{config.All} OR from:{config.All} OR @{config.All}"
     if config.Near:
-        config.Near = config.Near.replace(" ", "%20")
-        config.Near = config.Near.replace(",", "%2C")
-        url += f"%20near%3A%22{config.Near}%22"
+        q += f' near:"{config.Near}"'
     if config.Images:
-        url += "%20filter%3Aimages"
+        q += " filter:images"
     if config.Videos:
-        url += "%20filter%3Avideos"
+        q += " filter:videos"
     if config.Media:
-        url += "%20filter%3Amedia"
+        q += " filter:media"
     if config.Replies:
-        url += "%20filter%3Areplies"
+        q += " filter:replies"
 
-    return url
+    params.append(("q", q))
+    return url, params