Merge pull request #2 from minamotorin/twintproject/twint#1335

Twintproject/twint#1335

README.md

@@ -12,6 +12,18 @@ This problem doesn't happen recently.
 - [twintproject/twint#1061](https://github.com/twintproject/twint/issues/1061)
 - [twintproject/twint#1114](https://github.com/twintproject/twint/issues/1114)
 
+### json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
+
+The fix is **not complete**.
+`twint.run.Profile` will work but `twint.run.db` will not.
+This means [`test.py`](./test.py) causes an error.
+
+I think this is because the fields of the result table are not exactly the same as the traditional ones.
+
+#### Related 
+
+- [twintproject/twint#1335](https://github.com/twintproject/twint/issues/1335)
+
 # TWINT - Twitter Intelligence Tool
 ![2](https://i.imgur.com/iaH3s7z.png)
 ![3](https://i.imgur.com/hVeCrqL.png)

twint/feed.py

@@ -59,9 +59,12 @@ def _get_cursor(response):
         next_cursor = response['timeline']['instructions'][0]['addEntries']['entries'][-1]['content'][
             'operation']['cursor']['value']
     except KeyError:
+        try:
             # this is needed because after the first request location of cursor is changed
             next_cursor = response['timeline']['instructions'][-1]['replaceEntry']['entry']['content']['operation'][
                 'cursor']['value']
+        except KeyError:
+            next_cursor = response['timeline']['instructions'][0]['entries'][-1]['content']['value']
     return next_cursor
 
 
@@ -77,10 +80,11 @@ def Json(response):
 def parse_tweets(config, response):
     logme.debug(__name__ + ':parse_tweets')
     response = loads(response)
+    feed = []
+    if 'globalObjects' in response:
         if len(response['globalObjects']['tweets']) == 0:
             msg = 'No more data!'
             raise NoMoreTweetsException(msg)
-    feed = []
         for timeline_entry in response['timeline']['instructions'][0]['addEntries']['entries']:
             # this will handle the cases when the timeline entry is a tweet
             if (config.TwitterSearch or config.Profile) and (timeline_entry['entryId'].startswith('sq-I-t-') or
@@ -118,5 +122,12 @@ def parse_tweets(config, response):
                         'retweet_date': _dt,
                     }
                 feed.append(temp_obj)
+    else:
+        response = response['data']['user']['result']['timeline']
+        for timeline_entry in response['timeline']['instructions'][0]['entries']:
+            if timeline_entry['content'].get('itemContent'):
+                temp_obj = timeline_entry['content']['itemContent']['tweet_results']['result']['legacy']
+                temp_obj['user_data'] = timeline_entry['content']['itemContent']['tweet_results']['result']['core']['user_results']['result']['legacy']
+                feed.append(temp_obj)
     next_cursor = _get_cursor(response)
     return feed, next_cursor

twint/url.py

 import datetime
+import json
 from sys import platform
 import logging as logme
 from urllib.parse import urlencode
@@ -168,37 +169,27 @@ async def Search(config, init):
 
 def SearchProfile(config, init=None):
     logme.debug(__name__ + ':SearchProfile')
-    _url = 'https://api.twitter.com/2/timeline/profile/{user_id}.json'.format(user_id=config.User_id)
+    _url = 'https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies'
     tweet_count = 100
-    params = [
-        # some of the fields are not required, need to test which ones aren't required
-        ('include_profile_interstitial_type', '1'),
-        ('include_blocking', '1'),
-        ('include_blocked_by', '1'),
-        ('include_followed_by', '1'),
-        ('include_want_retweets', '1'),
-        ('include_mute_edge', '1'),
-        ('include_can_dm', '1'),
-        ('include_can_media_tag', '1'),
-        ('skip_status', '1'),
-        ('cards_platform', 'Web - 12'),
-        ('include_cards', '1'),
-        ('include_ext_alt_text', 'true'),
-        ('include_quote_count', 'true'),
-        ('include_reply_count', '1'),
-        ('tweet_mode', 'extended'),
-        ('include_entities', 'true'),
-        ('include_user_entities', 'true'),
-        ('include_ext_media_color', 'true'),
-        ('include_ext_media_availability', 'true'),
-        ('send_error_codes', 'true'),
-        ('simple_quoted_tweet', 'true'),
-        ('include_tweet_replies', 'true'),
-        ('count', tweet_count),
-        ('ext', 'mediaStats%2ChighlightedLabel'),
-    ]
-
+    variables = {
+        "userId": config.User_id,
+        "count": tweet_count,
+        "includePromotedContent": True,
+        "withCommunity": True,
+        "withSuperFollowsUserFields": True,
+        "withBirdwatchPivots": False,
+        "withDownvotePerspective": False,
+        "withReactionsMetadata": False,
+        "withReactionsPerspective": False,
+        "withSuperFollowsTweetFields": True,
+        "withVoice": True,
+        "withV2Timeline": False,
+        "__fs_interactive_text": False,
+        "__fs_dont_mention_me_view_api_enabled": False,
+    }
     if type(init) == str:
-        params.append(('cursor', str(init)))
+        variables['cursor'] = init
+    params = [('variables', json.dumps(variables, separators=(',',':')))]
+
     _serialQuery = _sanitizeQuery(_url, params)
-    return _url, params, _serialQuery
+    return _serialQuery, [], _serialQuery