Merge branch 'another' into dev

.gitignore

@@ -22,7 +22,7 @@ Icon?
 Thumbs.db
 
 # Babel ES6 compiles files
-dist
+dist/*
 # Documentation build
 doc/build
 

Procfile

+web: python3 backend_requests.py
\ No newline at end of file

README.md

+## Shadowban.eu Backend made to run on Heroku
+
 Pull
 ```
 git clone https://github.com/shadowban-eu/shadowban-eu-backend && cd shadowban-eu-backend

backend.py

@@ -12,7 +12,10 @@ import time
 
 from aiohttp import web
 from bs4 import BeautifulSoup
-from db import connect
+
+from db import Database
+from statistics import count_sensitives
+from typeahead import test as test_typeahead
 
 
 # This is a public value from the Twitter source code.
@@ -185,8 +188,10 @@ class TwitterSession:
         self.set_csrf_header()
         await self.refresh_old_token()
         try:
-            async with self._session.get(url, headers=self._headers) as r:
-                result = await r.json()
+            # async with await self._session.get(url, headers=self._headers) as r:
+            #     result = await r.json()
+            r = await self._session.get(url, headers=self._headers)
+            result = await r.json()
         except Exception as e:
             debug("EXCEPTION: " + str(type(e)))
             debug("EXCEPTION text: " + str(e))
@@ -206,9 +211,6 @@ class TwitterSession:
             additional_query = "&tweet_search_mode=live"
         return await self.get("https://api.twitter.com/2/search/adaptive.json?q="+urllib.parse.quote(query)+"&count=20&spelling_corrections=0" + additional_query)
 
-    async def typeahead_raw(self, query):
-        return await self.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query))
-
     async def profile_raw(self, username):
         return await self.get("https://api.twitter.com/1.1/users/show.json?screen_name=" + urllib.parse.quote(username))
 
@@ -427,6 +429,8 @@ class TwitterSession:
         if not profile["exists"] or profile.get("suspended", False) or profile.get("protected", False) or not profile.get('has_tweets'):
             return result
 
+        result["profile"]["sensitives"] = await count_sensitives(self, user_id)
+
         result["tests"] = {}
 
         search_raw = await self.search_raw("from:@" + username)
@@ -441,12 +445,7 @@ class TwitterSession:
         except (KeyError, IndexError):
             pass
 
-        typeahead_raw = await self.typeahead_raw("@" + username)
-        result["tests"]["typeahead"] = False
-        try:
-            result["tests"]["typeahead"] = len([1 for user in typeahead_raw["users"] if user["screen_name"].lower() == username.lower()]) > 0
-        except KeyError:
-            pass
+        result["tests"]["typeahead"] = await test_typeahead(self, username)
 
         if "search" in result["tests"] and result["tests"]["search"] == False:
             result["tests"]["ghost"] = await self.test_ghost_ban(user_id)
@@ -518,6 +517,10 @@ async def unlocked(request):
 async def api(request):
     global test_index
     screen_name = request.match_info['screen_name']
+    if screen_name == "wikileaks" and request.query_string != "watch":
+        debug("[wikileaks] Returning last watch result")
+        db_result = db.get_result_by_screen_name("wikileaks")
+        return web.json_response(db_result, headers={"Access-Control-Allow-Origin": args.cors_allow})
     session = guest_sessions[test_index % len(guest_sessions)]
     test_index += 1
     result = await session.test(screen_name)
@@ -562,11 +565,13 @@ parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip w
 parser.add_argument('--mongo-host', type=str, default=None, help='hostname or IP of mongoDB service to connect to')
 parser.add_argument('--mongo-port', type=int, default=27017, help='port of mongoDB service to connect to')
 parser.add_argument('--mongo-db', type=str, default='tester', help='name of mongo database to use')
-parser.add_argument('--twitter-auth-key', type=str, default=TWITTER_AUTH_KEY, help='auth key for twitter guest session')
+parser.add_argument('--mongo-username', type=str, default='', help='name of user in mongo database')
+parser.add_argument('--mongo-password', type=str, default='', help='password for user in mongo database')
+parser.add_argument('--twitter-auth-key', type=str, default=None, help='auth key for twitter guest session', required=False)
 parser.add_argument('--cors-allow', type=str, default=None, help='value for Access-Control-Allow-Origin header')
-args = parser.parse_args()
+args, unknown = parser.parse_known_args()
 
-TwitterSession.twitter_auth_key = args.twitter_auth_key
+TwitterSession.twitter_auth_key = TWITTER_AUTH_KEY
 
 if (args.cors_allow is None):
     debug('[CORS] Running without CORS headers')
@@ -602,7 +607,7 @@ def run():
     if args.mongo_host is not None:
         db = connect(host=args.mongo_host, port=args.mongo_port, db=args.mongo_db)
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(login_accounts(accounts, args.cookie_dir))
+    # loop.run_until_complete(login_accounts(accounts, args.cookie_dir))
     loop.run_until_complete(login_guests())
     app = web.Application()
     app.add_routes(routes)

backend_requests.py

+from flask import Flask
+from flask_cors import CORS
+from requests_oauthlib import OAuth1Session, OAuth2Session
+import os
+import time
+
+app = Flask(__name__)
+CORS(app)
+
+TWITTER_AUTH_KEY = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
+
+
+@app.route('/{screen_name}')
+def shadowban(screen_name):
+    # DO IT
+    pass
+
+
+
+
+@app.route("/<screen_name>")
+def searchban(screen_name):
+    returnjson = {
+                "timestamp": time.time(),
+                "profile": {
+                    # "id": "7080152",
+                    # "screenName": "TwitterJP",
+                    # "protected": False,
+                    # "suspended": False,
+                    # "has_tweets": True,
+                    "exists": False,
+                    "error": None
+                },
+                # "check": {
+                #     "search": 1484727214419628037,
+                #     "suggest": True,
+                #     "ghost": {"ban": True},
+                #     "reply": {"ban": False, "tweet": "1480819689898987523", "in_reply_to": "1369626114381901828"}
+                # }
+            }
+
+    
+    # twitter = OAuth1Session(TWITTER_IPHONE_CK, TWITTER_IPHONE_CS)
+    twitter_b = OAuth2Session()
+    twitter_b.headers["Authorization"] = "Bearer {}".format(TWITTER_AUTH_KEY)
+
+    # check rate limit
+    # response = twitter_b.get("https://api.twitter.com/1.1/application/rate_limit_status.json")
+    # print(response.json())
+
+    # profile_url = "https://api.twitter.com/1.1/users/show.json"
+    # params = {"screen_name": screen_name}
+    # profile_info = twitter_b.get(profile_url, params=params)
+    # profile_json = profile_info.json()
+    # print(profile_json)
+    # if profile_info.status_code == 200:
+    #     returnjson["profile"]["exists"] = True
+    #     returnjson["profile"]["id"] = profile_json["id_str"]
+    #     returnjson["profile"]["screenName"] = profile_json["screen_name"]
+    #     returnjson["profile"]["protected"] = profile_json["protected"]
+    # elif profile_info.status_code == 403:
+    #     returnjson["profile"]["suspended"] = True
+    #     return returnjson
+    # else:
+    #     returnjson["profile"]["error"] = profile_json["errors"][0]["message"]
+    #     # return returnjson
+
+    # if profile_json["protected"] == True:
+    #     returnjson["profile"]["protected"] = True
+    #     return returnjson
+
+    # if profile_json["statuses_count"] == 0:
+    #     returnjson["profile"]["has_tweets"] = False
+    #     return returnjson
+    # else:
+    #     returnjson["profile"]["has_tweets"] = True
+
+    # check whether the user has any tweets
+    usertlurl = "https://api.twitter.com/1.1/statuses/user_timeline.json"
+    params = {"screen_name": screen_name, "count": 200}
+
+    usertl_b = twitter_b.get(usertlurl, params=params)
+    usertl = usertl_b
+    usertl_json = usertl.json()
+    # # print(usertlb)
+    # if "errors" in usertlb:
+    #     return "An error occurred" # TODO: Better error handling
+    # if len(usertlb) == 0:
+    #     return "No tweets found"  # TODO: Better error handling
+
+    if len(usertl_json) == 0:
+        returnjson["profile"]["has_tweets"] = False
+        return returnjson
+
+    returnjson["profile"]["has_tweets"] = True
+
+    if usertl.status_code == 200:
+        returnjson["profile"]["exists"] = True
+        returnjson["profile"]["id"] = usertl_json[0]["user"]["id_str"]
+        returnjson["profile"]["screen_name"] = usertl_json[0]["user"]["screen_name"]
+        # returnjson["profile"]["protected"] = usertl_json["protected"]
+    elif usertl.status_code == 403:
+        returnjson["profile"]["suspended"] = True
+        return returnjson
+    else:
+        if "error" in usertl_json and usertl_json["error"] == "Not authorized.":
+            returnjson["profile"]["protected"] = True
+            returnjson["profile"]["suspended"] = True
+            returnjson["profile"]["has_tweets"] = False
+            return returnjson
+        returnjson["profile"]["error"] = usertl_json
+        return returnjson
+
+    # if usertl_json["protected"] == True:
+    #     returnjson["profile"]["protected"] = True  ## how do you determen protected and suspended
+    #     return returnjson
+
+    # if usertl_json["statuses_count"] == 0:
+    #     returnjson["profile"]["has_tweets"] = False
+    #     return returnjson
+    # else:
+    #     returnjson["profile"]["has_tweets"] = True
+
+    returnjson["tests"] = {
+    "search": True,
+    "typeahead": True, ## suggest ban
+    "ghost": {"ban": False},
+    "more_replies": {"ban": False, "tweet": "-1", "in_reply_to": "-1"}
+}
+
+    # searchurl = "https://api.twitter.com/1.1/users/search.json"
+    # params = {"q": "from:@{}".format(screen_name), "count": 1}
+    # search = twitter_b.get(searchurl, params=params).json()
+    # print(search)
+    # if len(search) == 0:
+    #     returnjson["test"]["search"] = "ban"
+    #     return returnjson
+    # else:
+    #     return returnjson
+
+    searchurl_v2 = "https://api.twitter.com/2/search/adaptive.json"
+    params_v2 = {"q": "from:@{}".format(screen_name), "count": 1, "spelling_corrections": 0, "tweet_search_mode": "live"}
+    search_v2 = twitter_b.get(searchurl_v2, params=params_v2).json()
+    search_tweets = search_v2["globalObjects"]["tweets"]
+    if search_tweets == {}:
+        returnjson["tests"]["search"] = False
+        returnjson["tests"]["typeahead"] = False
+        return returnjson
+    else:
+        return returnjson
+    print(search_v2)
+
+
+app.run(debug=False, port=os.environ.get("PORT", 5000), host="0.0.0.0")

db.py

 import copy
 import traceback
 import sys
-from pymongo import MongoClient, errors as MongoErrors
+from pymongo import MongoClient, errors as MongoErrors, DESCENDING
 
 class Database:
-    def __init__(self, host=None, port=27017, db='tester'):
+    def __init__(self, host=None, port=27017, username='', password='', db='tester'):
         # collection name definitions
         RESULTS_COLLECTION = 'results'
         RATELIMIT_COLLECTION = 'rate-limits'
@@ -13,7 +13,12 @@ class Database:
             print('[mongoDB] Connecting to ' + host + ':' + str(port))
             print('[mongoDB] Using Database `' + db + '`')
             # client and DB
-            self.client = MongoClient(host, port, serverSelectionTimeoutMS=3)
+            self.client = MongoClient(
+                host=host,
+                port=port,
+                username=username,
+                password=password,
+                serverSelectionTimeoutMS=3)
             self.db = self.client[db]
 
             # collections
@@ -38,8 +43,6 @@ class Database:
     def write_rate_limit(self, data):
         self.rate_limits.insert_one(data)
 
-def connect(host=None, port=27017, db='tester'):
-    if host is None:
-        raise ValueError('[mongoDB] Database constructor needs a `host`name or ip!')
+    def get_result_by_screen_name(self, screen_name):
+        return self.results.find_one({ "profile.screen_name": screen_name }, sort=[("_id", DESCENDING)], projection={"_id": False})
 
-    return Database(host=host, port=port, db=db)

requirements.txt

-aiohttp
-pymongo
-bs4
-daemon
+flask
+requests_oauthlib
+flask_cors
\ No newline at end of file

requirements.txt.old

+aiohttp==3.7.4.post0
+pymongo
+bs4==0.0.1
+daemon
+# gunicorn
\ No newline at end of file

runtime.txt

+python-3.9.9
\ No newline at end of file

statistics.py

+import sys
+
+# Count amount of "possibly_sensitive_editable" and "possibly_sensitive"
+# flagged tweets in user's timeline
+async def count_sensitives(session, user_id):
+  profile_timeline = await session.get_profile_tweets_raw(user_id)
+  profile_tweets = profile_timeline["globalObjects"]["tweets"].values()
+
+  counted = len(profile_tweets)
+  possibly_sensitive = len([1 for tweet in profile_tweets if "possibly_sensitive" in tweet.keys()])
+  possibly_sensitive_editable = len([1 for tweet in profile_tweets if "possibly_sensitive_editable" in tweet.keys()])
+
+  result = {
+    "counted": counted,
+    "possibly_sensitive": possibly_sensitive,
+    "possibly_sensitive_editable": possibly_sensitive_editable
+  }
+
+  return result

typeahead.py

+import urllib.parse
+
+async def test(session, screen_name):
+  query = "@" + screen_name
+  suggestions = await session.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query))
+  try:
+    result = len([1 for user in suggestions["users"] if user["screen_name"].lower() == screen_name.lower()]) > 0
+  except:
+    result = False
+
+  return result