Commit 964a6428 authored by Raphael Beer's avatar Raphael Beer

Merge branch 'rls/1.0.1'

parents 0b038131 fe40eed9
ACCOUNT_FILE=./.htaccounts
COOKIE_DIR=./.htcookies
LOG_FILE=./logs/results.log
DEBUG_FILE=./logs/debug.log
PORT=4040
HOST=127.0.0.1
MONGO_HOST=127.0.0.1
MONGO_PORT=27017
MONGO_DB=tester
TWITTER_AUTH_KEY=GRAPHQL_KEY
......@@ -52,4 +52,5 @@ node_modules
# .env
.env
.env.*
!.env.example
.ht*
FROM python:3.5.7-slim-buster
RUN mkdir /app
WORKDIR /app
ADD requirements.txt /app/
ADD . /app
RUN pip3 install --no-cache-dir -r ./requirements.txt
Pull
```
git clone https://github.com/shadowban-eu/shadowban-eu-backend && cd shadowban-eu-backend
```
Add config
```
cp ~/configs/.env.development|production ./
cp ~/configs/.htaccounts
```
Run
`./run.sh development|production`
See [.env.example](https://github.com/shadowban-eu/shadowban-eu-backend/blob/dev/.env.example) for .env variables
import aiohttp
import argparse
import asyncio
import daemon
import json
import os
import re
import traceback
import urllib.parse
......@@ -13,6 +15,9 @@ from db import connect
routes = web.RouteTableDef()
class UnexpectedApiError(Exception):
pass
def get_nested(obj, path, default=None):
for p in path:
if obj is None or not p in obj:
......@@ -20,14 +25,34 @@ def get_nested(obj, path, default=None):
obj = obj[p]
return obj
def is_error(result, code=None):
return isinstance(result.get("errors", None), list) and (len([x for x in result["errors"] if x.get("code", None) == code]) > 0 or code is None and len(result["errors"] > 0))
def is_another_error(result, codes):
return isinstance(result.get("errors", None), list) and len([x for x in result["errors"] if x.get("code", None) not in codes]) > 0
account_sessions = []
account_index = 0
log_file = None
debug_file = None
guest_session_pool_size = 10
guest_sessions = []
test_index = 0
def next_session():
def key(s):
remaining_time = s.reset - time.time()
if s.remaining <= 3 and remaining_time > 0:
return 900
return remaining_time
sessions = sorted([s for s in account_sessions if not s.locked], key=key)
if len(sessions) > 0:
return sessions[0]
class TwitterSession:
_auth = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
_user_url = "https://api.twitter.com/graphql/SEn6Mq-OakvVOT1CJqUO2A/UserByScreenName?variables="
twitter_auth_key = None
def __init__(self):
self._headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
......@@ -41,9 +66,11 @@ class TwitterSession:
# rate limit monitoring
self.limit = -1
self.remaining = -1
self.remaining = 180
self.reset = -1
self.overshot = -1
self.locked = False
self.next_refresh = None
# session user's @username
# this stays `None` for guest sessions
......@@ -55,90 +82,156 @@ class TwitterSession:
if cookie.key == 'ct0':
self._headers['X-Csrf-Token'] = cookie.value
async def login(self, username = None, password = None, email = None):
async def get_guest_token(self):
self._headers['Authorization'] = 'Bearer ' + self.twitter_auth_key
async with self._session.post("https://api.twitter.com/1.1/guest/activate.json", headers=self._headers) as r:
response = await r.json()
guest_token = response.get("guest_token", None)
if guest_token is None:
debug("Failed to fetch guest token")
return guest_token
async def renew_session(self):
await self.try_close()
self._session = aiohttp.ClientSession()
async def refresh_old_token(self):
if self.username is not None or self.next_refresh is None or time.time() < self.next_refresh:
return
debug("Refresh token: " + str(self._guest_token))
await self.login_guest()
debug("New token: " + str(self._guest_token))
async def try_close(self):
if self._session is not None:
try:
await self._session.close()
except:
pass
async def login_guest(self):
await self.renew_session()
self.set_csrf_header()
old_token = self._guest_token
new_token = await self.get_guest_token()
self._guest_token = new_token if new_token is not None else old_token
if new_token is not None:
self.next_refresh = time.time() + 3600
self._headers['X-Guest-Token'] = self._guest_token
async def login(self, username = None, password = None, email = None, cookie_dir=None):
self._session = aiohttp.ClientSession()
if password is not None:
async with self._session.get("https://twitter.com/login", headers=self._headers) as r:
login_page = await r.text()
form_data = {}
soup = BeautifulSoup(login_page, 'html.parser')
form_data["authenticity_token"] = soup.find('input', {'name': 'authenticity_token'}).get('value')
form_data["session[username_or_email]"] = email
form_data["session[password]"] = password
form_data["remember_me"] = "1"
async with self._session.post('https://twitter.com/sessions', data=form_data, headers=self._headers) as r:
await r.text()
if str(r.url) == "https://twitter.com/":
log("Login of %s successful" % username)
else:
log("Error logging in %s" % username)
login_required = True
cookie_file = None
if cookie_dir is not None:
cookie_file = os.path.join(cookie_dir, username)
if os.path.isfile(cookie_file):
log("Use cookie file for %s" % username)
self._session.cookie_jar.load(cookie_file)
login_required = False
store_cookies = True
if login_required:
async with self._session.get("https://twitter.com/login", headers=self._headers) as r:
login_page = await r.text()
form_data = {}
soup = BeautifulSoup(login_page, 'html.parser')
form_data["authenticity_token"] = soup.find('input', {'name': 'authenticity_token'}).get('value')
form_data["session[username_or_email]"] = email
form_data["session[password]"] = password
form_data["remember_me"] = "1"
async with self._session.post('https://twitter.com/sessions', data=form_data, headers=self._headers) as r:
response = await r.text()
if str(r.url) == "https://twitter.com/":
log("Login of %s successful" % username)
else:
store_cookies = False
log("Error logging in %s (%s)" % (username, r.url))
debug("ERROR PAGE\n" + response)
else:
async with self._session.get('https://twitter.com', headers=self._headers) as r:
await r.text()
self.set_csrf_header()
self.username = username
if cookie_file is not None and store_cookies:
self._session.cookie_jar.save(cookie_file)
else:
self._headers['Authorization'] = 'Bearer ' + self._auth
async with self._session.post("https://api.twitter.com/1.1/guest/activate.json", headers=self._headers) as r:
guest_token = await r.json()
self._guest_token = guest_token["guest_token"]
self._headers['X-Guest-Token'] = self._guest_token
await self.login_guest()
self._headers['Authorization'] = 'Bearer ' + self._auth
self._headers['Authorization'] = 'Bearer ' + self.twitter_auth_key
async def get(self, url, retries=0):
self.set_csrf_header()
await self.refresh_old_token()
try:
async with self._session.get(url, headers=self._headers) as r:
result = await r.json()
except Exception as e:
debug("EXCEPTION: " + str(type(e)))
if self.username is None:
await self.login_guest()
raise e
self.monitor_rate_limit(r.headers)
if self.username is None and self.remaining < 10 or is_error(result, 88) or is_error(result, 239):
await self.login_guest()
if retries > 0 and is_error(result, 353):
return await self.get(url, retries - 1)
if is_error(result, 326):
self.locked = True
return result
async def search_raw(self, query, live=True):
additional_query = ""
if live:
additional_query = "&tweet_search_mode=live"
async with self._session.get("https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_composer_source=true&include_ext_alt_text=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&q="+urllib.parse.quote(query)+"&qf_abuse=false&count=20&query_source=typed_query&pc=1&spelling_corrections=0&ext=mediaStats%2ChighlightedLabel%2CcameraMoment" + additional_query, headers=self._headers) as r:
return await r.json()
return await self.get("https://api.twitter.com/2/search/adaptive.json?q="+urllib.parse.quote(query)+"&count=20&spelling_corrections=0" + additional_query)
async def typeahead_raw(self, query):
async with self._session.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query), headers=self._headers) as r:
return await r.json()
return await self.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query))
async def profile_raw(self, username):
obj = json.dumps({"screen_name": username, "withHighlightedLabel": True})
async with self._session.get(self._user_url + urllib.parse.quote(obj), headers=self._headers) as r:
return await r.json()
return await self.get(self._user_url + urllib.parse.quote(obj))
async def get_profile_tweets_raw(self, user_id):
async with self._session.get("https://api.twitter.com/2/timeline/profile/" + str(user_id) +".json?include_tweet_replies=1&include_want_retweets=0&include_reply_count=1&count=1000", headers=self._headers) as r:
return await r.json()
async def test_detached_tweets():
pass
return await self.get("https://api.twitter.com/2/timeline/profile/" + str(user_id) +".json?include_tweet_replies=1&include_want_retweets=0&include_reply_count=1&count=1000")
async def tweet_raw(self, tweet_id, count=20, cursor=None, retry_csrf=True):
if cursor is None:
cursor = ""
else:
cursor = "&cursor=" + urllib.parse.quote(cursor)
async with self._session.get("https://api.twitter.com/2/timeline/conversation/" + tweet_id + ".json?include_reply_count=1&send_error_codes=true&count="+str(count)+ cursor, headers=self._headers) as r:
result = await r.json()
# debug('Tweet request ' + tweet_id + ':\n' + str(r) + '\n\n' + json.dumps(result) + '\n\n\n')
self.set_csrf_header()
if self.screen_name is not None:
self.monitor_rate_limit(r.headers)
if retry_csrf and isinstance(result.get("errors", None), list) and len([x for x in result["errors"] if x.get("code", None) == 353]):
return await self.tweet_raw(tweet_id, count, cursor, False)
return result
return await self.get("https://api.twitter.com/2/timeline/conversation/" + tweet_id + ".json?include_reply_count=1&send_error_codes=true&count="+str(count)+ cursor)
def monitor_rate_limit(self, headers):
# store last remaining count for reset detection
last_remaining = self.remaining
self.limit = int(headers.get('x-rate-limit-limit', -1))
self.remaining = int(headers.get('x-rate-limit-remaining', -1))
self.reset = int(headers.get('x-rate-limit-reset', -1))
limit = headers.get('x-rate-limit-limit', None)
remaining = headers.get('x-rate-limit-remaining', None)
reset = headers.get('x-rate-limit-reset', None)
if limit is not None:
self.limit = int(limit)
if remaining is not None:
self.remaining = int(remaining)
if reset is not None:
self.reset = int(reset)
# rate limit reset
if last_remaining < self.remaining and self.overshot > 0:
log('[rate-limit] Reset detected for ' + self.screen_name + '. Saving overshoot count...')
db.write_rate_limit({ 'screen_name': self.screen_name, 'overshot': self.overshot })
if last_remaining < self.remaining and self.overshot > 0 and self.username is not None:
log('[rate-limit] Reset detected for ' + self.username + '. Saving overshoot count...')
db.write_rate_limit({ 'screen_name': self.username, 'overshot': self.overshot })
self.overshot = 0
# count the requests that failed because of rate limiting
if self.remaining is 0:
log('[rate-limit] Limit hit by ' + self.screen_name + '.')
log('[rate-limit] Limit hit by ' + str(self.username) + '.')
self.overshot += 1
@classmethod
......@@ -230,9 +323,11 @@ class TwitterSession:
debug('Found:' + tid + '\n')
debug('In reply to:' + replied_to_id + '\n')
global account_sessions
reference_session = next_session()
if reference_session is None:
return
global account_index
reference_session = account_sessions[account_index % len(account_sessions)]
account_index += 1
before_barrier = await reference_session.tweet_raw(replied_to_id, 1000)
......@@ -272,10 +367,13 @@ class TwitterSession:
debug(traceback.format_exc())
async def test(self, username, more_replies_test=True):
await self.login()
result = {"timestamp": time.time()}
profile = {}
profile_raw = await self.profile_raw(username)
debug(str(profile_raw))
if is_another_error(profile_raw, [50, 63]):
debug("Other error:" + str(username))
raise UnexpectedApiError
try:
user_id = str(profile_raw["data"]["user"]["rest_id"])
......@@ -296,14 +394,10 @@ class TwitterSession:
profile["protected"] = profile_raw["data"]["user"]["legacy"]["protected"]
except KeyError:
pass
try:
profile["exists"] = len([1 for error in profile_raw["errors"] if error["code"] == 50]) == 0
except KeyError:
profile["exists"] = True
try:
profile["suspended"] = len([1 for error in profile_raw["errors"] if error["code"] == 63]) > 0
except KeyError:
pass
profile["exists"] = not is_error(profile_raw, 50)
suspended = is_error(profile_raw, 63)
if suspended:
profile["suspended"] = suspended
try:
profile["has_tweets"] = int(profile_raw["data"]["user"]["legacy"]["statuses_count"]) > 0
except KeyError:
......@@ -343,8 +437,7 @@ class TwitterSession:
if more_replies_test and not get_nested(result, ["tests", "ghost", "ban"], False):
result["tests"]["more_replies"] = await self.test_barrier(user_id)
debug('Writing result for ' + result['profile']['screen_name'] + ' to DB');
global db
debug('Writing result for ' + result['profile']['screen_name'] + ' to DB')
db.write_result(result)
return result
......@@ -353,7 +446,6 @@ class TwitterSession:
await self._session.close()
def debug(message):
global debug_file
if message.endswith('\n') is False:
message = message + '\n'
......@@ -364,7 +456,6 @@ def debug(message):
print(message)
def log(message):
global log_file
# ensure newline
if message.endswith('\n') is False:
message = message + '\n'
......@@ -375,56 +466,108 @@ def log(message):
else:
print(message)
def print_session_info(sessions):
text = ""
for session in sessions:
text += "\n%6d %5d %9d %5d" % (int(session.locked), session.limit, session.remaining, session.reset - int(time.time()))
return text
@routes.get('/.stats')
async def stats(request):
text = "--- GUEST SESSIONS ---\n\nLocked Limit Remaining Reset"
text += print_session_info(guest_sessions)
text += "\n\n\n--- ACCOUNTS ---\n\nLocked Limit Remaining Reset"
text += print_session_info(account_sessions)
return web.Response(text=text)
@routes.get('/.unlocked/{screen_name}')
async def unlocked(request):
screen_name = request.match_info['screen_name']
text = "Not unlocked"
for session in account_sessions:
if session.username.lower() != screen_name.lower():
continue
session.locked = False
text = "Unlocked"
return web.Response(text=text)
@routes.get('/{screen_name}')
async def hello(request):
async def api(request):
global test_index
screen_name = request.match_info['screen_name']
if screen_name == '.stats':
text = "Limit Remaining Reset"
for session in account_sessions:
text += "\n%5d %9d %5d" % (session.limit, session.remaining, session.reset - int(time.time()))
return web.Response(text=text)
session = TwitterSession()
session = guest_sessions[test_index % len(guest_sessions)]
test_index += 1
result = await session.test(screen_name)
log(json.dumps(result) + '\n')
await session.close()
return web.json_response(result)
async def login_accounts(accounts):
async def login_accounts(accounts, cookie_dir=None):
if cookie_dir is not None and not os.path.isdir(cookie_dir):
os.mkdir(cookie_dir, 0o700)
coroutines = []
for acc in accounts:
session = TwitterSession()
coroutines.append(session.login(*acc))
coroutines.append(session.login(*acc, cookie_dir=cookie_dir))
account_sessions.append(session)
await asyncio.gather(*coroutines)
async def login_guests():
for i in range(0, guest_session_pool_size):
session = TwitterSession()
guest_sessions.append(session)
await asyncio.gather(*[s.login() for s in guest_sessions])
log("Guest sessions created")
def ensure_dir(path):
if os.path.isdir(path) is False:
print('Creating directory %s' % path)
os.mkdir(path)
parser = argparse.ArgumentParser(description='Twitter Shadowban Tester')
parser.add_argument('--account-file', type=str, default='.htaccounts', help='json file with reference account credentials')
parser.add_argument('--cookie-dir', type=str, default=None, help='directory for session account storage')
parser.add_argument('--log', type=str, default=None, help='log file where test results are written to')
parser.add_argument('--daemon', action='store_true', help='run in background')
parser.add_argument('--debug', type=str, default=None, help='debug log file')
parser.add_argument('--port', type=int, default=8080, help='port which to listen on')
parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip which to listen on')
parser.add_argument('--mongo-host', type=str, default='localhost', help='hostname or IP of mongoDB service to connect to')
parser.add_argument('--mongo-port', type=int, default=27017, help='port of mongoDB service to connect to')
parser.add_argument('--mongo-db', type=str, default='tester', help='name of mongo database to use')
parser.add_argument('--mongo-collection', type=str, default='results', help='name of collection to save test results to')
parser.add_argument('--twitter-auth-key', type=str, default=None, help='auth key for twitter guest session', required=True)
args = parser.parse_args()
db = connect(host=args.mongo_host, port=args.mongo_port)
TwitterSession.twitter_auth_key = args.twitter_auth_key;
ensure_dir(args.cookie_dir)
with open(args.account_file, "r") as f:
accounts = json.loads(f.read())
if args.log is not None:
print("Logging test results to %s", args.log)
print("Logging test results to %s" % args.log)
log_dir = os.path.dirname(args.log)
ensure_dir(log_dir)
log_file = open(args.log, "a")
if args.debug is not None:
print("Logging debug output to %s", args.debug)
print("Logging debug output to %s" % args.debug)
debug_dir = os.path.dirname(args.debug)
ensure_dir(debug_dir)
debug_file = open(args.debug, "a")
loop = asyncio.get_event_loop()
loop.run_until_complete(login_accounts(accounts))
app = web.Application()
app.add_routes(routes)
web.run_app(app, host='127.0.0.1', port=args.port)
def run():
global db
db = connect(host=args.mongo_host, port=args.mongo_port)
loop = asyncio.get_event_loop()
loop.run_until_complete(login_accounts(accounts, args.cookie_dir))
loop.run_until_complete(login_guests())
app = web.Application()
app.add_routes(routes)
web.run_app(app, host=args.host, port=args.port)
if args.daemon:
with daemon.DaemonContext():
run()
else:
run()
......@@ -4,18 +4,23 @@ import sys
from pymongo import MongoClient, errors as MongoErrors
class Database:
def __init__(self, host=None, port=27017, db='tester', collection_name='results'):
def __init__(self, host=None, port=27017, db='tester'):
# collection name definitions
RESULTS_COLLECTION = 'results'
RATELIMIT_COLLECTION = 'rate-limits'
try:
print('[mongoDB] Connecting to ' + host + ':' + str(port))
print('[mongoDB] Using Collection `' + collection_name + '` in Database `' + db + '`')
print('[mongoDB] Using Database `' + db + '`')
# client and DB
self.client = MongoClient(host, port, serverSelectionTimeoutMS=3)
self.db = self.client[db]
# collection for test results
self.results = self.db[collection_name]
# collection for rate limit monitoring
self.rate_limits = self.db['rate-limits']
# test connection immediately, instead of
# collections
self.results = self.db[RESULTS_COLLECTION]
self.rate_limits = self.db[RATELIMIT_COLLECTION]
# Test connection immediately, instead of
# when trying to write in a request, later.
self.client.admin.command('ismaster')
except MongoErrors.ServerSelectionTimeoutError:
......@@ -33,8 +38,8 @@ class Database:
def write_rate_limit(self, data):
self.rate_limits.insert_one(data)
def connect(host=None, port=27017, db='tester', collection_name='results'):
def connect(host=None, port=27017, db='tester'):
if host is None:
raise ValueError('[mongoDB] Database constructor needs a `host`name or ip!')
return Database(host=host, port=port, db=db, collection_name=collection_name)
return Database(host=host, port=port, db=db)
version: "3.5"
services:
tester:
command: ["bash", "./run.sh", "development"]
version: "3.5"
services:
tester:
build: .
command: ["bash", "./docker-entry.sh"]
env_file: ${EXPECTED_ENV_FILE}
ports:
- "127.0.0.1:${PORT}:${PORT}"
volumes:
- .:/app
networks:
- mongodb
networks:
mongodb:
external:
name: shadowban-eu_mongodb
#/usr/bin/env bash
echo "Starting server..."
echo "--account-file $ACCOUNT_FILE"
echo "--cookie-dir $COOKIE_DIR"
echo "--log $LOG_FILE"
echo "--debug $DEBUG_FILE"
echo "--port "$PORT""
echo "--host "$HOST""
echo "--mongo-host $MONGO_HOST"
echo "--mongo-port $MONGO_PORT"
echo "--mongo-db $MONGO_DB"
echo "--twitter-auth-key $TWITTER_AUTH_KEY"
python3 -u ./backend.py \
--account-file $ACCOUNT_FILE \
--cookie-dir $COOKIE_DIR \
--log $LOG_FILE \
--debug $DEBUG_FILE \
--port "$PORT" \
--host "$HOST" \
--mongo-host $MONGO_HOST \
--mongo-port $MONGO_PORT \
--mongo-db $MONGO_DB \
--twitter-auth-key $TWITTER_AUTH_KEY
#!/usr/bin/env bash
echo -n "Looking for Python3: "
if ! hash python3; then
echo -n "\nPlease install Python3 to use this program!"
fi
echo "OK"
echo "Installing dependencies..."
pip3 install -r requirements.txt --no-cache-dir
echo -e "\n----------------------------"
echo "All done! \o/"
echo "Run 'PYTON_ENV=[development|prodcution] ./run.sh' to start the server!"
#/usr/bin/env bash
if [ "$1" != 'production' ] && [ "$1" != 'development' ]; then
echo "Please provide 'production' or 'development' as first argument"
echo "e.g. $ $0 development"
exit
fi
EXPECTED_ENV_FILE="./.env.$1"
if [ ! -f $EXPECTED_ENV_FILE ]; then
echo "Please provide a configuration file {$EXPECTED_ENV_FILE}!"
fi
echo "Using configuration from: $EXPECTED_ENV_FILE"
source $EXPECTED_ENV_FILE
echo "Listening on: $PORT"
PORT=$PORT EXPECTED_ENV_FILE=$EXPECTED_ENV_FILE docker-compose -f docker-compose.yml up
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment