Commit acb3ca49 authored by Your Name's avatar Your Name

merge

parents c78216b0 4b625881
This diff is collapsed.
...@@ -12,7 +12,10 @@ import time ...@@ -12,7 +12,10 @@ import time
from aiohttp import web from aiohttp import web
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from db import connect
from db import Database
from statistics import count_sensitives
from typeahead import test as test_typeahead
# This is a public value from the Twitter source code. # This is a public value from the Twitter source code.
...@@ -208,9 +211,6 @@ class TwitterSession: ...@@ -208,9 +211,6 @@ class TwitterSession:
additional_query = "&tweet_search_mode=live" additional_query = "&tweet_search_mode=live"
return await self.get("https://api.twitter.com/2/search/adaptive.json?q="+urllib.parse.quote(query)+"&count=20&spelling_corrections=0" + additional_query) return await self.get("https://api.twitter.com/2/search/adaptive.json?q="+urllib.parse.quote(query)+"&count=20&spelling_corrections=0" + additional_query)
async def typeahead_raw(self, query):
return await self.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query))
async def profile_raw(self, username): async def profile_raw(self, username):
return await self.get("https://api.twitter.com/1.1/users/show.json?screen_name=" + urllib.parse.quote(username)) return await self.get("https://api.twitter.com/1.1/users/show.json?screen_name=" + urllib.parse.quote(username))
...@@ -429,6 +429,8 @@ class TwitterSession: ...@@ -429,6 +429,8 @@ class TwitterSession:
if not profile["exists"] or profile.get("suspended", False) or profile.get("protected", False) or not profile.get('has_tweets'): if not profile["exists"] or profile.get("suspended", False) or profile.get("protected", False) or not profile.get('has_tweets'):
return result return result
result["profile"]["sensitives"] = await count_sensitives(self, user_id)
result["tests"] = {} result["tests"] = {}
search_raw = await self.search_raw("from:@" + username) search_raw = await self.search_raw("from:@" + username)
...@@ -443,12 +445,7 @@ class TwitterSession: ...@@ -443,12 +445,7 @@ class TwitterSession:
except (KeyError, IndexError): except (KeyError, IndexError):
pass pass
typeahead_raw = await self.typeahead_raw("@" + username) result["tests"]["typeahead"] = await test_typeahead(self, username)
result["tests"]["typeahead"] = False
try:
result["tests"]["typeahead"] = len([1 for user in typeahead_raw["users"] if user["screen_name"].lower() == username.lower()]) > 0
except KeyError:
pass
if "search" in result["tests"] and result["tests"]["search"] == False: if "search" in result["tests"] and result["tests"]["search"] == False:
result["tests"]["ghost"] = await self.test_ghost_ban(user_id) result["tests"]["ghost"] = await self.test_ghost_ban(user_id)
...@@ -520,6 +517,10 @@ async def unlocked(request): ...@@ -520,6 +517,10 @@ async def unlocked(request):
async def api(request): async def api(request):
global test_index global test_index
screen_name = request.match_info['screen_name'] screen_name = request.match_info['screen_name']
if screen_name == "wikileaks" and request.query_string != "watch":
debug("[wikileaks] Returning last watch result")
db_result = db.get_result_by_screen_name("wikileaks")
return web.json_response(db_result, headers={"Access-Control-Allow-Origin": args.cors_allow})
session = guest_sessions[test_index % len(guest_sessions)] session = guest_sessions[test_index % len(guest_sessions)]
test_index += 1 test_index += 1
result = await session.test(screen_name) result = await session.test(screen_name)
...@@ -564,7 +565,9 @@ parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip w ...@@ -564,7 +565,9 @@ parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip w
parser.add_argument('--mongo-host', type=str, default=None, help='hostname or IP of mongoDB service to connect to') parser.add_argument('--mongo-host', type=str, default=None, help='hostname or IP of mongoDB service to connect to')
parser.add_argument('--mongo-port', type=int, default=27017, help='port of mongoDB service to connect to') parser.add_argument('--mongo-port', type=int, default=27017, help='port of mongoDB service to connect to')
parser.add_argument('--mongo-db', type=str, default='tester', help='name of mongo database to use') parser.add_argument('--mongo-db', type=str, default='tester', help='name of mongo database to use')
parser.add_argument('--twitter-auth-key', type=str, default=TWITTER_AUTH_KEY, help='auth key for twitter guest session') parser.add_argument('--mongo-username', type=str, default='', help='name of user in mongo database')
parser.add_argument('--mongo-password', type=str, default='', help='password for user in mongo database')
parser.add_argument('--twitter-auth-key', type=str, default=None, help='auth key for twitter guest session', required=True)
parser.add_argument('--cors-allow', type=str, default=None, help='value for Access-Control-Allow-Origin header') parser.add_argument('--cors-allow', type=str, default=None, help='value for Access-Control-Allow-Origin header')
args, unknown = parser.parse_known_args() args, unknown = parser.parse_known_args()
...@@ -604,7 +607,7 @@ def run(): ...@@ -604,7 +607,7 @@ def run():
if args.mongo_host is not None: if args.mongo_host is not None:
db = connect(host=args.mongo_host, port=args.mongo_port) db = connect(host=args.mongo_host, port=args.mongo_port)
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete(login_accounts(accounts, args.cookie_dir)) # loop.run_until_complete(login_accounts(accounts, args.cookie_dir))
loop.run_until_complete(login_guests()) loop.run_until_complete(login_guests())
app = web.Application() app = web.Application()
app.add_routes(routes) app.add_routes(routes)
......
import copy import copy
import traceback import traceback
import sys import sys
from pymongo import MongoClient, errors as MongoErrors from pymongo import MongoClient, errors as MongoErrors, DESCENDING
class Database: class Database:
def __init__(self, host=None, port=27017, db='tester'): def __init__(self, host=None, port=27017, username='', password='', db='tester'):
# collection name definitions # collection name definitions
RESULTS_COLLECTION = 'results' RESULTS_COLLECTION = 'results'
RATELIMIT_COLLECTION = 'rate-limits' RATELIMIT_COLLECTION = 'rate-limits'
...@@ -13,7 +13,12 @@ class Database: ...@@ -13,7 +13,12 @@ class Database:
print('[mongoDB] Connecting to ' + host + ':' + str(port)) print('[mongoDB] Connecting to ' + host + ':' + str(port))
print('[mongoDB] Using Database `' + db + '`') print('[mongoDB] Using Database `' + db + '`')
# client and DB # client and DB
self.client = MongoClient(host, port, serverSelectionTimeoutMS=3) self.client = MongoClient(
host=host,
port=port,
username=username,
password=password,
serverSelectionTimeoutMS=3)
self.db = self.client[db] self.db = self.client[db]
# collections # collections
...@@ -38,8 +43,6 @@ class Database: ...@@ -38,8 +43,6 @@ class Database:
def write_rate_limit(self, data): def write_rate_limit(self, data):
self.rate_limits.insert_one(data) self.rate_limits.insert_one(data)
def connect(host=None, port=27017, db='tester'): def get_result_by_screen_name(self, screen_name):
if host is None: return self.results.find_one({ "profile.screen_name": screen_name }, sort=[("_id", DESCENDING)], projection={"_id": False})
raise ValueError('[mongoDB] Database constructor needs a `host`name or ip!')
return Database(host=host, port=port, db=db)
import sys
# Count amount of "possibly_sensitive_editable" and "possibly_sensitive"
# flagged tweets in user's timeline
async def count_sensitives(session, user_id):
profile_timeline = await session.get_profile_tweets_raw(user_id)
profile_tweets = profile_timeline["globalObjects"]["tweets"].values()
counted = len(profile_tweets)
possibly_sensitive = len([1 for tweet in profile_tweets if "possibly_sensitive" in tweet.keys()])
possibly_sensitive_editable = len([1 for tweet in profile_tweets if "possibly_sensitive_editable" in tweet.keys()])
result = {
"counted": counted,
"possibly_sensitive": possibly_sensitive,
"possibly_sensitive_editable": possibly_sensitive_editable
}
return result
import urllib.parse
async def test(session, screen_name):
query = "@" + screen_name
suggestions = await session.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query))
try:
result = len([1 for user in suggestions["users"] if user["screen_name"].lower() == screen_name.lower()]) > 0
except:
result = False
return result
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment