Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
S
shadowban-eu-backend
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages
Packages
List
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issues
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nanahira
shadowban-eu-backend
Commits
acb3ca49
Commit
acb3ca49
authored
Dec 07, 2021
by
Your Name
Browse files
Options
Browse Files
Download
Plain Diff
merge
parents
c78216b0
4b625881
Changes
5
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
730 additions
and
19 deletions
+730
-19
LICENSE.md
LICENSE.md
+675
-0
backend.py
backend.py
+15
-12
db.py
db.py
+10
-7
statistics.py
statistics.py
+19
-0
typeahead.py
typeahead.py
+11
-0
No files found.
LICENSE.md
0 → 100644
View file @
acb3ca49
This diff is collapsed.
Click to expand it.
backend.py
View file @
acb3ca49
...
@@ -12,7 +12,10 @@ import time
...
@@ -12,7 +12,10 @@ import time
from
aiohttp
import
web
from
aiohttp
import
web
from
bs4
import
BeautifulSoup
from
bs4
import
BeautifulSoup
from
db
import
connect
from
db
import
Database
from
statistics
import
count_sensitives
from
typeahead
import
test
as
test_typeahead
# This is a public value from the Twitter source code.
# This is a public value from the Twitter source code.
...
@@ -208,9 +211,6 @@ class TwitterSession:
...
@@ -208,9 +211,6 @@ class TwitterSession:
additional_query
=
"&tweet_search_mode=live"
additional_query
=
"&tweet_search_mode=live"
return
await
self
.
get
(
"https://api.twitter.com/2/search/adaptive.json?q="
+
urllib
.
parse
.
quote
(
query
)
+
"&count=20&spelling_corrections=0"
+
additional_query
)
return
await
self
.
get
(
"https://api.twitter.com/2/search/adaptive.json?q="
+
urllib
.
parse
.
quote
(
query
)
+
"&count=20&spelling_corrections=0"
+
additional_query
)
async
def
typeahead_raw
(
self
,
query
):
return
await
self
.
get
(
"https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q="
+
urllib
.
parse
.
quote
(
query
))
async
def
profile_raw
(
self
,
username
):
async
def
profile_raw
(
self
,
username
):
return
await
self
.
get
(
"https://api.twitter.com/1.1/users/show.json?screen_name="
+
urllib
.
parse
.
quote
(
username
))
return
await
self
.
get
(
"https://api.twitter.com/1.1/users/show.json?screen_name="
+
urllib
.
parse
.
quote
(
username
))
...
@@ -429,6 +429,8 @@ class TwitterSession:
...
@@ -429,6 +429,8 @@ class TwitterSession:
if
not
profile
[
"exists"
]
or
profile
.
get
(
"suspended"
,
False
)
or
profile
.
get
(
"protected"
,
False
)
or
not
profile
.
get
(
'has_tweets'
):
if
not
profile
[
"exists"
]
or
profile
.
get
(
"suspended"
,
False
)
or
profile
.
get
(
"protected"
,
False
)
or
not
profile
.
get
(
'has_tweets'
):
return
result
return
result
result
[
"profile"
][
"sensitives"
]
=
await
count_sensitives
(
self
,
user_id
)
result
[
"tests"
]
=
{}
result
[
"tests"
]
=
{}
search_raw
=
await
self
.
search_raw
(
"from:@"
+
username
)
search_raw
=
await
self
.
search_raw
(
"from:@"
+
username
)
...
@@ -443,12 +445,7 @@ class TwitterSession:
...
@@ -443,12 +445,7 @@ class TwitterSession:
except
(
KeyError
,
IndexError
):
except
(
KeyError
,
IndexError
):
pass
pass
typeahead_raw
=
await
self
.
typeahead_raw
(
"@"
+
username
)
result
[
"tests"
][
"typeahead"
]
=
await
test_typeahead
(
self
,
username
)
result
[
"tests"
][
"typeahead"
]
=
False
try
:
result
[
"tests"
][
"typeahead"
]
=
len
([
1
for
user
in
typeahead_raw
[
"users"
]
if
user
[
"screen_name"
]
.
lower
()
==
username
.
lower
()])
>
0
except
KeyError
:
pass
if
"search"
in
result
[
"tests"
]
and
result
[
"tests"
][
"search"
]
==
False
:
if
"search"
in
result
[
"tests"
]
and
result
[
"tests"
][
"search"
]
==
False
:
result
[
"tests"
][
"ghost"
]
=
await
self
.
test_ghost_ban
(
user_id
)
result
[
"tests"
][
"ghost"
]
=
await
self
.
test_ghost_ban
(
user_id
)
...
@@ -520,6 +517,10 @@ async def unlocked(request):
...
@@ -520,6 +517,10 @@ async def unlocked(request):
async
def
api
(
request
):
async
def
api
(
request
):
global
test_index
global
test_index
screen_name
=
request
.
match_info
[
'screen_name'
]
screen_name
=
request
.
match_info
[
'screen_name'
]
if
screen_name
==
"wikileaks"
and
request
.
query_string
!=
"watch"
:
debug
(
"[wikileaks] Returning last watch result"
)
db_result
=
db
.
get_result_by_screen_name
(
"wikileaks"
)
return
web
.
json_response
(
db_result
,
headers
=
{
"Access-Control-Allow-Origin"
:
args
.
cors_allow
})
session
=
guest_sessions
[
test_index
%
len
(
guest_sessions
)]
session
=
guest_sessions
[
test_index
%
len
(
guest_sessions
)]
test_index
+=
1
test_index
+=
1
result
=
await
session
.
test
(
screen_name
)
result
=
await
session
.
test
(
screen_name
)
...
@@ -564,7 +565,9 @@ parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip w
...
@@ -564,7 +565,9 @@ parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip w
parser
.
add_argument
(
'--mongo-host'
,
type
=
str
,
default
=
None
,
help
=
'hostname or IP of mongoDB service to connect to'
)
parser
.
add_argument
(
'--mongo-host'
,
type
=
str
,
default
=
None
,
help
=
'hostname or IP of mongoDB service to connect to'
)
parser
.
add_argument
(
'--mongo-port'
,
type
=
int
,
default
=
27017
,
help
=
'port of mongoDB service to connect to'
)
parser
.
add_argument
(
'--mongo-port'
,
type
=
int
,
default
=
27017
,
help
=
'port of mongoDB service to connect to'
)
parser
.
add_argument
(
'--mongo-db'
,
type
=
str
,
default
=
'tester'
,
help
=
'name of mongo database to use'
)
parser
.
add_argument
(
'--mongo-db'
,
type
=
str
,
default
=
'tester'
,
help
=
'name of mongo database to use'
)
parser
.
add_argument
(
'--twitter-auth-key'
,
type
=
str
,
default
=
TWITTER_AUTH_KEY
,
help
=
'auth key for twitter guest session'
)
parser
.
add_argument
(
'--mongo-username'
,
type
=
str
,
default
=
''
,
help
=
'name of user in mongo database'
)
parser
.
add_argument
(
'--mongo-password'
,
type
=
str
,
default
=
''
,
help
=
'password for user in mongo database'
)
parser
.
add_argument
(
'--twitter-auth-key'
,
type
=
str
,
default
=
None
,
help
=
'auth key for twitter guest session'
,
required
=
True
)
parser
.
add_argument
(
'--cors-allow'
,
type
=
str
,
default
=
None
,
help
=
'value for Access-Control-Allow-Origin header'
)
parser
.
add_argument
(
'--cors-allow'
,
type
=
str
,
default
=
None
,
help
=
'value for Access-Control-Allow-Origin header'
)
args
,
unknown
=
parser
.
parse_known_args
()
args
,
unknown
=
parser
.
parse_known_args
()
...
@@ -604,7 +607,7 @@ def run():
...
@@ -604,7 +607,7 @@ def run():
if
args
.
mongo_host
is
not
None
:
if
args
.
mongo_host
is
not
None
:
db
=
connect
(
host
=
args
.
mongo_host
,
port
=
args
.
mongo_port
)
db
=
connect
(
host
=
args
.
mongo_host
,
port
=
args
.
mongo_port
)
loop
=
asyncio
.
get_event_loop
()
loop
=
asyncio
.
get_event_loop
()
loop
.
run_until_complete
(
login_accounts
(
accounts
,
args
.
cookie_dir
))
#
loop.run_until_complete(login_accounts(accounts, args.cookie_dir))
loop
.
run_until_complete
(
login_guests
())
loop
.
run_until_complete
(
login_guests
())
app
=
web
.
Application
()
app
=
web
.
Application
()
app
.
add_routes
(
routes
)
app
.
add_routes
(
routes
)
...
...
db.py
View file @
acb3ca49
import
copy
import
copy
import
traceback
import
traceback
import
sys
import
sys
from
pymongo
import
MongoClient
,
errors
as
MongoErrors
from
pymongo
import
MongoClient
,
errors
as
MongoErrors
,
DESCENDING
class
Database
:
class
Database
:
def
__init__
(
self
,
host
=
None
,
port
=
27017
,
db
=
'tester'
):
def
__init__
(
self
,
host
=
None
,
port
=
27017
,
username
=
''
,
password
=
''
,
db
=
'tester'
):
# collection name definitions
# collection name definitions
RESULTS_COLLECTION
=
'results'
RESULTS_COLLECTION
=
'results'
RATELIMIT_COLLECTION
=
'rate-limits'
RATELIMIT_COLLECTION
=
'rate-limits'
...
@@ -13,7 +13,12 @@ class Database:
...
@@ -13,7 +13,12 @@ class Database:
print
(
'[mongoDB] Connecting to '
+
host
+
':'
+
str
(
port
))
print
(
'[mongoDB] Connecting to '
+
host
+
':'
+
str
(
port
))
print
(
'[mongoDB] Using Database `'
+
db
+
'`'
)
print
(
'[mongoDB] Using Database `'
+
db
+
'`'
)
# client and DB
# client and DB
self
.
client
=
MongoClient
(
host
,
port
,
serverSelectionTimeoutMS
=
3
)
self
.
client
=
MongoClient
(
host
=
host
,
port
=
port
,
username
=
username
,
password
=
password
,
serverSelectionTimeoutMS
=
3
)
self
.
db
=
self
.
client
[
db
]
self
.
db
=
self
.
client
[
db
]
# collections
# collections
...
@@ -38,8 +43,6 @@ class Database:
...
@@ -38,8 +43,6 @@ class Database:
def
write_rate_limit
(
self
,
data
):
def
write_rate_limit
(
self
,
data
):
self
.
rate_limits
.
insert_one
(
data
)
self
.
rate_limits
.
insert_one
(
data
)
def
connect
(
host
=
None
,
port
=
27017
,
db
=
'tester'
):
def
get_result_by_screen_name
(
self
,
screen_name
):
if
host
is
None
:
return
self
.
results
.
find_one
({
"profile.screen_name"
:
screen_name
},
sort
=
[(
"_id"
,
DESCENDING
)],
projection
=
{
"_id"
:
False
})
raise
ValueError
(
'[mongoDB] Database constructor needs a `host`name or ip!'
)
return
Database
(
host
=
host
,
port
=
port
,
db
=
db
)
statistics.py
0 → 100644
View file @
acb3ca49
import
sys
# Count amount of "possibly_sensitive_editable" and "possibly_sensitive"
# flagged tweets in user's timeline
async
def
count_sensitives
(
session
,
user_id
):
profile_timeline
=
await
session
.
get_profile_tweets_raw
(
user_id
)
profile_tweets
=
profile_timeline
[
"globalObjects"
][
"tweets"
]
.
values
()
counted
=
len
(
profile_tweets
)
possibly_sensitive
=
len
([
1
for
tweet
in
profile_tweets
if
"possibly_sensitive"
in
tweet
.
keys
()])
possibly_sensitive_editable
=
len
([
1
for
tweet
in
profile_tweets
if
"possibly_sensitive_editable"
in
tweet
.
keys
()])
result
=
{
"counted"
:
counted
,
"possibly_sensitive"
:
possibly_sensitive
,
"possibly_sensitive_editable"
:
possibly_sensitive_editable
}
return
result
typeahead.py
0 → 100644
View file @
acb3ca49
import
urllib.parse
async
def
test
(
session
,
screen_name
):
query
=
"@"
+
screen_name
suggestions
=
await
session
.
get
(
"https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q="
+
urllib
.
parse
.
quote
(
query
))
try
:
result
=
len
([
1
for
user
in
suggestions
[
"users"
]
if
user
[
"screen_name"
]
.
lower
()
==
screen_name
.
lower
()])
>
0
except
:
result
=
False
return
result
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment