Merge branch 'rls/1.0.1'

.env.example

+ACCOUNT_FILE=./.htaccounts
+COOKIE_DIR=./.htcookies
+LOG_FILE=./logs/results.log
+DEBUG_FILE=./logs/debug.log
+PORT=4040
+HOST=127.0.0.1
+MONGO_HOST=127.0.0.1
+MONGO_PORT=27017
+MONGO_DB=tester
+TWITTER_AUTH_KEY=GRAPHQL_KEY

.gitignore

@@ -52,4 +52,5 @@ node_modules
 # .env
 .env
 .env.*
+!.env.example
 .ht*

Dockerfile

+FROM python:3.5.7-slim-buster
+
+RUN mkdir /app
+WORKDIR /app
+ADD requirements.txt /app/
+ADD . /app
+
+RUN pip3 install --no-cache-dir -r ./requirements.txt

README

+Pull
+```
+git clone https://github.com/shadowban-eu/shadowban-eu-backend && cd shadowban-eu-backend
+```
+
+Add config
+```
+cp ~/configs/.env.development|production ./
+cp ~/configs/.htaccounts
+```
+
+Run
+`./run.sh development|production`
+
+See [.env.example](https://github.com/shadowban-eu/shadowban-eu-backend/blob/dev/.env.example) for .env variables

backend.py

 import aiohttp
 import argparse
 import asyncio
+import daemon
 import json
+import os
 import re
 import traceback
 import urllib.parse
@@ -13,6 +15,9 @@ from db import connect
 
 routes = web.RouteTableDef()
 
+class UnexpectedApiError(Exception):
+    pass
+
 def get_nested(obj, path, default=None):
     for p in path:
         if obj is None or not p in obj:
@@ -20,14 +25,34 @@ def get_nested(obj, path, default=None):
         obj = obj[p]
     return obj
 
+def is_error(result, code=None):
+    return isinstance(result.get("errors", None), list) and (len([x for x in result["errors"] if x.get("code", None) == code]) > 0 or code is None and len(result["errors"] > 0))
+
+def is_another_error(result, codes):
+    return isinstance(result.get("errors", None), list) and len([x for x in result["errors"] if x.get("code", None) not in codes]) > 0
+
 account_sessions = []
 account_index = 0
 log_file = None
 debug_file = None
+guest_session_pool_size = 10
+guest_sessions = []
+test_index = 0
+
+def next_session():
+    def key(s):
+        remaining_time = s.reset - time.time()
+        if s.remaining <= 3 and remaining_time > 0:
+            return 900
+        return remaining_time
+    sessions = sorted([s for s in account_sessions if not s.locked], key=key)
+    if len(sessions) > 0:
+        return sessions[0]
 
 class TwitterSession:
-    _auth = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
     _user_url = "https://api.twitter.com/graphql/SEn6Mq-OakvVOT1CJqUO2A/UserByScreenName?variables="
+    twitter_auth_key = None
+
     def __init__(self):
         self._headers = {
             "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
@@ -41,9 +66,11 @@ class TwitterSession:
 
         # rate limit monitoring
         self.limit = -1
-        self.remaining = -1
+        self.remaining = 180
         self.reset = -1
         self.overshot = -1
+        self.locked = False
+        self.next_refresh = None
 
         # session user's @username
         # this stays `None` for guest sessions
@@ -55,90 +82,156 @@ class TwitterSession:
             if cookie.key == 'ct0':
                 self._headers['X-Csrf-Token'] = cookie.value
 
-    async def login(self, username = None, password = None, email = None):
+    async def get_guest_token(self):
+        self._headers['Authorization'] = 'Bearer ' + self.twitter_auth_key
+        async with self._session.post("https://api.twitter.com/1.1/guest/activate.json", headers=self._headers) as r:
+            response = await r.json()
+        guest_token = response.get("guest_token", None)
+        if guest_token is None:
+            debug("Failed to fetch guest token")
+        return guest_token
+
+    async def renew_session(self):
+        await self.try_close()
+        self._session = aiohttp.ClientSession()
+
+    async def refresh_old_token(self):
+        if self.username is not None or self.next_refresh is None or time.time() < self.next_refresh:
+            return
+        debug("Refresh token: " + str(self._guest_token))
+        await self.login_guest()
+        debug("New token: " + str(self._guest_token))
+
+    async def try_close(self):
+        if self._session is not None:
+            try:
+                await self._session.close()
+            except:
+                pass
+
+    async def login_guest(self):
+        await self.renew_session()
+        self.set_csrf_header()
+        old_token = self._guest_token
+        new_token = await self.get_guest_token()
+        self._guest_token = new_token if new_token is not None else old_token
+        if new_token is not None:
+            self.next_refresh = time.time() + 3600
+        self._headers['X-Guest-Token'] = self._guest_token
+
+
+    async def login(self, username = None, password = None, email = None, cookie_dir=None):
         self._session = aiohttp.ClientSession()
 
         if password is not None:
-            async with self._session.get("https://twitter.com/login", headers=self._headers) as r:
-                login_page = await r.text()
-            form_data = {}
-            soup = BeautifulSoup(login_page, 'html.parser')
-            form_data["authenticity_token"] = soup.find('input', {'name': 'authenticity_token'}).get('value')
-            form_data["session[username_or_email]"] = email
-            form_data["session[password]"] = password
-            form_data["remember_me"] = "1"
-            async with self._session.post('https://twitter.com/sessions', data=form_data, headers=self._headers) as r:
-                await r.text()
-                if str(r.url) == "https://twitter.com/":
-                    log("Login of %s successful" % username)
-                else:
-                    log("Error logging in %s" % username)
+            login_required = True
+            cookie_file = None
+            if cookie_dir is not None:
+                cookie_file = os.path.join(cookie_dir, username)
+                if os.path.isfile(cookie_file):
+                    log("Use cookie file for %s" % username)
+                    self._session.cookie_jar.load(cookie_file)
+                    login_required = False
+
+            store_cookies = True
+
+            if login_required:
+                async with self._session.get("https://twitter.com/login", headers=self._headers) as r:
+                    login_page = await r.text()
+                form_data = {}
+                soup = BeautifulSoup(login_page, 'html.parser')
+                form_data["authenticity_token"] = soup.find('input', {'name': 'authenticity_token'}).get('value')
+                form_data["session[username_or_email]"] = email
+                form_data["session[password]"] = password
+                form_data["remember_me"] = "1"
+                async with self._session.post('https://twitter.com/sessions', data=form_data, headers=self._headers) as r:
+                    response = await r.text()
+                    if str(r.url) == "https://twitter.com/":
+                        log("Login of %s successful" % username)
+                    else:
+                        store_cookies = False
+                        log("Error logging in %s (%s)" % (username, r.url))
+                        debug("ERROR PAGE\n" + response)
+            else:
+                async with self._session.get('https://twitter.com', headers=self._headers) as r:
+                    await r.text()
+
             self.set_csrf_header()
             self.username = username
+
+            if cookie_file is not None and store_cookies:
+                self._session.cookie_jar.save(cookie_file)
+
         else:
-            self._headers['Authorization'] = 'Bearer ' + self._auth
-            async with self._session.post("https://api.twitter.com/1.1/guest/activate.json", headers=self._headers) as r:
-                guest_token = await r.json()
-            self._guest_token = guest_token["guest_token"]
-            self._headers['X-Guest-Token'] = self._guest_token
+            await self.login_guest()
 
-        self._headers['Authorization'] = 'Bearer ' + self._auth
+        self._headers['Authorization'] = 'Bearer ' + self.twitter_auth_key
+
+    async def get(self, url, retries=0):
+        self.set_csrf_header()
+        await self.refresh_old_token()
+        try:
+            async with self._session.get(url, headers=self._headers) as r:
+                result = await r.json()
+        except Exception as e:
+            debug("EXCEPTION: " + str(type(e)))
+            if self.username is None:
+                await self.login_guest()
+            raise e
+        self.monitor_rate_limit(r.headers)
+        if self.username is None and self.remaining < 10 or is_error(result, 88) or is_error(result, 239):
+            await self.login_guest()
+        if retries > 0 and is_error(result, 353):
+            return await self.get(url, retries - 1)
+        if is_error(result, 326):
+            self.locked = True
+        return result
 
     async def search_raw(self, query, live=True):
         additional_query = ""
         if live:
             additional_query = "&tweet_search_mode=live"
-        async with self._session.get("https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_composer_source=true&include_ext_alt_text=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&q="+urllib.parse.quote(query)+"&qf_abuse=false&count=20&query_source=typed_query&pc=1&spelling_corrections=0&ext=mediaStats%2ChighlightedLabel%2CcameraMoment" + additional_query, headers=self._headers) as r:
-            return await r.json()
+        return await self.get("https://api.twitter.com/2/search/adaptive.json?q="+urllib.parse.quote(query)+"&count=20&spelling_corrections=0" + additional_query)
 
     async def typeahead_raw(self, query):
-        async with self._session.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query), headers=self._headers) as r:
-            return await r.json()
+        return await self.get("https://api.twitter.com/1.1/search/typeahead.json?src=search_box&result_type=users&q=" + urllib.parse.quote(query))
 
     async def profile_raw(self, username):
         obj = json.dumps({"screen_name": username, "withHighlightedLabel": True})
-        async with self._session.get(self._user_url + urllib.parse.quote(obj), headers=self._headers) as r:
-            return await r.json()
+        return await self.get(self._user_url + urllib.parse.quote(obj))
 
     async def get_profile_tweets_raw(self, user_id):
-        async with self._session.get("https://api.twitter.com/2/timeline/profile/" + str(user_id) +".json?include_tweet_replies=1&include_want_retweets=0&include_reply_count=1&count=1000", headers=self._headers) as r:
-            return await r.json()
-
-    async def test_detached_tweets():
-        pass
+        return await self.get("https://api.twitter.com/2/timeline/profile/" + str(user_id) +".json?include_tweet_replies=1&include_want_retweets=0&include_reply_count=1&count=1000")
 
     async def tweet_raw(self, tweet_id, count=20, cursor=None, retry_csrf=True):
         if cursor is None:
             cursor = ""
         else:
             cursor = "&cursor=" + urllib.parse.quote(cursor)
-        async with self._session.get("https://api.twitter.com/2/timeline/conversation/" + tweet_id + ".json?include_reply_count=1&send_error_codes=true&count="+str(count)+ cursor, headers=self._headers) as r:
-            result = await r.json()
-            # debug('Tweet request ' + tweet_id + ':\n' + str(r) + '\n\n' + json.dumps(result) + '\n\n\n')
-            self.set_csrf_header()
-            if self.screen_name is not None:
-                self.monitor_rate_limit(r.headers)
-            if retry_csrf and isinstance(result.get("errors", None), list) and len([x for x in result["errors"] if x.get("code", None) == 353]):
-                return await self.tweet_raw(tweet_id, count, cursor, False)
-            return result
+        return await self.get("https://api.twitter.com/2/timeline/conversation/" + tweet_id + ".json?include_reply_count=1&send_error_codes=true&count="+str(count)+ cursor)
 
     def monitor_rate_limit(self, headers):
         # store last remaining count for reset detection
         last_remaining = self.remaining
-
-        self.limit = int(headers.get('x-rate-limit-limit', -1))
-        self.remaining = int(headers.get('x-rate-limit-remaining', -1))
-        self.reset = int(headers.get('x-rate-limit-reset', -1))
+        limit = headers.get('x-rate-limit-limit', None)
+        remaining = headers.get('x-rate-limit-remaining', None)
+        reset = headers.get('x-rate-limit-reset', None)
+        if limit is not None:
+            self.limit = int(limit)
+        if remaining is not None:
+            self.remaining = int(remaining)
+        if reset is not None:
+            self.reset = int(reset)
 
         # rate limit reset
-        if last_remaining < self.remaining and self.overshot > 0:
-            log('[rate-limit] Reset detected for ' + self.screen_name + '. Saving overshoot count...')
-            db.write_rate_limit({ 'screen_name': self.screen_name, 'overshot': self.overshot })
+        if last_remaining < self.remaining and self.overshot > 0 and self.username is not None:
+            log('[rate-limit] Reset detected for ' + self.username + '. Saving overshoot count...')
+            db.write_rate_limit({ 'screen_name': self.username, 'overshot': self.overshot })
             self.overshot = 0
 
         # count the requests that failed because of rate limiting
         if self.remaining is 0:
-            log('[rate-limit] Limit hit by ' + self.screen_name + '.')
+            log('[rate-limit] Limit hit by ' + str(self.username) + '.')
             self.overshot += 1
 
     @classmethod
@@ -230,9 +323,11 @@ class TwitterSession:
                 debug('Found:' + tid + '\n')
                 debug('In reply to:' + replied_to_id + '\n')
 
-                global account_sessions
+                reference_session = next_session()
+                if reference_session is None:
+                    return
+
                 global account_index
-                reference_session = account_sessions[account_index % len(account_sessions)]
                 account_index += 1
 
                 before_barrier = await reference_session.tweet_raw(replied_to_id, 1000)
@@ -272,10 +367,13 @@ class TwitterSession:
             debug(traceback.format_exc())
 
     async def test(self, username, more_replies_test=True):
-        await self.login()
         result = {"timestamp": time.time()}
         profile = {}
         profile_raw = await self.profile_raw(username)
+        debug(str(profile_raw))
+        if is_another_error(profile_raw, [50, 63]):
+            debug("Other error:" + str(username))
+            raise UnexpectedApiError
 
         try:
             user_id = str(profile_raw["data"]["user"]["rest_id"])
@@ -296,14 +394,10 @@ class TwitterSession:
             profile["protected"] = profile_raw["data"]["user"]["legacy"]["protected"]
         except KeyError:
             pass
-        try:
-            profile["exists"] = len([1 for error in profile_raw["errors"] if error["code"] == 50]) == 0
-        except KeyError:
-            profile["exists"] = True
-        try:
-            profile["suspended"] = len([1 for error in profile_raw["errors"] if error["code"] == 63]) > 0
-        except KeyError:
-            pass
+        profile["exists"] = not is_error(profile_raw, 50)
+        suspended = is_error(profile_raw, 63)
+        if suspended:
+            profile["suspended"] = suspended
         try:
             profile["has_tweets"] = int(profile_raw["data"]["user"]["legacy"]["statuses_count"]) > 0
         except KeyError:
@@ -343,8 +437,7 @@ class TwitterSession:
         if more_replies_test and not get_nested(result, ["tests", "ghost", "ban"], False):
             result["tests"]["more_replies"] = await self.test_barrier(user_id)
 
-        debug('Writing result for ' + result['profile']['screen_name'] + ' to DB');
-        global db
+        debug('Writing result for ' + result['profile']['screen_name'] + ' to DB')
         db.write_result(result)
         return result
 
@@ -353,7 +446,6 @@ class TwitterSession:
         await self._session.close()
 
 def debug(message):
-    global debug_file
     if message.endswith('\n') is False:
         message = message + '\n'
 
@@ -364,7 +456,6 @@ def debug(message):
         print(message)
 
 def log(message):
-    global log_file
     # ensure newline
     if message.endswith('\n') is False:
          message = message + '\n'
@@ -375,56 +466,108 @@ def log(message):
     else:
         print(message)
 
+def print_session_info(sessions):
+    text = ""
+    for session in sessions:
+        text += "\n%6d %5d %9d %5d" % (int(session.locked), session.limit, session.remaining, session.reset - int(time.time()))
+    return text
+
+@routes.get('/.stats')
+async def stats(request):
+    text = "--- GUEST SESSIONS ---\n\nLocked Limit Remaining Reset"
+    text += print_session_info(guest_sessions)
+    text += "\n\n\n--- ACCOUNTS ---\n\nLocked Limit Remaining Reset"
+    text += print_session_info(account_sessions)
+    return web.Response(text=text)
+
+@routes.get('/.unlocked/{screen_name}')
+async def unlocked(request):
+    screen_name = request.match_info['screen_name']
+    text = "Not unlocked"
+    for session in account_sessions:
+        if session.username.lower() != screen_name.lower():
+            continue
+        session.locked = False
+        text = "Unlocked"
+    return web.Response(text=text)
+
+
 @routes.get('/{screen_name}')
-async def hello(request):
+async def api(request):
+    global test_index
     screen_name = request.match_info['screen_name']
-    if screen_name == '.stats':
-        text = "Limit Remaining Reset"
-        for session in account_sessions:
-            text += "\n%5d %9d %5d" % (session.limit, session.remaining, session.reset - int(time.time()))
-        return web.Response(text=text)
-    session = TwitterSession()
+    session = guest_sessions[test_index % len(guest_sessions)]
+    test_index += 1
     result = await session.test(screen_name)
     log(json.dumps(result) + '\n')
-    await session.close()
     return web.json_response(result)
 
-async def login_accounts(accounts):
+async def login_accounts(accounts, cookie_dir=None):
+    if cookie_dir is not None and not os.path.isdir(cookie_dir):
+        os.mkdir(cookie_dir, 0o700)
     coroutines = []
     for acc in accounts:
         session = TwitterSession()
-        coroutines.append(session.login(*acc))
+        coroutines.append(session.login(*acc, cookie_dir=cookie_dir))
         account_sessions.append(session)
     await asyncio.gather(*coroutines)
 
+async def login_guests():
+    for i in range(0, guest_session_pool_size):
+        session = TwitterSession()
+        guest_sessions.append(session)
+    await asyncio.gather(*[s.login() for s in guest_sessions])
+    log("Guest sessions created")
+
+def ensure_dir(path):
+    if os.path.isdir(path) is False:
+        print('Creating directory %s' % path)
+        os.mkdir(path)
 
 parser = argparse.ArgumentParser(description='Twitter Shadowban Tester')
 parser.add_argument('--account-file', type=str, default='.htaccounts', help='json file with reference account credentials')
+parser.add_argument('--cookie-dir', type=str, default=None, help='directory for session account storage')
 parser.add_argument('--log', type=str, default=None, help='log file where test results are written to')
+parser.add_argument('--daemon', action='store_true', help='run in background')
 parser.add_argument('--debug', type=str, default=None, help='debug log file')
 parser.add_argument('--port', type=int, default=8080, help='port which to listen on')
+parser.add_argument('--host', type=str, default='127.0.0.1', help='hostname/ip which to listen on')
 parser.add_argument('--mongo-host', type=str, default='localhost', help='hostname or IP of mongoDB service to connect to')
 parser.add_argument('--mongo-port', type=int, default=27017, help='port of mongoDB service to connect to')
 parser.add_argument('--mongo-db', type=str, default='tester', help='name of mongo database to use')
-parser.add_argument('--mongo-collection', type=str, default='results', help='name of collection to save test results to')
+parser.add_argument('--twitter-auth-key', type=str, default=None, help='auth key for twitter guest session', required=True)
 args = parser.parse_args()
 
-db = connect(host=args.mongo_host, port=args.mongo_port)
+TwitterSession.twitter_auth_key = args.twitter_auth_key;
+ensure_dir(args.cookie_dir)
 
 with open(args.account_file, "r") as f:
     accounts = json.loads(f.read())
 
 if args.log is not None:
-    print("Logging test results to %s", args.log)
+    print("Logging test results to %s" % args.log)
+    log_dir = os.path.dirname(args.log)
+    ensure_dir(log_dir)
     log_file = open(args.log, "a")
 
 if args.debug is not None:
-    print("Logging debug output to %s", args.debug)
+    print("Logging debug output to %s" % args.debug)
+    debug_dir = os.path.dirname(args.debug)
+    ensure_dir(debug_dir)
     debug_file = open(args.debug, "a")
 
-
-loop = asyncio.get_event_loop()
-loop.run_until_complete(login_accounts(accounts))
-app = web.Application()
-app.add_routes(routes)
-web.run_app(app, host='127.0.0.1', port=args.port)
+def run():
+    global db
+    db = connect(host=args.mongo_host, port=args.mongo_port)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(login_accounts(accounts, args.cookie_dir))
+    loop.run_until_complete(login_guests())
+    app = web.Application()
+    app.add_routes(routes)
+    web.run_app(app, host=args.host, port=args.port)
+
+if args.daemon:
+    with daemon.DaemonContext():
+        run()
+else:
+    run()

db.py

@@ -4,18 +4,23 @@ import sys
 from pymongo import MongoClient, errors as MongoErrors
 
 class Database:
-    def __init__(self, host=None, port=27017, db='tester', collection_name='results'):
+    def __init__(self, host=None, port=27017, db='tester'):
+        # collection name definitions
+        RESULTS_COLLECTION = 'results'
+        RATELIMIT_COLLECTION = 'rate-limits'
+
         try:
             print('[mongoDB] Connecting to ' + host + ':' + str(port))
-            print('[mongoDB] Using Collection `' + collection_name + '` in Database `' + db + '`')
+            print('[mongoDB] Using Database `' + db + '`')
+            # client and DB
             self.client = MongoClient(host, port, serverSelectionTimeoutMS=3)
             self.db = self.client[db]
-            # collection for test results
-            self.results = self.db[collection_name]
-            # collection for rate limit monitoring
-            self.rate_limits = self.db['rate-limits']
 
-            # test connection immediately, instead of
+            # collections
+            self.results = self.db[RESULTS_COLLECTION]
+            self.rate_limits = self.db[RATELIMIT_COLLECTION]
+
+            # Test connection immediately, instead of
             # when trying to write in a request, later.
             self.client.admin.command('ismaster')
         except MongoErrors.ServerSelectionTimeoutError:
@@ -33,8 +38,8 @@ class Database:
     def write_rate_limit(self, data):
         self.rate_limits.insert_one(data)
 
-def connect(host=None, port=27017, db='tester', collection_name='results'):
+def connect(host=None, port=27017, db='tester'):
     if host is None:
         raise ValueError('[mongoDB] Database constructor needs a `host`name or ip!')
 
-    return Database(host=host, port=port, db=db, collection_name=collection_name)
+    return Database(host=host, port=port, db=db)

docker-compose.development.yml

+version: "3.5"
+services:
+  tester:
+    command: ["bash", "./run.sh", "development"]

docker-compose.yml

+version: "3.5"
+services:
+  tester:
+    build: .
+    command: ["bash", "./docker-entry.sh"]
+    env_file: ${EXPECTED_ENV_FILE}
+    ports:
+      - "127.0.0.1:${PORT}:${PORT}"
+    volumes:
+      - .:/app
+    networks:
+      - mongodb
+
+networks:
+  mongodb:
+    external:
+      name: shadowban-eu_mongodb

docker-entry.sh

+#/usr/bin/env bash
+
+echo "Starting server..."
+echo "--account-file $ACCOUNT_FILE"
+echo "--cookie-dir $COOKIE_DIR"
+echo "--log $LOG_FILE"
+echo "--debug $DEBUG_FILE"
+echo "--port "$PORT""
+echo "--host "$HOST""
+echo "--mongo-host $MONGO_HOST"
+echo "--mongo-port $MONGO_PORT"
+echo "--mongo-db $MONGO_DB"
+echo "--twitter-auth-key $TWITTER_AUTH_KEY"
+
+python3 -u ./backend.py \
+  --account-file $ACCOUNT_FILE \
+  --cookie-dir $COOKIE_DIR \
+  --log $LOG_FILE \
+  --debug $DEBUG_FILE \
+  --port "$PORT" \
+  --host "$HOST" \
+  --mongo-host $MONGO_HOST \
+  --mongo-port $MONGO_PORT \
+  --mongo-db $MONGO_DB \
+  --twitter-auth-key $TWITTER_AUTH_KEY

install.sh

+#!/usr/bin/env bash
+
+echo -n "Looking for Python3: "
+if ! hash python3; then
+  echo -n "\nPlease install Python3 to use this program!"
+fi
+echo "OK"
+
+echo "Installing dependencies..."
+pip3 install -r requirements.txt --no-cache-dir
+
+echo -e "\n----------------------------"
+echo "All done! \o/"
+echo "Run 'PYTON_ENV=[development|prodcution] ./run.sh' to start the server!"

requirements.txt

 aiohttp
 pymongo
 bs4
+daemon

run.sh

+#/usr/bin/env bash
+
+if [ "$1" != 'production' ] && [ "$1" != 'development' ]; then
+  echo "Please provide 'production' or 'development' as first argument"
+  echo "e.g. $ $0 development"
+  exit
+fi
+
+
+EXPECTED_ENV_FILE="./.env.$1"
+if [ ! -f $EXPECTED_ENV_FILE ]; then
+  echo "Please provide a configuration file {$EXPECTED_ENV_FILE}!"
+fi
+echo "Using configuration from: $EXPECTED_ENV_FILE"
+source $EXPECTED_ENV_FILE
+echo "Listening on: $PORT"
+
+PORT=$PORT EXPECTED_ENV_FILE=$EXPECTED_ENV_FILE docker-compose -f docker-compose.yml up