Commit 144a228e authored by Francesco Poldi's avatar Francesco Poldi

Added new requirement, fixed proxy, added proxy config

parent 5f73d332
...@@ -5,4 +5,5 @@ cchardet ...@@ -5,4 +5,5 @@ cchardet
elasticsearch elasticsearch
pysocks pysocks
pandas pandas
aiohttp_socks aiohttp_socks
\ No newline at end of file schedule
\ No newline at end of file
...@@ -52,3 +52,6 @@ class Config: ...@@ -52,3 +52,6 @@ class Config:
ES_count = {"likes":True,"replies":True,"retweets":True} ES_count = {"likes":True,"replies":True,"retweets":True}
Lowercase = False Lowercase = False
Pandas_au = True Pandas_au = True
Proxy_host = None
Proxy_port = 0
Proxy_type = None
...@@ -11,6 +11,7 @@ from . import url ...@@ -11,6 +11,7 @@ from . import url
from .output import Tweets, Users from .output import Tweets, Users
async def RequestUrl(config, init): async def RequestUrl(config, init):
_connector = None
if config.Proxy_host is not None: if config.Proxy_host is not None:
if config.Proxy_host.lower() == "tor": if config.Proxy_host.lower() == "tor":
connector = SocksConnector( connector = SocksConnector(
...@@ -43,13 +44,13 @@ async def RequestUrl(config, init): ...@@ -43,13 +44,13 @@ async def RequestUrl(config, init):
if config.Profile: if config.Profile:
if config.Profile_full: if config.Profile_full:
_url = await url.MobileProfile(config.Username, init) _url = await url.MobileProfile(config.Username, init)
response = await MobileRequest(_url, _connector) response = await MobileRequest(_url, connector=_connector)
else: else:
_url = await url.Profile(config.Username, init) _url = await url.Profile(config.Username, init)
response = await Request(_url, _connector) response = await Request(_url, connector=_connector)
elif config.TwitterSearch: elif config.TwitterSearch:
_url = await url.Search(config, init) _url = await url.Search(config, init)
response = await Request(_url, _connector) response = await Request(_url, options=_connector)
else: else:
if config.Following: if config.Following:
_url = await url.Following(config.Username, init) _url = await url.Following(config.Username, init)
...@@ -57,20 +58,28 @@ async def RequestUrl(config, init): ...@@ -57,20 +58,28 @@ async def RequestUrl(config, init):
_url = await url.Followers(config.Username, init) _url = await url.Followers(config.Username, init)
else: else:
_url = await url.Favorites(config.Username, init) _url = await url.Favorites(config.Username, init)
response = await MobileRequest(_url, _connector) response = await MobileRequest(_url, connector=_connector)
if config.Debug: if config.Debug:
print(_url, file=open("twint-request_urls.log", "a", encoding="utf-8")) print(_url, file=open("twint-request_urls.log", "a", encoding="utf-8"))
return response return response
async def MobileRequest(url, connector): async def MobileRequest(url, **options):
ua = {'User-Agent': 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12'} ua = {'User-Agent': 'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/0.8.12'}
async with aiohttp.ClientSession(headers=ua, connector=connector) as session: connector = options.get("_connector")
if connector:
async with aiohttp.ClientSession(headers=ua, connector=connector) as session:
return await Response(session, url)
async with aiohttp.ClientSession(headers=ua) as session:
return await Response(session, url) return await Response(session, url)
async def Request(url, connector): async def Request(url, **options):
async with aiohttp.ClientSession(connector=connector) as session: connector = options.get("_connector")
if connector:
async with aiohttp.ClientSession(connector=connector) as session:
return await Response(session, url)
async with aiohttp.ClientSession() as session:
return await Response(session, url) return await Response(session, url)
async def Response(session, url): async def Response(session, url):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment