Commit 25a9d99d authored by Francesco Poldi's avatar Francesco Poldi

Fixed logger

parent 95bebc68
...@@ -2,19 +2,30 @@ ...@@ -2,19 +2,30 @@
TWINT - Twitter Intelligence Tool (formerly known as Tweep). TWINT - Twitter Intelligence Tool (formerly known as Tweep).
See wiki on Github for in-depth details. See wiki on Github for in-depth details.
https://github.com/haccer/twint/wiki https://github.com/twintproject/twint/wiki
Licensed under MIT License Licensed under MIT License
Copyright (c) 2018 Cody Zacharias Copyright (c) 2018 Cody Zacharias
''' '''
import logging, os
from .config import Config from .config import Config
from . import run from . import run
#import logging _levels = {
#logger = logging.getLogger() 'info': logging.INFO,
#handler = logging.FileHandler('twint.log') 'debug': logging.DEBUG
#formatter = logging.Formatter( }
# '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
#handler.setFormatter(formatter) _level = os.getenv('TWINT_DEBUG', 'info')
#logger.addHandler(handler) _logLevel = _levels[_level]
#logger.setLevel(logging.DEBUG)
\ No newline at end of file if _level == "debug":
logger = logging.getLogger()
_output_fn = 'twint.log'
logger.setLevel(_logLevel)
formatter = logging.Formatter('%(levelname)s:%(asctime)s:%(name)s:%(message)s')
fileHandler = logging.FileHandler(_output_fn)
fileHandler.setLevel(_logLevel)
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
VERSION = (1, 2, 1) VERSION = (1, 2, 1)
__version__ = '.'.join(map(str, VERSION)) __version__ = '.'.join(map(str, VERSION))
\ No newline at end of file
import logging
LEVEL = logging.INFO
class _logger:
def __init__(self, loggerName):
self._level = LEVEL
self._output_fn = 'twint.log'
self.logger = logging.getLogger(loggerName)
self.logger.setLevel(self._level)
self.formatter = logging.Formatter('%(levelname)s:%(asctime)s:%(name)s:%(message)s')
self.fileHandler = logging.FileHandler(self._output_fn)
self.fileHandler.setLevel(self._level)
self.fileHandler.setFormatter(self.formatter)
self.logger.addHandler(self.fileHandler)
def critical(self, message):
self.logger.critical(message)
def info(self, message):
self.logger.info(message)
def debug(self, message):
self.logger.debug(message)
\ No newline at end of file
import datetime import datetime
from . import _logme import logging as logme
logme = _logme._logger(__name__)
class Datelock: class Datelock:
_until = None _until = None
...@@ -10,7 +8,7 @@ class Datelock: ...@@ -10,7 +8,7 @@ class Datelock:
_since_def_user = None _since_def_user = None
def Set(Until, Since): def Set(Until, Since):
logme.debug('Set') logme.debug(__name__+':Set')
d = Datelock() d = Datelock()
if Until: if Until:
......
...@@ -2,36 +2,34 @@ from bs4 import BeautifulSoup ...@@ -2,36 +2,34 @@ from bs4 import BeautifulSoup
from re import findall from re import findall
from json import loads from json import loads
from . import _logme import logging as logme
logme = _logme._logger(__name__)
def Follow(response): def Follow(response):
logme.debug('Follow') logme.debug(__name__+':Follow')
soup = BeautifulSoup(response, "html.parser") soup = BeautifulSoup(response, "html.parser")
follow = soup.find_all("td", "info fifty screenname") follow = soup.find_all("td", "info fifty screenname")
cursor = soup.find_all("div", "w-button-more") cursor = soup.find_all("div", "w-button-more")
try: try:
cursor = findall(r'cursor=(.*?)">', str(cursor))[0] cursor = findall(r'cursor=(.*?)">', str(cursor))[0]
except IndexError: except IndexError:
logme.critical('Follow:IndexError') logme.critical(__name__+':Follow:IndexError')
return follow, cursor return follow, cursor
def Mobile(response): def Mobile(response):
logme.debug('Mobile') logme.debug(__name__+':Mobile')
soup = BeautifulSoup(response, "html.parser") soup = BeautifulSoup(response, "html.parser")
tweets = soup.find_all("span", "metadata") tweets = soup.find_all("span", "metadata")
max_id = soup.find_all("div", "w-button-more") max_id = soup.find_all("div", "w-button-more")
try: try:
max_id = findall(r'max_id=(.*?)">', str(max_id))[0] max_id = findall(r'max_id=(.*?)">', str(max_id))[0]
except Exception as e: except Exception as e:
logme.critical('Mobile:' + str(e)) logme.critical(__name__+':Mobile:' + str(e))
return tweets, max_id return tweets, max_id
def profile(response): def profile(response):
logme.debug('profile') logme.debug(__name__+':profile')
json_response = loads(response) json_response = loads(response)
html = json_response["items_html"] html = json_response["items_html"]
soup = BeautifulSoup(html, "html.parser") soup = BeautifulSoup(html, "html.parser")
...@@ -40,7 +38,7 @@ def profile(response): ...@@ -40,7 +38,7 @@ def profile(response):
return feed, feed[-1]["data-item-id"] return feed, feed[-1]["data-item-id"]
def Json(response): def Json(response):
logme.debug('Json') logme.debug(__name__+':Json')
json_response = loads(response) json_response = loads(response)
html = json_response["items_html"] html = json_response["items_html"]
soup = BeautifulSoup(html, "html.parser") soup = BeautifulSoup(html, "html.parser")
......
from . import _logme import logging as logme
logme = _logme._logger(__name__)
def Tweet(config, t): def Tweet(config, t):
if config.Format: if config.Format:
logme.debug('Tweet:Format') logme.debug(__name__+':Tweet:Format')
output = config.Format.replace("{id}", t.id_str) output = config.Format.replace("{id}", t.id_str)
output = output.replace("{date}", t.datestamp) output = output.replace("{date}", t.datestamp)
output = output.replace("{time}", t.timestamp) output = output.replace("{time}", t.timestamp)
...@@ -21,7 +19,7 @@ def Tweet(config, t): ...@@ -21,7 +19,7 @@ def Tweet(config, t):
output = output.replace("{is_retweet}", str(t.retweet)) output = output.replace("{is_retweet}", str(t.retweet))
output = output.replace("{mentions}", str(t.mentions)) output = output.replace("{mentions}", str(t.mentions))
else: else:
logme.debug('Tweet:notFormat') logme.debug(__name__+':Tweet:notFormat')
output = f"{t.id_str} {t.datestamp} {t.timestamp} {t.timezone} " output = f"{t.id_str} {t.datestamp} {t.timestamp} {t.timezone} "
if t.retweet == 1: if t.retweet == 1:
...@@ -41,7 +39,7 @@ def Tweet(config, t): ...@@ -41,7 +39,7 @@ def Tweet(config, t):
def User(_format, u): def User(_format, u):
if _format: if _format:
logme.debug('User:Format') logme.debug(__name__+':User:Format')
output = _format.replace("{id}", u.id) output = _format.replace("{id}", u.id)
output += output.replace("{name}", u.name) output += output.replace("{name}", u.name)
output += output.replace("{username}", u.username) output += output.replace("{username}", u.username)
...@@ -59,7 +57,7 @@ def User(_format, u): ...@@ -59,7 +57,7 @@ def User(_format, u):
output += output.replace("{verified}", str(u.is_verified)) output += output.replace("{verified}", str(u.is_verified))
output += output.replace("{avatar}", u.avatar) output += output.replace("{avatar}", u.avatar)
else: else:
logme.debug('User:notFormat') logme.debug(__name__+':User:notFormat')
output = f"{u.id} | {u.name} | @{u.username} | Private: " output = f"{u.id} | {u.name} | @{u.username} | Private: "
output += f"{u.is_private} | Verified: {u.is_verified} |" output += f"{u.is_private} | Verified: {u.is_verified} |"
output += f" Bio: {u.bio} | Location: {u.location} | Url: " output += f" Bio: {u.bio} | Location: {u.location} | Url: "
......
...@@ -13,14 +13,13 @@ from aiohttp_socks import SocksConnector, SocksVer ...@@ -13,14 +13,13 @@ from aiohttp_socks import SocksConnector, SocksVer
from . import url from . import url
from .output import Tweets, Users from .output import Tweets, Users
from .user import inf from .user import inf
from . import _logme
logme = _logme._logger(__name__) import logging as logme
httpproxy = None httpproxy = None
def get_connector(config): def get_connector(config):
logme.debug('get_connector') logme.debug(__name__+':get_connector')
_connector = None _connector = None
if config.Proxy_host is not None: if config.Proxy_host is not None:
if config.Proxy_host.lower() == "tor": if config.Proxy_host.lower() == "tor":
...@@ -48,12 +47,12 @@ def get_connector(config): ...@@ -48,12 +47,12 @@ def get_connector(config):
port=config.Proxy_port, port=config.Proxy_port,
rdns=True) rdns=True)
else: else:
logme.critical('get_connector:proxy-port-type-error') logme.critical(__name__+':get_connector:proxy-port-type-error')
print("Error: Please specify --proxy-host, --proxy-port, and --proxy-type") print("Error: Please specify --proxy-host, --proxy-port, and --proxy-type")
sys.exit(1) sys.exit(1)
else: else:
if config.Proxy_port or config.Proxy_type: if config.Proxy_port or config.Proxy_type:
logme.critical('get_connector:proxy-host-arg-error') logme.critical(__name__+':get_connector:proxy-host-arg-error')
print("Error: Please specify --proxy-host, --proxy-port, and --proxy-type") print("Error: Please specify --proxy-host, --proxy-port, and --proxy-type")
sys.exit(1) sys.exit(1)
...@@ -61,31 +60,31 @@ def get_connector(config): ...@@ -61,31 +60,31 @@ def get_connector(config):
async def RequestUrl(config, init, headers = []): async def RequestUrl(config, init, headers = []):
logme.debug('RequestUrl') logme.debug(__name__+':RequestUrl')
_connector = get_connector(config) _connector = get_connector(config)
if config.Profile: if config.Profile:
if config.Profile_full: if config.Profile_full:
logme.debug('RequestUrl:Profile_full') logme.debug(__name__+':RequestUrl:Profile_full')
_url = await url.MobileProfile(config.Username, init) _url = await url.MobileProfile(config.Username, init)
response = await MobileRequest(_url, connector=_connector) response = await MobileRequest(_url, connector=_connector)
else: else:
logme.debug('RequestUrl:notProfile_full') logme.debug(__name__+':RequestUrl:notProfile_full')
_url = await url.Profile(config.Username, init) _url = await url.Profile(config.Username, init)
response = await Request(_url, connector=_connector, headers=headers) response = await Request(_url, connector=_connector, headers=headers)
elif config.TwitterSearch: elif config.TwitterSearch:
logme.debug('RequestUrl:TwitterSearch') logme.debug(__name__+':RequestUrl:TwitterSearch')
_url, params = await url.Search(config, init) _url, params = await url.Search(config, init)
response = await Request(_url, params=params, connector=_connector, headers=headers) response = await Request(_url, params=params, connector=_connector, headers=headers)
else: else:
if config.Following: if config.Following:
logme.debug('RequestUrl:Following') logme.debug(__name__+':RequestUrl:Following')
_url = await url.Following(config.Username, init) _url = await url.Following(config.Username, init)
elif config.Followers: elif config.Followers:
logme.debug('RequestUrl:Followers') logme.debug(__name__+':RequestUrl:Followers')
_url = await url.Followers(config.Username, init) _url = await url.Followers(config.Username, init)
else: else:
logme.debug('RequestUrl:Favorites') logme.debug(__name__+':RequestUrl:Favorites')
_url = await url.Favorites(config.Username, init) _url = await url.Favorites(config.Username, init)
response = await MobileRequest(_url, connector=_connector) response = await MobileRequest(_url, connector=_connector)
...@@ -97,51 +96,51 @@ async def RequestUrl(config, init, headers = []): ...@@ -97,51 +96,51 @@ async def RequestUrl(config, init, headers = []):
async def MobileRequest(url, **options): async def MobileRequest(url, **options):
connector = options.get("connector") connector = options.get("connector")
if connector: if connector:
logme.debug('MobileRequest:Connector') logme.debug(__name__+':MobileRequest:Connector')
async with aiohttp.ClientSession(connector=connector) as session: async with aiohttp.ClientSession(connector=connector) as session:
return await Response(session, url) return await Response(session, url)
logme.debug('MobileRequest:notConnector') logme.debug(__name__+':MobileRequest:notConnector')
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
return await Response(session, url) return await Response(session, url)
def ForceNewTorIdentity(config): def ForceNewTorIdentity(config):
logme.debug('ForceNewTorIdentity') logme.debug(__name__+':ForceNewTorIdentity')
try: try:
tor_c = socket.create_connection(('127.0.0.1', config.Tor_control_port)) tor_c = socket.create_connection(('127.0.0.1', config.Tor_control_port))
tor_c.send('AUTHENTICATE "{}"\r\nSIGNAL NEWNYM\r\n'.format(config.Tor_control_password).encode()) tor_c.send('AUTHENTICATE "{}"\r\nSIGNAL NEWNYM\r\n'.format(config.Tor_control_password).encode())
response = tor_c.recv(1024) response = tor_c.recv(1024)
if response != b'250 OK\r\n250 OK\r\n': if response != b'250 OK\r\n250 OK\r\n':
sys.stderr.write('Unexpected response from Tor control port: {}\n'.format(response)) sys.stderr.write('Unexpected response from Tor control port: {}\n'.format(response))
logme.critical('ForceNewTorIdentity:unexpectedResponse') logme.critical(__name__+':ForceNewTorIdentity:unexpectedResponse')
except Exception as e: except Exception as e:
logme.debug('ForceNewTorIdentity:errorConnectingTor') logme.debug(__name__+':ForceNewTorIdentity:errorConnectingTor')
sys.stderr.write('Error connecting to Tor control port: {}\n'.format(repr(e))) sys.stderr.write('Error connecting to Tor control port: {}\n'.format(repr(e)))
sys.stderr.write('If you want to rotate Tor ports automatically - enable Tor control port\n') sys.stderr.write('If you want to rotate Tor ports automatically - enable Tor control port\n')
async def Request(url, connector=None, params=[], headers=[]): async def Request(url, connector=None, params=[], headers=[]):
if connector: if connector:
logme.debug('Request:Connector') logme.debug(__name__+':Request:Connector')
async with aiohttp.ClientSession(connector=connector, headers=headers) as session: async with aiohttp.ClientSession(connector=connector, headers=headers) as session:
return await Response(session, url, params) return await Response(session, url, params)
logme.debug('Request:notConnector') logme.debug(__name__+':Request:notConnector')
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
return await Response(session, url, params) return await Response(session, url, params)
async def Response(session, url, params=[]): async def Response(session, url, params=[]):
logme.debug('Response') logme.debug(__name__+':Response')
with timeout(30): with timeout(30):
async with session.get(url, ssl=False, params=params, proxy=httpproxy) as response: async with session.get(url, ssl=False, params=params, proxy=httpproxy) as response:
return await response.text() return await response.text()
async def RandomUserAgent(): async def RandomUserAgent():
logme.debug('RandomUserAgent') logme.debug(__name__+':RandomUserAgent')
url = "https://fake-useragent.herokuapp.com/browsers/0.1.8" url = "https://fake-useragent.herokuapp.com/browsers/0.1.8"
r = await Request(url) r = await Request(url)
browsers = loads(r)['browsers'] browsers = loads(r)['browsers']
return random.choice(browsers[random.choice(list(browsers))]) return random.choice(browsers[random.choice(list(browsers))])
async def Username(_id): async def Username(_id):
logme.debug('Username') logme.debug(__name__+':Username')
url = f"https://twitter.com/intent/user?user_id={_id}&lang=en" url = f"https://twitter.com/intent/user?user_id={_id}&lang=en"
r = await Request(url) r = await Request(url)
soup = BeautifulSoup(r, "html.parser") soup = BeautifulSoup(r, "html.parser")
...@@ -149,7 +148,7 @@ async def Username(_id): ...@@ -149,7 +148,7 @@ async def Username(_id):
return soup.find("a", "fn url alternate-context")["href"].replace("/", "") return soup.find("a", "fn url alternate-context")["href"].replace("/", "")
async def Tweet(url, config, conn): async def Tweet(url, config, conn):
logme.debug('Tweet') logme.debug(__name__+':Tweet')
try: try:
response = await Request(url) response = await Request(url)
soup = BeautifulSoup(response, "html.parser") soup = BeautifulSoup(response, "html.parser")
...@@ -158,11 +157,11 @@ async def Tweet(url, config, conn): ...@@ -158,11 +157,11 @@ async def Tweet(url, config, conn):
tweets = soup.find_all("div", "tweet") tweets = soup.find_all("div", "tweet")
await Tweets(tweets, location, config, conn, url) await Tweets(tweets, location, config, conn, url)
except Exception as e: except Exception as e:
logme.critical('Tweet:' + str(e)) logme.critical(__name__+':Tweet:' + str(e))
print(str(e) + " [x] get.Tweet") print(str(e) + " [x] get.Tweet")
async def User(url, config, conn, user_id = False): async def User(url, config, conn, user_id = False):
logme.debug('User') logme.debug(__name__+':User')
_connector = get_connector(config) _connector = get_connector(config)
try: try:
response = await Request(url, connector=_connector) response = await Request(url, connector=_connector)
...@@ -170,16 +169,16 @@ async def User(url, config, conn, user_id = False): ...@@ -170,16 +169,16 @@ async def User(url, config, conn, user_id = False):
if user_id: if user_id:
return int(inf(soup, "id")) return int(inf(soup, "id"))
except Exception as e: except Exception as e:
logme.critical('User:' + str(e)) logme.critical(__name__+':User:' + str(e))
print(str(e) + " [x] get.User") print(str(e) + " [x] get.User")
def Limit(Limit, count): def Limit(Limit, count):
logme.critical('Limit') logme.debug(__name__+':Limit')
if Limit is not None and count >= int(Limit): if Limit is not None and count >= int(Limit):
return True return True
async def Multi(feed, config, conn): async def Multi(feed, config, conn):
logme.debug('Multi') logme.debug(__name__+':Multi')
count = 0 count = 0
try: try:
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
...@@ -188,34 +187,34 @@ async def Multi(feed, config, conn): ...@@ -188,34 +187,34 @@ async def Multi(feed, config, conn):
for tweet in feed: for tweet in feed:
count += 1 count += 1
if config.Favorites or config.Profile_full: if config.Favorites or config.Profile_full:
logme.debug('Multi:Favorites-profileFull') logme.debug(__name__+':Multi:Favorites-profileFull')
link = tweet.find("a")["href"] link = tweet.find("a")["href"]
url = f"https://twitter.com{link}&lang=en" url = f"https://twitter.com{link}&lang=en"
elif config.User_full: elif config.User_full:
logme.debug('Multi:userFull') logme.debug(__name__+':Multi:userFull')
username = tweet.find("a")["name"] username = tweet.find("a")["name"]
url = f"http://twitter.com/{username}?lang=en" url = f"http://twitter.com/{username}?lang=en"
else: else:
logme.debug('Multi:else-url') logme.debug(__name__+':Multi:else-url')
link = tweet.find("a", "tweet-timestamp js-permalink js-nav js-tooltip")["href"] link = tweet.find("a", "tweet-timestamp js-permalink js-nav js-tooltip")["href"]
url = f"https://twitter.com{link}?lang=en" url = f"https://twitter.com{link}?lang=en"
if config.User_full: if config.User_full:
logme.debug('Multi:user-full-Run') logme.debug(__name__+':Multi:user-full-Run')
futures.append(loop.run_in_executor(executor, await User(url, futures.append(loop.run_in_executor(executor, await User(url,
config, conn))) config, conn)))
else: else:
logme.debug('Multi:notUser-full-Run') logme.debug(__name__+':Multi:notUser-full-Run')
futures.append(loop.run_in_executor(executor, await Tweet(url, futures.append(loop.run_in_executor(executor, await Tweet(url,
config, conn))) config, conn)))
logme.debug('Multi:asyncioGather') logme.debug(__name__+':Multi:asyncioGather')
await asyncio.gather(*futures) await asyncio.gather(*futures)
except Exception as e: except Exception as e:
# TODO: fix error not error # TODO: fix error not error
# print(str(e) + " [x] get.Multi") # print(str(e) + " [x] get.Multi")
# will return "'NoneType' object is not callable" # will return "'NoneType' object is not callable"
# but still works # but still works
logme.critical('Multi:' + str(e)) logme.critical(__name__+':Multi:' + str(e))
pass pass
return count return count
...@@ -4,9 +4,8 @@ from . import format, get ...@@ -4,9 +4,8 @@ from . import format, get
from .tweet import Tweet from .tweet import Tweet
from .user import User from .user import User
from .storage import db, elasticsearch, write, panda from .storage import db, elasticsearch, write, panda
from . import _logme
logme = _logme._logger(__name__) import logging as logme
follow_object = {} follow_object = {}
tweets_object = [] tweets_object = []
...@@ -18,41 +17,41 @@ author_list.pop() ...@@ -18,41 +17,41 @@ author_list.pop()
_follow_list = [] _follow_list = []
def clean_follow_list(): def clean_follow_list():
logme.debug('clean_follow_list') logme.debug(__name__+':clean_follow_list')
global _follow_list global _follow_list
_follow_list = [] _follow_list = []
def datecheck(datestamp, config): def datecheck(datestamp, config):
logme.debug('datecheck') logme.debug(__name__+':datecheck')
if config.Since and config.Until: if config.Since and config.Until:
logme.debug('datecheck:dateRangeTrue') logme.debug(__name__+':datecheck:dateRangeTrue')
d = int(datestamp.replace("-", "")) d = int(datestamp.replace("-", ""))
s = int(config.Since.replace("-", "")) s = int(config.Since.replace("-", ""))
if d < s: if d < s:
return False return False
logme.debug('datecheck:dateRangeFalse') logme.debug(__name__+':datecheck:dateRangeFalse')
return True return True
def is_tweet(tw): def is_tweet(tw):
try: try:
tw["data-item-id"] tw["data-item-id"]
logme.debug('is_tweet:True') logme.debug(__name__+':is_tweet:True')
return True return True
except: except:
logme.critical('is_tweet:False') logme.critical(__name__+':is_tweet:False')
return False return False
def _output(obj, output, config, **extra): def _output(obj, output, config, **extra):
logme.debug('_output') logme.debug(__name__+':_output')
if config.Lowercase: if config.Lowercase:
if isinstance(obj, str): if isinstance(obj, str):
logme.debug('_output:Lowercase:username') logme.debug(__name__+':_output:Lowercase:username')
obj = obj.lower() obj = obj.lower()
elif obj.__class__.__name__ == "user": elif obj.__class__.__name__ == "user":
logme.debug('_output:Lowercase:user') logme.debug(__name__+':_output:Lowercase:user')
pass pass
elif obj.__class__.__name__ == "tweet": elif obj.__class__.__name__ == "tweet":
logme.debug('_output:Lowercase:tweet') logme.debug(__name__+':_output:Lowercase:tweet')
obj.username = obj.username.lower() obj.username = obj.username.lower()
author_list.update({obj.username}) author_list.update({obj.username})
for i in range(len(obj.mentions)): for i in range(len(obj.mentions)):
...@@ -67,45 +66,45 @@ def _output(obj, output, config, **extra): ...@@ -67,45 +66,45 @@ def _output(obj, output, config, **extra):
if config.Store_csv: if config.Store_csv:
try: try:
write.Csv(obj, config) write.Csv(obj, config)
logme.debug('_output:CSV') logme.debug(__name__+':_output:CSV')
except Exception as e: except Exception as e:
logme.critical('_output:CSV:Error:' + str(e)) logme.critical(__name__+':_output:CSV:Error:' + str(e))
print(str(e) + " [x] output._output") print(str(e) + " [x] output._output")
elif config.Store_json: elif config.Store_json:
write.Json(obj, config) write.Json(obj, config)
logme.debug('_output:JSON') logme.debug(__name__+':_output:JSON')
else: else:
write.Text(output, config.Output) write.Text(output, config.Output)
logme.debug('_output:Text') logme.debug(__name__+':_output:Text')
if config.Pandas and obj.type == "user": if config.Pandas and obj.type == "user":
logme.debug('_output:Pandas+user') logme.debug(__name__+':_output:Pandas+user')
panda.update(obj, config) panda.update(obj, config)
if extra.get("follow_list"): if extra.get("follow_list"):
logme.debug('_output:follow_list') logme.debug(__name__+':_output:follow_list')
follow_object.username = config.Username follow_object.username = config.Username
follow_object.action = config.Following*"following" + config.Followers*"followers" follow_object.action = config.Following*"following" + config.Followers*"followers"
follow_object.users = _follow_list follow_object.users = _follow_list
panda.update(follow_object, config.Essid) panda.update(follow_object, config.Essid)
if config.Elasticsearch: if config.Elasticsearch:
logme.debug('_output:Elasticsearch') logme.debug(__name__+':_output:Elasticsearch')
print("", end=".", flush=True) print("", end=".", flush=True)
else: else:
if not config.Hide_output: if not config.Hide_output:
try: try:
print(output) print(output)
except UnicodeEncodeError: except UnicodeEncodeError:
logme.critical('_output:UnicodeEncodeError') logme.critical(__name__+':_output:UnicodeEncodeError')
print("unicode error [x] output._output") print("unicode error [x] output._output")
async def checkData(tweet, location, config, conn): async def checkData(tweet, location, config, conn):
logme.debug('checkData') logme.debug(__name__+':checkData')
copyright = tweet.find("div", "StreamItemContent--withheld") copyright = tweet.find("div", "StreamItemContent--withheld")
if copyright is None and is_tweet(tweet): if copyright is None and is_tweet(tweet):
tweet = Tweet(tweet, location, config) tweet = Tweet(tweet, location, config)
if not tweet.datestamp: if not tweet.datestamp:
logme.critical('checkData:hiddenTweetFound') logme.critical(__name__+':checkData:hiddenTweetFound')
print("[x] Hidden tweet found, account suspended due to violation of TOS") print("[x] Hidden tweet found, account suspended due to violation of TOS")
return return
...@@ -113,56 +112,57 @@ async def checkData(tweet, location, config, conn): ...@@ -113,56 +112,57 @@ async def checkData(tweet, location, config, conn):
output = format.Tweet(config, tweet) output = format.Tweet(config, tweet)
if config.Database: if config.Database:
logme.debug('checkData:Database') logme.debug(__name__+':checkData:Database')
db.tweets(conn, tweet, config) db.tweets(conn, tweet, config)
if config.Pandas: if config.Pandas:
logme.debug('checkData:Pandas') logme.debug(__name__+':checkData:Pandas')
panda.update(tweet, config) panda.update(tweet, config)
if config.Store_object: if config.Store_object:
logme.debug('checkData:Store_object') logme.debug(__name__+':checkData:Store_object')
tweets_object.append(tweet) tweets_object.append(tweet)
if config.Elasticsearch: if config.Elasticsearch:
logme.debug('checkData:Elasticsearch') logme.debug(__name__+':checkData:Elasticsearch')
elasticsearch.Tweet(tweet, config) elasticsearch.Tweet(tweet, config)
_output(tweet, output, config) _output(tweet, output, config)
logme.critical('checkData:copyrightedTweet') else:
logme.critical(__name__+':checkData:copyrightedTweet')
async def Tweets(tweets, location, config, conn, url=''): async def Tweets(tweets, location, config, conn, url=''):
logme.debug('Tweets') logme.debug(__name__+':Tweets')
if (config.Profile_full or config.Location) and config.Get_replies: if (config.Profile_full or config.Location) and config.Get_replies:
logme.debug('Tweets:full+loc+replies') logme.debug(__name__+':Tweets:full+loc+replies')
for tw in tweets: for tw in tweets:
await checkData(tw, location, config, conn) await checkData(tw, location, config, conn)
elif config.Favorites or config.Profile_full or config.Location: elif config.Favorites or config.Profile_full or config.Location:
logme.debug('Tweets:fav+full+loc') logme.debug(__name__+':Tweets:fav+full+loc')
for tw in tweets: for tw in tweets:
if tw['data-item-id'] == url.split('?')[0].split('/')[-1]: if tw['data-item-id'] == url.split('?')[0].split('/')[-1]:
await checkData(tw, location, config, conn) await checkData(tw, location, config, conn)
elif config.TwitterSearch: elif config.TwitterSearch:
logme.debug('Tweets:TwitterSearch') logme.debug(__name__+':Tweets:TwitterSearch')
await checkData(tweets, location, config, conn) await checkData(tweets, location, config, conn)
else: else:
logme.debug('Tweets:else') logme.debug(__name__+':Tweets:else')
if int(tweets["data-user-id"]) == config.User_id: if int(tweets["data-user-id"]) == config.User_id:
await checkData(tweets, location, config, conn) await checkData(tweets, location, config, conn)
async def Users(u, config, conn): async def Users(u, config, conn):
logme.debug('User') logme.debug(__name__+':User')
global user_object global user_object
user = User(u) user = User(u)
output = format.User(config.Format, user) output = format.User(config.Format, user)
if config.Database: if config.Database:
logme.debug('User:Database') logme.debug(__name__+':User:Database')
db.user(conn, config, user) db.user(conn, config, user)
if config.Elasticsearch: if config.Elasticsearch:
logme.debug('User:Elasticsearch') logme.debug(__name__+':User:Elasticsearch')
_save_date = user.join_date _save_date = user.join_date
_save_time = user.join_time _save_time = user.join_time
user.join_date = str(datetime.strptime(user.join_date, "%d %b %Y")).split()[0] user.join_date = str(datetime.strptime(user.join_date, "%d %b %Y")).split()[0]
...@@ -172,32 +172,32 @@ async def Users(u, config, conn): ...@@ -172,32 +172,32 @@ async def Users(u, config, conn):
user.join_time = _save_time user.join_time = _save_time
if config.Store_object: if config.Store_object:
logme.debug('User:Store_object') logme.debug(__name__+':User:Store_object')
user_object.append(user) # twint.user.user user_object.append(user) # twint.user.user
_output(user, output, config) _output(user, output, config)
async def Username(username, config, conn): async def Username(username, config, conn):
logme.debug('Username') logme.debug(__name__+':Username')
global follow_object global follow_object
follow_var = config.Following*"following" + config.Followers*"followers" follow_var = config.Following*"following" + config.Followers*"followers"
if config.Database: if config.Database:
logme.debug('Username:Database') logme.debug(__name__+':Username:Database')
db.follow(conn, config.Username, config.Followers, username) db.follow(conn, config.Username, config.Followers, username)
if config.Elasticsearch: if config.Elasticsearch:
logme.debug('Username:Elasticsearch') logme.debug(__name__+':Username:Elasticsearch')
elasticsearch.Follow(username, config) elasticsearch.Follow(username, config)
if config.Store_object or config.Pandas: if config.Store_object or config.Pandas:
logme.debug('Username:object+pandas') logme.debug(__name__+':Username:object+pandas')
try: try:
_ = follow_object[config.Username][follow_var] _ = follow_object[config.Username][follow_var]
except KeyError: except KeyError:
follow_object.update({config.Username: {follow_var: []}}) follow_object.update({config.Username: {follow_var: []}})
follow_object[config.Username][follow_var].append(username) follow_object[config.Username][follow_var].append(username)
if config.Pandas_au: if config.Pandas_au:
logme.debug('Username:object+pandas+au') logme.debug(__name__+':Username:object+pandas+au')
panda.update(follow_object[config.Username], config) panda.update(follow_object[config.Username], config)
_output(username, username, config, follow_list=_follow_list) _output(username, username, config, follow_list=_follow_list)
This diff is collapsed.
from time import strftime, localtime from time import strftime, localtime
import json import json
from . import _logme import logging as logme
logme = _logme._logger(__name__)
class tweet: class tweet:
"""Define Tweet class """Define Tweet class
...@@ -16,7 +14,7 @@ class tweet: ...@@ -16,7 +14,7 @@ class tweet:
def getMentions(tw): def getMentions(tw):
"""Extract ment from tweet """Extract ment from tweet
""" """
logme.debug('getMentions') logme.debug(__name__+':getMentions')
try: try:
mentions = tw["data-mentions"].split(" ") mentions = tw["data-mentions"].split(" ")
except: except:
...@@ -27,7 +25,7 @@ def getMentions(tw): ...@@ -27,7 +25,7 @@ def getMentions(tw):
def getQuoteURL(tw): def getQuoteURL(tw):
"""Extract quote from tweet """Extract quote from tweet
""" """
logme.debug('getQuoteURL') logme.debug(__name__+':getQuoteURL')
base_twitter = "https://twitter.com" base_twitter = "https://twitter.com"
quote_url = "" quote_url = ""
try: try:
...@@ -41,7 +39,7 @@ def getQuoteURL(tw): ...@@ -41,7 +39,7 @@ def getQuoteURL(tw):
def getText(tw): def getText(tw):
"""Replace some text """Replace some text
""" """
logme.debug('getText') logme.debug(__name__+':getText')
text = tw.find("p", "tweet-text").text text = tw.find("p", "tweet-text").text
text = text.replace("\n", " ") text = text.replace("\n", " ")
text = text.replace("http", " http") text = text.replace("http", " http")
...@@ -52,21 +50,21 @@ def getText(tw): ...@@ -52,21 +50,21 @@ def getText(tw):
def getStat(tw, _type): def getStat(tw, _type):
"""Get stats about Tweet """Get stats about Tweet
""" """
logme.debug('getStat') logme.debug(__name__+':getStat')
st = f"ProfileTweet-action--{_type} u-hiddenVisually" st = f"ProfileTweet-action--{_type} u-hiddenVisually"
return tw.find("span", st).find("span")["data-tweet-stat-count"] return tw.find("span", st).find("span")["data-tweet-stat-count"]
def getRetweet(profile, username, user): def getRetweet(profile, username, user):
"""Get Retweet """Get Retweet
""" """
logme.debug('getRetweet') logme.debug(__name__+':getRetweet')
if profile and username.lower() != user.lower(): if profile and username.lower() != user.lower():
return 1 return 1
def Tweet(tw, location, config): def Tweet(tw, location, config):
"""Create Tweet object """Create Tweet object
""" """
logme.debug('Tweet') logme.debug(__name__+':Tweet')
t = tweet() t = tweet()
t.id = int(tw["data-item-id"]) t.id = int(tw["data-item-id"])
t.id_str = tw["data-item-id"] t.id_str = tw["data-item-id"]
......
from . import _logme import logging as logme
logme = _logme._logger(__name__)
mobile = "https://mobile.twitter.com" mobile = "https://mobile.twitter.com"
base = "https://twitter.com/i" base = "https://twitter.com/i"
async def Favorites(username, init): async def Favorites(username, init):
logme.debug('Favorites') logme.debug(__name__+':Favorites')
url = f"{mobile}/{username}/favorites?lang=en" url = f"{mobile}/{username}/favorites?lang=en"
if init != -1: if init != -1:
...@@ -14,7 +12,7 @@ async def Favorites(username, init): ...@@ -14,7 +12,7 @@ async def Favorites(username, init):
return url return url
async def Followers(username, init): async def Followers(username, init):
logme.debug('Followers') logme.debug(__name__+':Followers')
url = f"{mobile}/{username}/followers?lang=en" url = f"{mobile}/{username}/followers?lang=en"
if init != -1: if init != -1:
...@@ -23,7 +21,7 @@ async def Followers(username, init): ...@@ -23,7 +21,7 @@ async def Followers(username, init):
return url return url
async def Following(username, init): async def Following(username, init):
logme.debug('Following') logme.debug(__name__+':Following')
url = f"{mobile}/{username}/following?lang=en" url = f"{mobile}/{username}/following?lang=en"
if init != -1: if init != -1:
...@@ -32,7 +30,7 @@ async def Following(username, init): ...@@ -32,7 +30,7 @@ async def Following(username, init):
return url return url
async def MobileProfile(username, init): async def MobileProfile(username, init):
logme.debug('MobileProfile') logme.debug(__name__+':MobileProfile')
url = f"{mobile}/{username}?lang=en" url = f"{mobile}/{username}?lang=en"
if init != -1: if init != -1:
...@@ -41,7 +39,7 @@ async def MobileProfile(username, init): ...@@ -41,7 +39,7 @@ async def MobileProfile(username, init):
return url return url
async def Profile(username, init): async def Profile(username, init):
logme.debug('Profile') logme.debug(__name__+':Profile')
url = f"{base}/profiles/show/{username}/timeline/tweets?include_" url = f"{base}/profiles/show/{username}/timeline/tweets?include_"
url += "available_features=1&lang=en&include_entities=1" url += "available_features=1&lang=en&include_entities=1"
url += "&include_new_items_bar=true" url += "&include_new_items_bar=true"
...@@ -52,7 +50,7 @@ async def Profile(username, init): ...@@ -52,7 +50,7 @@ async def Profile(username, init):
return url return url
async def Search(config, init): async def Search(config, init):
logme.debug('Search') logme.debug(__name__+':Search')
url = f"{base}/search/timeline" url = f"{base}/search/timeline"
params = [ params = [
('f', 'tweets'), ('f', 'tweets'),
......
from . import _logme import logging as logme
logme = _logme._logger(__name__)
class user: class user:
type = "user" type = "user"
...@@ -9,7 +7,7 @@ class user: ...@@ -9,7 +7,7 @@ class user:
pass pass
def inf(ur, _type): def inf(ur, _type):
logme.debug('inf') logme.debug(__name__+':inf')
try: try:
group = ur.find("div", "user-actions btn-group not-following") group = ur.find("div", "user-actions btn-group not-following")
if group == None : if group == None :
...@@ -33,7 +31,7 @@ def inf(ur, _type): ...@@ -33,7 +31,7 @@ def inf(ur, _type):
return ret return ret
def card(ur, _type): def card(ur, _type):
logme.debug('card') logme.debug(__name__+':card')
if _type == "bio": if _type == "bio":
try: try:
ret = ur.find("p", "ProfileHeaderCard-bio u-dir").text.replace("\n", " ") ret = ur.find("p", "ProfileHeaderCard-bio u-dir").text.replace("\n", " ")
...@@ -54,12 +52,12 @@ def card(ur, _type): ...@@ -54,12 +52,12 @@ def card(ur, _type):
return ret return ret
def join(ur): def join(ur):
logme.debug('join') logme.debug(__name__+':join')
jd = ur.find("span", "ProfileHeaderCard-joinDateText js-tooltip u-dir")["title"] jd = ur.find("span", "ProfileHeaderCard-joinDateText js-tooltip u-dir")["title"]
return jd.split(" - ") return jd.split(" - ")
def convertToInt(x): def convertToInt(x):
logme.debug('contertToInt') logme.debug(__name__+':contertToInt')
multDict = { multDict = {
"k" : 1000, "k" : 1000,
"m" : 1000000, "m" : 1000000,
...@@ -83,7 +81,7 @@ def convertToInt(x): ...@@ -83,7 +81,7 @@ def convertToInt(x):
return 0 return 0
def stat(ur, _type): def stat(ur, _type):
logme.debug('stat') logme.debug(__name__+':stat')
_class = f"ProfileNav-item ProfileNav-item--{_type}" _class = f"ProfileNav-item ProfileNav-item--{_type}"
stat = ur.find("li", _class) stat = ur.find("li", _class)
try : try :
...@@ -93,7 +91,7 @@ def stat(ur, _type): ...@@ -93,7 +91,7 @@ def stat(ur, _type):
return r return r
def media(ur): def media(ur):
logme.debug('media') logme.debug(__name__+':media')
try: try:
media_count = ur.find("a", "PhotoRail-headingWithCount js-nav").text.strip().split(" ")[0] media_count = ur.find("a", "PhotoRail-headingWithCount js-nav").text.strip().split(" ")[0]
media_count = convertToInt(media_count) media_count = convertToInt(media_count)
...@@ -103,7 +101,7 @@ def media(ur): ...@@ -103,7 +101,7 @@ def media(ur):
return media_count return media_count
def verified(ur): def verified(ur):
logme.debug('verified') logme.debug(__name__+':verified')
try: try:
is_verified = ur.find("span", "ProfileHeaderCard-badges").text is_verified = ur.find("span", "ProfileHeaderCard-badges").text
if "Verified account" in is_verified: if "Verified account" in is_verified:
...@@ -116,7 +114,7 @@ def verified(ur): ...@@ -116,7 +114,7 @@ def verified(ur):
return is_verified return is_verified
def User(ur): def User(ur):
logme.debug('User') logme.debug(__name__+':User')
u = user() u = user()
for img in ur.findAll("img", "Emoji Emoji--forText"): for img in ur.findAll("img", "Emoji Emoji--forText"):
img.replaceWith(img["alt"]) img.replaceWith(img["alt"])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment