Commit e6767bdb authored by Francesco Poldi's avatar Francesco Poldi

Added option to save user-info into elasticsearch

parent 7dc58655
......@@ -162,7 +162,7 @@ def options():
ap.add_argument("--user-full",
help="Collect all user information (Use with followers or following only).",
action="store_true")
ap.add_argument("--user-info", help="Scrape user's info in tweet", action="store_false")
ap.add_argument("--user-info", help="Scrape user's info in tweet", action="store_true")
ap.add_argument("--profile-full",
help="Slow, but effective method of collecting a user's Tweets and RT.",
action="store_true")
......
......@@ -82,17 +82,20 @@ async def tweetUserData(tweet,config, conn):
user_ids = set()
usernames = []
for user in tweet.mentions:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
if config.Database:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
user_ids.add(user["id"])
usernames.append(user["screen_name"])
usernames.append(user["screen_name"])
for user in tweet.tags:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
if config.Database:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
user_ids.add(user["id"])
usernames.append(user["screen_name"])
usernames.append(user["screen_name"])
for user in tweet.replies:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
if config.Database:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
user_ids.add(user["id"])
usernames.append(user["screen_name"])
usernames.append(user["screen_name"])
for user in usernames:
url = f"http://twitter.com/{user}?lang=en"
await get.User(url, config, conn)
......@@ -103,7 +106,7 @@ async def Tweets(tw, location, config, conn):
if copyright is None and is_tweet(tw):
tweet = Tweet(tw, location, config)
if config.Database is not None and config.User_info:
if (config.Database or config.Elasticsearch) and config.User_info:
await tweetUserData(tweet, config, conn)
if datecheck(tweet.datestamp, config):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment