Commit e6767bdb authored by Francesco Poldi's avatar Francesco Poldi

Added option to save user-info into elasticsearch

parent 7dc58655
...@@ -162,7 +162,7 @@ def options(): ...@@ -162,7 +162,7 @@ def options():
ap.add_argument("--user-full", ap.add_argument("--user-full",
help="Collect all user information (Use with followers or following only).", help="Collect all user information (Use with followers or following only).",
action="store_true") action="store_true")
ap.add_argument("--user-info", help="Scrape user's info in tweet", action="store_false") ap.add_argument("--user-info", help="Scrape user's info in tweet", action="store_true")
ap.add_argument("--profile-full", ap.add_argument("--profile-full",
help="Slow, but effective method of collecting a user's Tweets and RT.", help="Slow, but effective method of collecting a user's Tweets and RT.",
action="store_true") action="store_true")
......
...@@ -82,17 +82,20 @@ async def tweetUserData(tweet,config, conn): ...@@ -82,17 +82,20 @@ async def tweetUserData(tweet,config, conn):
user_ids = set() user_ids = set()
usernames = [] usernames = []
for user in tweet.mentions: for user in tweet.mentions:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids: if config.Database:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
user_ids.add(user["id"]) user_ids.add(user["id"])
usernames.append(user["screen_name"]) usernames.append(user["screen_name"])
for user in tweet.tags: for user in tweet.tags:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids: if config.Database:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
user_ids.add(user["id"]) user_ids.add(user["id"])
usernames.append(user["screen_name"]) usernames.append(user["screen_name"])
for user in tweet.replies: for user in tweet.replies:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids: if config.Database:
if db.get_user_id(conn, user["id"]) == -1 and user["id"] not in user_ids:
user_ids.add(user["id"]) user_ids.add(user["id"])
usernames.append(user["screen_name"]) usernames.append(user["screen_name"])
for user in usernames: for user in usernames:
url = f"http://twitter.com/{user}?lang=en" url = f"http://twitter.com/{user}?lang=en"
await get.User(url, config, conn) await get.User(url, config, conn)
...@@ -103,7 +106,7 @@ async def Tweets(tw, location, config, conn): ...@@ -103,7 +106,7 @@ async def Tweets(tw, location, config, conn):
if copyright is None and is_tweet(tw): if copyright is None and is_tweet(tw):
tweet = Tweet(tw, location, config) tweet = Tweet(tw, location, config)
if config.Database is not None and config.User_info: if (config.Database or config.Elasticsearch) and config.User_info:
await tweetUserData(tweet, config, conn) await tweetUserData(tweet, config, conn)
if datecheck(tweet.datestamp, config): if datecheck(tweet.datestamp, config):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment