Commit 8bb3ac6b authored by Cody Zacharias's avatar Cody Zacharias Committed by GitHub

Cleanup & Moving stuff around

parent 6f1f5629
#!/usr/bin/python3 #!/usr/bin/python3
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from elasticsearch import Elasticsearch
from time import gmtime, strftime from time import gmtime, strftime
import argparse import argparse
import aiohttp import aiohttp
...@@ -7,12 +8,10 @@ import asyncio ...@@ -7,12 +8,10 @@ import asyncio
import async_timeout import async_timeout
import csv import csv
import datetime import datetime
import hashlib
import json import json
import re import re
import sys import sys
import hashlib
from elasticsearch import Elasticsearch
async def getUrl(init): async def getUrl(init):
''' '''
...@@ -151,18 +150,6 @@ async def outTweet(tweet): ...@@ -151,18 +150,6 @@ async def outTweet(tweet):
except: except:
pass pass
jObject = {
"tweetid": tweetid,
"datestamp": date + " " + time,
"timezone": timezone,
"text": text,
"hashtags": re.findall(r'(?i)\#\w+', text, flags=re.UNICODE),
"replies": replies,
"retweets": retweets,
"likes": likes,
"username": username
}
# Preparing to output # Preparing to output
''' '''
...@@ -172,28 +159,38 @@ async def outTweet(tweet): ...@@ -172,28 +159,38 @@ async def outTweet(tweet):
modes exist. modes exist.
''' '''
if arg.elasticsearch: if arg.elasticsearch:
jObject = {
"tweetid": tweetid,
"datestamp": date + " " + time,
"timezone": timezone,
"text": text,
"hashtags": re.findall(r'(?i)\#\w+', text, flags=re.UNICODE),
"replies": replies,
"retweets": retweets,
"likes": likes,
"username": username
}
es = Elasticsearch(arg.elasticsearch) es = Elasticsearch(arg.elasticsearch)
es.index(index="tweep", doc_type="items", id=tweetid, body=json.dumps(jObject)) es.index(index="tweep", doc_type="items", id=tweetid, body=json.dumps(jObject))
return "" output = ""
elif arg.users:
output = username
elif arg.tweets:
output = tweets
else: else:
if arg.users: '''
output = username The standard output is how I like it, although
elif arg.tweets: this can be modified to your desire. Uncomment
output = tweets the bottom line and add in the variables in the
else: order you want them or how you want it to look.
''' '''
The standard output is how I like it, although # output = ""
this can be modified to your desire. Uncomment output = "{} {} {} {} <{}> {}".format(tweetid, date, time, timezone, username, text)
the bottom line and add in the variables in the if arg.hashtags:
order you want them or how you want it to look. output+= " {}".format(hashtags)
''' if arg.stats:
# output = "" output+= " | {} replies {} retweets {} likes".format(replies, retweets, likes)
output = "{} {} {} {} <{}> {}".format(tweetid, date, time, timezone, username, text)
if arg.hashtags:
output+= " {}".format(hashtags)
if arg.stats:
output+= " | {} replies {} retweets {} likes".format(replies, retweets, likes)
# Output section # Output section
...@@ -208,7 +205,7 @@ async def outTweet(tweet): ...@@ -208,7 +205,7 @@ async def outTweet(tweet):
# Writes or appends to a file. # Writes or appends to a file.
print(output, file=open(arg.o, "a")) print(output, file=open(arg.o, "a"))
return output return output
async def getTweets(init): async def getTweets(init):
''' '''
...@@ -249,6 +246,10 @@ async def main(): ...@@ -249,6 +246,10 @@ async def main():
''' '''
Putting it all together. Putting it all together.
''' '''
if arg.elasticsearch:
print("Indexing to Elasticsearch @" + str(arg.elasticsearch))
if arg.userid is not None: if arg.userid is not None:
arg.u = await getUsername() arg.u = await getUsername()
...@@ -313,8 +314,6 @@ if __name__ == "__main__": ...@@ -313,8 +314,6 @@ if __name__ == "__main__":
arg = ap.parse_args() arg = ap.parse_args()
check() check()
if arg.elasticsearch:
print("Indexing to Elasticsearch @" + str(arg.elasticsearch))
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete(main()) loop.run_until_complete(main())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment