Commit 8bb3ac6b authored by Cody Zacharias's avatar Cody Zacharias Committed by GitHub

Cleanup & Moving stuff around

parent 6f1f5629
#!/usr/bin/python3 #!/usr/bin/python3
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from elasticsearch import Elasticsearch
from time import gmtime, strftime from time import gmtime, strftime
import argparse import argparse
import aiohttp import aiohttp
...@@ -7,12 +8,10 @@ import asyncio ...@@ -7,12 +8,10 @@ import asyncio
import async_timeout import async_timeout
import csv import csv
import datetime import datetime
import hashlib
import json import json
import re import re
import sys import sys
import hashlib
from elasticsearch import Elasticsearch
async def getUrl(init): async def getUrl(init):
''' '''
...@@ -151,6 +150,15 @@ async def outTweet(tweet): ...@@ -151,6 +150,15 @@ async def outTweet(tweet):
except: except:
pass pass
# Preparing to output
'''
There were certain cases where I used Tweep
to gather a list of users and then fed that
generated list into Tweep. That's why these
modes exist.
'''
if arg.elasticsearch:
jObject = { jObject = {
"tweetid": tweetid, "tweetid": tweetid,
"datestamp": date + " " + time, "datestamp": date + " " + time,
...@@ -163,21 +171,10 @@ async def outTweet(tweet): ...@@ -163,21 +171,10 @@ async def outTweet(tweet):
"username": username "username": username
} }
# Preparing to output
'''
There were certain cases where I used Tweep
to gather a list of users and then fed that
generated list into Tweep. That's why these
modes exist.
'''
if arg.elasticsearch:
es = Elasticsearch(arg.elasticsearch) es = Elasticsearch(arg.elasticsearch)
es.index(index="tweep", doc_type="items", id=tweetid, body=json.dumps(jObject)) es.index(index="tweep", doc_type="items", id=tweetid, body=json.dumps(jObject))
return "" output = ""
elif arg.users:
else:
if arg.users:
output = username output = username
elif arg.tweets: elif arg.tweets:
output = tweets output = tweets
...@@ -249,6 +246,10 @@ async def main(): ...@@ -249,6 +246,10 @@ async def main():
''' '''
Putting it all together. Putting it all together.
''' '''
if arg.elasticsearch:
print("Indexing to Elasticsearch @" + str(arg.elasticsearch))
if arg.userid is not None: if arg.userid is not None:
arg.u = await getUsername() arg.u = await getUsername()
...@@ -313,8 +314,6 @@ if __name__ == "__main__": ...@@ -313,8 +314,6 @@ if __name__ == "__main__":
arg = ap.parse_args() arg = ap.parse_args()
check() check()
if arg.elasticsearch:
print("Indexing to Elasticsearch @" + str(arg.elasticsearch))
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete(main()) loop.run_until_complete(main())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment