Added elasticsearch fun stuff

index.json

+PUT tweep
+{
+  "mappings" : {
+    "items": {
+      "properties": { 
+        "tweetid": {"type": "long"},
+        "datestamp": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
+        "timezone": {"type": "text"},
+        "hashtags": {"type": "text"},
+        "replies": {"type": "integer"},
+        "retweets": {"type": "integer"},
+        "likes": {"type": "integer"},
+        "username": {"type": "keyword"}
+      }
+    }
+  }
+  ,
+  "settings": {
+    "number_of_shards": 1
+  }
+}

requirements.txt

@@ -2,3 +2,4 @@ aiohttp
 aiodns
 beautifulsoup4
 cchardet
+elasticsearch

tweep.py

@@ -10,6 +10,9 @@ import datetime
 import json
 import re
 import sys
+import hashlib
+
+from elasticsearch import Elasticsearch
 
 async def getUrl(init):
     '''
@@ -148,6 +151,18 @@ async def outTweet(tweet):
     except:
         pass
     
+    jObject = {
+        "tweetid": tweetid,
+        "datestamp": date + " " + time,
+        "timezone": timezone,
+        "text": text,
+        "hashtags": re.findall(r'(?i)\#\w+', text, flags=re.UNICODE),
+        "replies": replies,
+        "retweets": retweets,
+        "likes": likes,
+        "username": username
+    }
+
     # Preparing to output
 
     '''
@@ -156,6 +171,12 @@ async def outTweet(tweet):
     generated list into Tweep. That's why these
     modes exist.
     '''
+    if arg.elasticsearch:
+        es = Elasticsearch(arg.elasticsearch)
+        es.index(index="tweep", doc_type="items", id=tweetid, body=json.dumps(jObject))
+        return ""
+
+    else:
         if arg.users:
             output = username
         elif arg.tweets:
@@ -207,6 +228,9 @@ async def getTweets(init):
         copyright = tweet.find("div","StreamItemContent--withheld")
         if copyright is None:
             count +=1
+            if arg.elasticsearch:
+                print(await outTweet(tweet),end=".", flush=True)
+            else:
                 print(await outTweet(tweet))
 
     return tweets, init, count
@@ -285,9 +309,12 @@ if __name__ == "__main__":
     ap.add_argument("--limit", help="Number of Tweets to pull (Increments of 20).")
     ap.add_argument("--count", help="Display number Tweets scraped at the end of session.", action="store_true")
     ap.add_argument("--stats", help="Show number of replies, retweets, and likes", action="store_true")
+    ap.add_argument("-es", "--elasticsearch", help="Index to Elasticsearch")
     arg = ap.parse_args()
 
     check()
+    if arg.elasticsearch:
+        print("Indexing to Elasticsearch @" + str(arg.elasticsearch))
 
     loop = asyncio.get_event_loop()
     loop.run_until_complete(main())