Added source field

twint/cli.py

@@ -120,6 +120,7 @@ def initialize(args):
     c.Min_retweets = args.min_retweets
     c.Min_replies = args.min_replies
     c.Links = args.links
+    c.Source = args.source
     return c
 
 def options():
@@ -211,6 +212,7 @@ def options():
     ap.add_argument("--min-replies", help="Filter the tweets by minimum number of replies.")
     ap.add_argument("--links", help="Include or exclude tweets containing one o more links. If not specified"+
                     " you will get both tweets that might contain links or not.")
+    ap.add_argument("--source", help="Filter the tweets for specific source client.")
     args = ap.parse_args()
 
     return args

twint/storage/db.py

@@ -77,6 +77,7 @@ def init(db):
                     video integer,
                     geo text,
                     near text,
+                    source text,
                     time_update integer not null,
                     PRIMARY KEY (id)
                 );
@@ -246,8 +247,9 @@ def tweets(conn, Tweet, config):
                     Tweet.video,
                     Tweet.geo,
                     Tweet.near,
+                    Tweet.source,
                     time_ms)
-        cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
+        cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
 
         if config.Favorites:
             query = 'INSERT INTO favorites VALUES(?,?)'

twint/storage/elasticsearch.py

@@ -87,7 +87,8 @@ def createIndex(config, instance, **scope):
                         "geo_tweet": {"type": "geo_point"},
                         "photos": {"type": "text"},
                         "user_rt_id": {"type": "integer"},
-                        "mentions": {"type": "keyword"}
+                        "mentions": {"type": "keyword"},
+                        "source": {"type": "keyword"}
                         }
                     },
                     "settings": {
@@ -247,6 +248,8 @@ def Tweet(Tweet, config):
         _t_place = getLocation(Tweet.place)
         if _t_place:
             j_data["_source"].update({"geo_tweet": getLocation(Tweet.place)})
+    if Tweet.source:
+        j_data["_source"].update({"source": Tweet.Source})
     actions.append(j_data)
 
     es = Elasticsearch(config.Elasticsearch, verify_certs=config.Skip_certs)

twint/storage/panda.py

@@ -89,18 +89,11 @@ def update(object, config):
             "nretweets": int(Tweet.retweets_count),
             "quote_url": Tweet.quote_url,
             "search": str(config.Search),
-            "near": config.Near
+            "near": Tweet.near,
+            "geo": Tweet.geo,
+            "source": Tweet.source,
+            "user_rt_id": Tweet.user_rt_id
             }
-        if Tweet.retweet:
-            _data.update({"user_rt_id": Tweet.user_rt_id})
-        try:
-            _data.update({"near": Tweet.near})
-        except AttributeError:
-            pass
-        try:
-            _data.update({"geo": Tweet.geo})
-        except AttributeError:
-            pass
         _object_blocks[_type].append(_data)
     elif _type == "user":
         user = object

twint/storage/write_meta.py

@@ -22,18 +22,12 @@ def tweetData(t):
             "link": t.link,
             "retweet": t.retweet,
             "quote_url": t.quote_url,
-            "video": t.video
+            "video": t.video,
+            "user_rt_id": t.user_rt_id,
+            "near": t.near,
+            "geo": t.geo,
+            "source": t.source
             }
-    if t.retweet:
-        data.update({"user_rt_id": t.user_rt_id})
-    try:
-        data.update({"near": t.near})
-    except AttributeError:
-        pass
-    try:
-        data.update({"geo": t.geo})
-    except AttributeError:
-        pass
     return data
 
 def tweetFieldnames():
@@ -63,7 +57,8 @@ def tweetFieldnames():
             "video",
             "user_rt_id",
             "near",
-            "geo"
+            "geo",
+            "source"
             ]
     return fieldnames
 

twint/tweet.py

@@ -100,4 +100,5 @@ def Tweet(tw, config):
     t.quote_url = getQuoteURL(tw)
     t.near = config.Near if config.Near else ""
     t.geo = config.Geo if config.Geo else ""
+    t.source = config.Source if config.Source else ""
     return t