add Googletranslation (#574)

* add py-translate, update sqlite3 insertion with translations

* add cli argument for translate feature

* add translation to json

* fix output formatting

* add language detection

* add translation source and destination

* extend timeout to 120

* fix tweet and write_meta

* update elasticsearch, tweet exception

* add new arg for destination language for translations

* fix indent

* fix types for elasticsearch

* fix try/expection for translate func

* ignore mysql.py from commits

* fix error

* fix attr names, update panda

* Upgrade to v2.1.8

.gitignore

@@ -2,10 +2,13 @@
 __pycache__/
 *.py[cod]
 *$py.class
-
+tweets.db
 # C extensions
 *.so
 
+config.ini
+twint/storage/mysql.py
+
 # Node Dependency directories
 node_modules/
 jspm_packages/

elasticsearch/index-tweets.json

@@ -33,7 +33,10 @@ PUT twinttweets
           "geo_near": {"type": "geo_point"},
           "geo_tweet": {"type": "geo_point"},
           "photos": {"type": "text"},
-          "mentions": {"type": "text"}
+          "mentions": {"type": "text"},
+          "translation": {"type": "text"},          
+          "trans_src": {"type": "keyword"},
+          "trans_dev": {"type": "keyword"},
       }
     }
   }

requirements.txt

@@ -9,3 +9,4 @@ aiohttp_socks
 schedule
 geopy
 fake-useragent
+googletrans
\ No newline at end of file

setup.py

@@ -17,7 +17,7 @@ VERSION = None
 REQUIRED = [
 		'aiohttp', 'aiodns', 'beautifulsoup4', 'cchardet', 
                 'elasticsearch', 'pysocks', 'pandas', 'aiohttp_socks',
-		'schedule', 'geopy', 'fake-useragent'
+		'schedule', 'geopy', 'fake-useragent','googletrans'
 		]
 
 here = os.path.abspath(os.path.dirname(__file__))

twint/__version__.py

-VERSION = (2, 1, 7)
+VERSION = (2, 1, 8)
 
 __version__ = '.'.join(map(str, VERSION))

twint/cli.py

@@ -17,6 +17,8 @@ from . import run
 from . import config
 from . import storage
 
+# from googletrans import LANGUAGES
+
 def error(_error, message):
     """ Print errors to stdout
     """
@@ -131,6 +133,8 @@ def initialize(args):
     c.Source = args.source
     c.Members_list = args.members_list
     c.Filter_retweets = args.filter_retweets
+    c.Translate = args.translate
+    c.TranslateDest = args.translate_dest
     return c
 
 def options():
@@ -190,6 +194,11 @@ def options():
     ap.add_argument("--profile-full",
                     help="Slow, but effective method of collecting a user's Tweets and RT.",
                     action="store_true")
+    ap.add_argument("--translate",
+                    help="Get tweets translated by Google Translate.",
+                    action="store_true")
+    ap.add_argument("--translate-dest", help="Translate tweet to language (ISO2).",
+                    default="en")
     ap.add_argument("--store-pandas", help="Save Tweets in a DataFrame (Pandas) file.")
     ap.add_argument("--pandas-type",
                     help="Specify HDF5 or Pickle (HDF5 as default)", nargs="?", default="HDF5")

twint/config.py

@@ -73,3 +73,6 @@ class Config:
     Source = None
     Members_list = None
     Filter_retweets = False
+    Translate = False
+    TranslateSrc = "en"
+    TranslateDest = "en"    
\ No newline at end of file

twint/format.py

@@ -28,6 +28,9 @@ def Tweet(config, t):
         output = output.replace("{near}", t.near)
         output = output.replace("{geo}", t.geo)
         output = output.replace("{mentions}", ",".join(t.mentions))
+        output = output.replace("{translate}", t.translate)
+        output = output.replace("{trans_src}", t.trans_src)
+        output = output.replace("{trans_dest}", t.trans_dest)
     else:
         logme.debug(__name__+':Tweet:notFormat')
         output = f"{t.id_str} {t.datestamp} {t.timestamp} {t.timezone} "
@@ -45,7 +48,8 @@ def Tweet(config, t):
             output += f" {cashtags}"
         if config.Stats:
             output += f" | {t.replies_count} replies {t.retweets_count} retweets {t.likes_count} likes"
-
+        if config.Translate:
+            output += f" {t.translate} {t.trans_src} {t.trans_dest}"
     return output
 
 def User(_format, u):

twint/get.py

@@ -148,7 +148,7 @@ async def Request(url, connector=None, params=[], headers=[]):
 
 async def Response(session, url, params=[]):
     logme.debug(__name__+':Response')
-    with timeout(100):
+    with timeout(120):
         async with session.get(url, ssl=True, params=params, proxy=httpproxy) as response:
             return await response.text()
 

twint/storage/db.py

@@ -81,6 +81,9 @@ def init(db):
                     near text,
                     source text,
                     time_update integer not null,
+                    `translate` text default '',
+                    trans_src text default '',
+                    trans_dest text default '',
                     PRIMARY KEY (id)
                 );
         """
@@ -265,8 +268,11 @@ def tweets(conn, Tweet, config):
                     Tweet.geo,
                     Tweet.near,
                     Tweet.source,
-                    time_ms)
-        cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
+                    time_ms,
+                    Tweet.translate,
+                    Tweet.trans_src,
+                    Tweet.trans_dest)
+        cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
 
         if config.Favorites:
             query = 'INSERT INTO favorites VALUES(?,?)'

twint/storage/elasticsearch.py

@@ -98,7 +98,10 @@ def createIndex(config, instance, **scope):
                             }
                         },
                         "retweet_date": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
-                        "urls": {"type": "keyword"}
+                        "urls": {"type": "keyword"},
+                        "translate": {"type": "text"},
+                        "trans_src": {"type": "keyword"},
+                        "trans_dest": {"type": "keyword"},
                         }
                     },
                     "settings": {
@@ -278,6 +281,11 @@ def Tweet(Tweet, config):
             j_data["_source"].update({"geo_tweet": getLocation(Tweet.place)})
     if Tweet.source:
         j_data["_source"].update({"source": Tweet.Source})
+    if config.Translate:
+        j_data["_source"].update({"translate": Tweet.translate})        
+        j_data["_source"].update({"trans_src": Tweet.trans_src})
+        j_data["_source"].update({"trans_dest": Tweet.trans_dest})
+
     actions.append(j_data)
 
     es = Elasticsearch(config.Elasticsearch, verify_certs=config.Skip_certs)

twint/storage/panda.py

@@ -96,7 +96,10 @@ def update(object, config):
             "user_rt": Tweet.user_rt,
             "retweet_id": Tweet.retweet_id,
             "reply_to": Tweet.reply_to,
-            "retweet_date": Tweet.retweet_date
+            "retweet_date": Tweet.retweet_date,
+            "translate": Tweet.translate,
+            "trans_src": Tweet.trans_src,
+            "trans_dest": Tweet.trans_dest
             }
         _object_blocks[_type].append(_data)
     elif _type == "user":

twint/storage/write_meta.py

@@ -30,7 +30,10 @@ def tweetData(t):
             "user_rt": t.user_rt,
             "retweet_id": t.retweet_id,
             "reply_to": t.reply_to,
-            "retweet_date": t.retweet_date
+            "retweet_date": t.retweet_date,
+            "translate": t.translate,
+            "trans_src": t.trans_src,
+            "trans_dest": t.trans_dest
             }
     return data
 
@@ -66,7 +69,10 @@ def tweetFieldnames():
             "user_rt",
             "retweet_id",
             "reply_to",
-            "retweet_date"
+            "retweet_date",
+            "translate",
+            "trans_src",
+            "trans_dest"
             ]
     return fieldnames
 

twint/tweet.py

@@ -3,6 +3,11 @@ from datetime import datetime
 import json
 
 import logging as logme
+from googletrans import Translator
+
+# ref. 
+# - https://github.com/x0rzkov/py-googletrans#basic-usage
+translator = Translator()
 
 class tweet:
     """Define Tweet class
@@ -104,4 +109,17 @@ def Tweet(tw, config):
     t.geo = config.Geo if config.Geo else ""
     t.source = config.Source if config.Source else ""
     t.reply_to = [{'user_id': t['id_str'], 'username': t['screen_name']} for t in json.loads(tw["data-reply-to-users-json"])]
+    t.translate = ''
+    t.trans_src = ''
+    t.trans_dest = ''
+    if config.Translate == True:
+        try:
+            ts = translator.translate(text=t.tweet, dest=config.TranslateDest)
+            t.translate = ts.text
+            t.trans_src = ts.src
+            t.trans_dest = ts.dest
+        # ref. https://github.com/SuniTheFish/ChainTranslator/blob/master/ChainTranslator/__main__.py#L31
+        except ValueError as e:
+            # raise Exception("Invalid destination language: {}".format(config.TranslateDest))
+            logme.debug(__name__+':Tweet:translator.translate:'+str(e))
     return t