Merge pull request #155 from twintproject/master

Merging master into dev

README.md

@@ -21,9 +21,9 @@ Some of the benefits of using Twint vs Twitter API:
 - `pip3 install -r requirements.txt`
 
 ## Installing
-- **Git**: `git clone https://github.com/haccer/twint.git`
+- **Git**: `git clone https://github.com/twintproject/twint.git`
 - **Pip**: `pip3 install twint`
-- **With MySQL**: `git clone -b mysql --single-branch https://github.com/haccer/twint.git twint-mysql`
+- **With MySQL**: `git clone -b mysql --single-branch https://github.com/twintproject/twint.git twint-mysql`
 
 ## Basic Examples and Combos.
 A few simple examples to help you understand the basics:
@@ -48,10 +48,10 @@ A few simple examples to help you understand the basics:
 - `python3 twint.py -u username --profile-full` - Use a slow, but effective method to gather all the Tweets from a user's profile (Including Retweets).
 - `python3 twint.py -u username --retweets` - Use a quick method to gather the last 900 Tweets (that includes retweets) from a user's profile.
 
-More detail about the commands and options are located in the [wiki](https://github.com/haccer/twint/wiki/Commands)
+More detail about the commands and options are located in the [wiki](https://github.com/twintproject/twint/wiki/Commands)
 
 ## Using Twint as a Module (Recommended)
-Twint can now be used as a module and supports custom formatting. **More details are located in the [wiki](https://github.com/haccer/twint/wiki/Module)**
+Twint can now be used as a module and supports custom formatting. **More details are located in the [wiki](https://github.com/twintproject/twint/wiki/Module)**
 
 #### Example
 ```python
@@ -79,19 +79,15 @@ twint.run.Search(c)
 - MySQL (See MySQL Branch)
 ### Elasticsearch Setup
 
-Details on setting up Elasticsearch with Twint is located in the [wiki](https://github.com/haccer/twint/wiki/Elasticsearch). 
+Details on setting up Elasticsearch with Twint is located in the [wiki](https://github.com/twintproject/twint/wiki/Elasticsearch). 
 
 ### Graph Visualization
 ![graph](https://i.imgur.com/EEJqB8n.png)
 
-[Graph](https://github.com/haccer/twint/tree/master/graph) details are also located in the [wiki](https://github.com/haccer/twint/wiki/Graph). 
+[Graph](https://github.com/twintproject/twint/tree/master/graph) details are also located in the [wiki](https://github.com/twintproject/twint/wiki/Graph). 
 
 We are testing a (free) graph plugin for Kibana, details located in the Wiki!
 
-## Thanks
-Thanks to [@hpiedcoq](https://github.com/hpiedcoq) & [@pielco11](https://github.com/pielco11) for contributing several features!
-
 ## Contact
-Shout me out on Twitter: [@now](https://twitter.com/now)
 
-If you have problems or have suggestions don't hesitate to open an issue or ask about it directly. 
+If you have any questions, want to join in on discussions, or need extra help, you are welcome to join our OSINT focused [Slack server](https://os-int.slack.com/join/shared_invite/enQtMzc4NzY5ODI3NDI3LWRlOGNhN2U3OTUwY2Q1OTk5MDI2YjliOWQ1OTI5NzAyZjc0MDhiYTQ3NTY4MjMxY2E0MTRhOTVlN2M0ZmJhMjI).

twint/Pandas.py

@@ -24,7 +24,8 @@ def update(Tweet, session):
                 "link": Tweet.link,
                 "retweet": Tweet.retweet,
                 "user_rt": Tweet.user_rt,
-                "essid": str(session)
+                "essid": str(session),
+                'mentions': Tweet.mentions
                 }
     _blocks.append(_data)
 

twint/config.py

@@ -44,4 +44,4 @@ class Config:
     Search_name = "-" #for identify a records in mysql with the search it provides from.  it cannot be null for DB requirements. a tweet must be in several search so the PK are tweet ID and search_name
     Index_tweets = "twint"
     Index_follow = "twintGraph"
-    Index_users = "twintUser"
+    Index_users = "twintUser"
\ No newline at end of file

twint/db.py

@@ -198,7 +198,7 @@ def tweets(conn, Tweet, config):
                     Tweet.user_rt,
                     ",".join(Tweet.mentions),
                     date_time,
-                    config.Search_name,)
+                    config.search_name,)
         cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
         conn.commit()
     except sqlite3.IntegrityError:

twint/dbmysql.py

@@ -21,7 +21,7 @@ def init(hostname,Database,db_user,db_pwd):
                      passwd=db_pwd,  # your password
                      db=Database,# name of the data base
 					 charset='utf8mb4',
-                     use_unicode=True)
+                     use_unicode=True)          
         cursor = conn.cursor()
        #here would be the code for creating the tables if them don't exist
         return conn
@@ -33,7 +33,6 @@ def fTable(Followers):
         table = "followers_names"
     else:
         table = "following_names"
-
     return table
 
 def uTable(Followers):
@@ -41,7 +40,6 @@ def uTable(Followers):
         table = "followers"
     else:
         table = "following"
-
     return table
 
 def follow(conn, Username, Followers, User):
@@ -74,13 +72,12 @@ def user(conn, Username, Followers,  User):
                 User.media_count,
                 User.is_private,
                 User.is_verified,
-                User.avatar,
+                User.avatar, 
                 date_time,
                 Username,)
         query = 'INSERT INTO {} VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(uTable(Followers))
         cursor.execute(query, entry)
         conn.commit()
-
     except MySQLdb.IntegrityError:
         pass
 
@@ -106,7 +103,7 @@ def tweets(conn, Tweet, config):
                     Tweet.user_rt,
                     ",".join(Tweet.mentions),
                     date_time,
-                    config.Search_name,)
+                    config.search_name,)
         cursor.execute('INSERT INTO tweets VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', entry)
         conn.commit()
     except MySQLdb.IntegrityError:

twint/output.py

@@ -42,7 +42,10 @@ def _output(obj, output, config):
         if config.Store_object:
             tweets_object.append(obj)
         else:
-            print(output)
+            try:
+                print(output)
+            except UnicodeEncodeError:
+                pass
 
 async def Tweets(tw, location, config, conn):
     copyright = tw.find("div", "StreamItemContent--withheld")