Merge pull request #146 from haccer/master

Update

.travis.yml

@@ -14,4 +14,4 @@ install:
 
 # Test
 script: 
-- python Twint.py -u now -s pineapple --since 2018-01-22 --until 2018-01-23
+- python test.py

README.md

@@ -20,6 +20,11 @@ Some of the benefits of using Twint vs Twitter API:
 - Python 3.5/3.6
 - `pip3 install -r requirements.txt`
 
+## Installing
+- **Git**: `git clone https://github.com/haccer/twint.git`
+- **Pip**: `pip3 install twint`
+- **With MySQL**: `git clone -b mysql --single-branch https://github.com/haccer/twint.git twint-mysql`
+
 ## Basic Examples and Combos.
 A few simple examples to help you understand the basics:
 
@@ -48,9 +53,6 @@ More detail about the commands and options are located in the [wiki](https://git
 ## Using Twint as a Module (Recommended)
 Twint can now be used as a module and supports custom formatting. **More details are located in the [wiki](https://github.com/haccer/twint/wiki/Module)**
 
-#### Install
-- `sudo pip3 install twint`
-
 #### Example
 ```python
 import twint
@@ -72,8 +74,9 @@ twint.run.Search(c)
 - CSV
 - JSON
 - SQLite
+- Mysql (DB collation utf8mb4)
 - Elasticsearch
-
+- MySQL (See MySQL Branch)
 ### Elasticsearch Setup
 
 Details on setting up Elasticsearch with Twint is located in the [wiki](https://github.com/haccer/twint/wiki/Elasticsearch). 

Twint.py

@@ -31,6 +31,10 @@ def check(args):
             error("Error", "Please specify an output file (Example: -o file.csv).")
         elif args.json:
             error("Error", "Please specify an output file (Example: -o file.json).")
+    if args.hostname:
+        if args.Database is None or args.DB_user is None or args.DB_pwd is None:
+            error("Error", "Please specify database name, user and password")
+
 
     if not args.followers and not args.following:
         if args.user_full:
@@ -96,7 +100,10 @@ def initialize(args):
     c.Limit = args.limit
     c.Count = args.count
     c.Stats = args.stats
+    c.hostname = args.hostname
     c.Database = args.database
+    c.DB_user = args.DB_user
+    c.DB_pwd = args.DB_pwd
     c.To = args.to
     c.All = args.all
     c.Essid = args.essid
@@ -105,6 +112,7 @@ def initialize(args):
     c.Profile_full = args.profile_full
     c.Store_pandas = args.store_pandas
     c.Pandas_type = args.pandas_type
+    c.search_name = args.search_name
     return c
 
 def options():
@@ -134,7 +142,10 @@ def options():
     ap.add_argument("--count", help="Display number of Tweets scraped at the end of session.",
             action="store_true")
     ap.add_argument("--stats", help="Show number of replies, retweets, and likes.", action="store_true")
-    ap.add_argument("-db", "--database", help="Store Tweets in a sqlite3 database.")
+    ap.add_argument("--hostname", help="Store the mysql database host")
+    ap.add_argument("-db", "--database", help="Store Tweets in a sqlite3  or mysql database.")
+    ap.add_argument("--DB_user", help="Store the mysql database user")
+    ap.add_argument("--DB_pwd", help="Store the mysql database pwd")
     ap.add_argument("--to", help="Search Tweets to a user.")
     ap.add_argument("--all", help="Search all Tweets associated with a user.")
     ap.add_argument("--followers", help="Scrape a person's followers.", action="store_true")
@@ -154,7 +165,9 @@ def options():
             action="store_true")
     ap.add_argument("--store-pandas", help="Save Tweets in a DataFrame (Pandas) file.")
     ap.add_argument("--pandas-type", help="Specify HDF5 or Pickle (HDF5 as default)")
+    ap.add_argument("--search_name", help="name for identify the search like -3dprinter stuff- only for mysql")
     args = ap.parse_args()
+    
     return args
 
 def main():

db-schema/twint_db_mysql.sql

+CREATE DATABASE  IF NOT EXISTS `twitterdata_v9` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
+USE `twitterdata_v9`;
+-- MySQL dump 10.13  Distrib 5.7.22, for Linux (x86_64)
+--
+-- Host: localhost    Database: twitterdata_v9
+-- ------------------------------------------------------
+-- Server version	5.7.22-0ubuntu0.16.04.1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `followers`
+--
+
+DROP TABLE IF EXISTS `followers`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `followers` (
+  `id` bigint(30) NOT NULL,
+  `name` mediumtext,
+  `username` text NOT NULL,
+  `bio` longtext,
+  `location` tinytext,
+  `url` longtext,
+  `join_date` tinytext NOT NULL,
+  `join_time` tinytext NOT NULL,
+  `tweets` int(11) DEFAULT NULL,
+  `following` int(11) DEFAULT NULL,
+  `followers` int(11) DEFAULT NULL,
+  `likes` int(11) DEFAULT NULL,
+  `media` int(11) DEFAULT NULL,
+  `private` tinytext NOT NULL,
+  `verified` tinytext NOT NULL,
+  `avatar` longtext NOT NULL,
+  `date_update` datetime NOT NULL,
+  `follower` text NOT NULL,
+  PRIMARY KEY (`follower`(255),`username`(255),`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `followers_names`
+--
+
+DROP TABLE IF EXISTS `followers_names`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `followers_names` (
+  `user` text NOT NULL,
+  `date_update` datetime NOT NULL,
+  `follower` text NOT NULL,
+  PRIMARY KEY (`user`(255),`follower`(255))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `following`
+--
+
+DROP TABLE IF EXISTS `following`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `following` (
+  `id` bigint(30) NOT NULL,
+  `name` mediumtext,
+  `username` text NOT NULL,
+  `bio` longtext,
+  `location` tinytext,
+  `url` longtext,
+  `join_date` tinytext NOT NULL,
+  `join_time` tinytext NOT NULL,
+  `tweets` int(11) DEFAULT NULL,
+  `following` int(11) DEFAULT NULL,
+  `followers` int(11) DEFAULT NULL,
+  `likes` int(11) DEFAULT NULL,
+  `media` int(11) DEFAULT NULL,
+  `private` tinytext NOT NULL,
+  `verified` tinytext NOT NULL,
+  `avatar` longtext NOT NULL,
+  `date_update` datetime NOT NULL,
+  `follows` text NOT NULL,
+  PRIMARY KEY (`id`,`username`(255),`follows`(255))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `following_names`
+--
+
+DROP TABLE IF EXISTS `following_names`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `following_names` (
+  `user` text NOT NULL,
+  `date_update` datetime NOT NULL,
+  `follows` text NOT NULL,
+  PRIMARY KEY (`user`(255),`follows`(255))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `tweets`
+--
+
+DROP TABLE IF EXISTS `tweets`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `tweets` (
+  `id` bigint(30) NOT NULL,
+  `user_id` bigint(30) DEFAULT NULL,
+  `date` date NOT NULL,
+  `time` time NOT NULL,
+  `timezone` tinytext NOT NULL,
+  `location` tinytext NOT NULL,
+  `user` text NOT NULL,
+  `tweet` longtext NOT NULL,
+  `replies` int(11) DEFAULT NULL,
+  `likes` int(11) DEFAULT NULL,
+  `retweets` int(11) DEFAULT NULL,
+  `hashtags` longtext,
+  `link` longtext,
+  `retweet` int(1) DEFAULT NULL,
+  `user_rt` text,
+  `mentions` longtext,
+  `date_update` datetime NOT NULL,
+  `search_name` mediumtext NOT NULL COMMENT 'user can use this field to know from which search the info comes. max 255 chars. if the user do not especify, it must be set to "-" ',
+  PRIMARY KEY (`id`,`search_name`(255))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-06-03 14:52:08

requirements.txt

@@ -6,3 +6,4 @@ elasticsearch
 pysocks
 pandas
 tables
+mysqlclient

test.py

+import twint
+import os
+
+'''
+Test.py - Testing TWINT to make sure everything works.
+'''
+
+def test_reg(c, run):
+    print("[+] Beginning vanilla test in {}".format(str(run)))
+    run(c)
+
+def test_db(c, run):
+    print("[+] Beginning DB test in {}".format(str(run)))
+    c.Database = "test_twint.db"
+    run(c)
+
+def custom(c, run, _type):
+    print("[+] Beginning custom {} test in {}".format(_type, str(run)))
+    c.Custom = ["id", "username"]
+    run(c)
+
+def test_json(c, run):
+    c.Store_json = True
+    c.Output = "test_twint.json"
+    custom(c, run, "JSON")
+    print("[+] Beginning JSON test in {}".format(str(run)))
+    run(c)
+
+def test_csv(c, run):
+    c.Store_csv = True
+    c.Output = "test_twint.csv"
+    custom(c, run, "CSV")
+    print("[+] Beginning CSV test in {}".format(str(run)))
+    run(c)
+
+def main():
+    c = twint.Config()
+    c.Username = "verified"
+    c.Limit = 20
+    c.Store_object = True
+
+    # Seperate objects are neccessary.
+
+    f = twint.Config()
+    f.Username = "verified"
+    f.Limit = 20
+    f.Store_object = True
+    f.User_full = True
+    
+    runs = [twint.run.Following,
+            twint.run.Followers,
+            twint.run.Search,
+            twint.run.Profile,
+            twint.run.Favorites
+            ]
+    
+    tests = [test_reg, test_json, test_csv, test_db]
+    
+    # Something breaks if we don't split these up
+
+    for run in runs[:2]: 
+        for test in tests:
+            test(f, run)
+
+    for run in runs[2:]:
+        if run == twint.run.Search:
+            c.Since = "2012-1-1"
+            c.Until = "2017-1-1"
+        else:
+            c.Since = ""
+            c.Until = ""
+        
+        for test in tests:
+            test(c, run)
+
+    files = ["test_twint.db", "test_twint.json", "test_twint.csv"]
+    for _file in files:
+       os.remove(_file)
+
+
+    print("[+] Testing complete!")
+
+if __name__ == '__main__':
+    main()

twint/__version__.py

-VERSION = (1, 1, 3, 4)
+VERSION = (1, 1, 3, 4) #mysql support
 
 __version__ = '.'.join(map(str, VERSION))

twint/config.py

@@ -21,7 +21,10 @@ class Config:
     Limit = None
     Count = None
     Stats = False  
+    hostname = None #mysql
     Database = None
+    DB_user = None #mysql
+    DB_pwd = None  #mysql
     To = None
     All = None
     Debug = False
@@ -38,3 +41,4 @@ class Config:
     Store_pandas = False
     Pandas_type = None
     Pandas = False
+    search_name = "-" #for identify a records in mysql with the search it provides from.  it cannot be null for DB requirements. a tweet must be in several search so the PK are tweet ID and search_name

twint/db.py

@@ -21,7 +21,7 @@ def init(db):
         table_tweets = """
             CREATE TABLE IF NOT EXISTS
                 tweets (
-                    id integer primary key,
+                    id integer not null,
                     user_id integer,
                     date text not null,
                     time text not null,
@@ -37,7 +37,9 @@ def init(db):
                     retweet bool,
                     user_rt text,
                     mentions text,
-                    date_update text not null
+                    date_update text not null,
+                    search_name text not null,
+                    PRIMARY KEY (id, search_name)
                 );
         """
         cursor.execute(table_tweets)
@@ -175,7 +177,7 @@ def user(conn, Username, Followers,  User):
     except sqlite3.IntegrityError:
         pass
 
-def tweets(conn, Tweet):
+def tweets(conn, Tweet, config):
     try:
         date_time = str(datetime.now())
         cursor = conn.cursor()
@@ -195,8 +197,9 @@ def tweets(conn, Tweet):
                     Tweet.retweet,
                     Tweet.user_rt,
                     ",".join(Tweet.mentions),
-                    date_time,)
-        cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
+                    date_time,
+                    config.search_name,)
+        cursor.execute('INSERT INTO tweets VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', entry)
         conn.commit()
     except sqlite3.IntegrityError:
         pass

twint/dbmysql.py

+from datetime import datetime
+import MySQLdb
+import sys
+
+def Conn(hostname,Database,db_user,db_pwd):
+    if Database:
+        print("[+] Inserting into Database: " + str(Database))
+        conn = init(hostname,Database,db_user,db_pwd)
+        if isinstance(conn, str):
+            print(str)
+            sys.exit(1)
+    else:
+        conn = ""
+
+    return conn
+
+def init(hostname,Database,db_user,db_pwd):
+    try:
+        conn = MySQLdb.connect(host=hostname,    # your host, usually localhost
+                     user=db_user,         # your username
+                     passwd=db_pwd,  # your password
+                     db=Database,# name of the data base
+					 charset='utf8mb4',
+                     use_unicode=True)          
+        cursor = conn.cursor()
+       #here would be the code for creating the tables if them don't exist
+        return conn
+    except Exception as e:
+        return str(e)
+
+def fTable(Followers):
+    if Followers:
+        table = "followers_names"
+    else:
+        table = "following_names"
+    
+    return table
+
+def uTable(Followers):
+    if Followers:
+        table = "followers"
+    else:
+        table = "following"
+    
+    return table
+
+def follow(conn, Username, Followers, User):
+    try:
+        date_time = str(datetime.now())
+        cursor = conn.cursor()
+        entry = (User, date_time, Username,)
+        query = 'INSERT INTO {} VALUES(%s,%s,%s)'.format(fTable(Followers))
+        cursor.execute(query, entry)
+        conn.commit()
+    except MySQLdb.IntegrityError:
+        pass
+
+def user(conn, Username, Followers,  User):
+    try:
+        date_time = str(datetime.now())
+        cursor = conn.cursor()
+        entry = (User.id,
+                User.name,
+                User.username,
+                User.bio,
+                User.location,
+                User.url,
+                User.join_date,
+                User.join_time,
+                User.tweets,
+                User.following,
+                User.followers,
+                User.likes,
+                User.media_count,
+                User.is_private,
+                User.is_verified,
+                User.avatar, 
+                date_time,
+                Username,)
+        query = 'INSERT INTO {} VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(uTable(Followers))
+        cursor.execute(query, entry)
+        conn.commit()
+ 
+    except MySQLdb.IntegrityError:
+        pass
+           
+
+def tweets(conn, Tweet, config):
+    try:
+        date_time = str(datetime.now())
+        cursor = conn.cursor()
+        entry = (Tweet.id,
+                    Tweet.user_id,
+                    Tweet.datestamp,
+                    Tweet.timestamp,
+                    Tweet.timezone,
+                    Tweet.location,
+                    Tweet.username,
+                    Tweet.tweet,
+                    Tweet.replies,
+                    Tweet.likes,
+                    Tweet.retweets,
+                    ",".join(Tweet.hashtags),
+                    Tweet.link,
+                    Tweet.retweet,
+                    Tweet.user_rt,
+                    ",".join(Tweet.mentions),
+                    date_time,
+                    config.search_name,)
+        cursor.execute('INSERT INTO tweets VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', entry)
+        conn.commit()
+    except MySQLdb.IntegrityError:
+        pass

twint/favorites.py

-from . import feed, get, db, output, verbose
+from . import feed, get, db, output, verbose, dbmysql
 
 class Favorites:
     def __init__(self, config):
@@ -6,6 +6,9 @@ class Favorites:
         self.feed = [-1]
         self.count = 0
         self.config = config
+        if config.hostname:
+            self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
+        else:
             self.conn = db.Conn(config.Database)
         self.config.Favorites = True
         

twint/follow.py

-from . import feed, get, db, output, verbose
+from . import feed, get, db, output, verbose, dbmysql
 
 class Follow:
     def __init__(self, config):
@@ -6,6 +6,9 @@ class Follow:
         self.feed = [-1]
         self.count = 0
         self.config = config
+        if config.hostname:
+            self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
+        else:
             self.conn = db.Conn(config.Database)
         verbose.Elastic(config)
 

twint/output.py

 from datetime import datetime
 
-from . import db, elasticsearch, format, write, Pandas
+from . import db, elasticsearch, format, write, Pandas, dbmysql
 from .tweet import Tweet
 from .user import User
 
+
 tweets_object = []
 
 def datecheck(datestamp, config):
@@ -32,7 +33,6 @@ def _output(obj, output, config):
 
     if config.Pandas:
         Pandas.update(obj, config.Essid)
-
     if config.Elasticsearch:
         if config.Store_object:
             tweets_object.append(obj)
@@ -51,18 +51,21 @@ async def Tweets(tw, location, config, conn):
         if datecheck(tweet.datestamp, config):
             output = format.Tweet(config, tweet)
            
-            if config.Database:
-                db.tweets(conn, tweet)
+            if config.hostname:
+                dbmysql.tweets(conn, tweet, config)
+            elif config.Database:
+                db.tweets(conn, tweet, config)
             if config.Elasticsearch:
                 elasticsearch.Tweet(tweet, config.Elasticsearch, config.Essid)
-                
             _output(tweet, output, config)
 
 async def Users(u, config, conn):
     user = User(u)
     output = format.User(config.Format, user)
 
-    if config.Database:
+    if config.hostname:
+        dbmysql.user(conn, config.Username, config.Followers, user)
+    elif config.Database:
         db.user(conn, config.Username, config.Followers, user)
 
     if config.Elasticsearch:
@@ -78,7 +81,9 @@ async def Users(u, config, conn):
     _output(user, output, config)
 
 async def Username(username, config, conn):
-    if config.Database:
+    if config.hostname:
+        dbmysql.follow(conn, config.Username, config.Followers, username)
+    elif config.Database:
         db.follow(conn, config.Username, config.Followers, username)
 
     if config.Elasticsearch:

twint/profile.py

-from . import db, get, feed, output, verbose
+from . import db, get, feed, output, verbose, dbmysql
 
 class Profile:
     def __init__(self, config):
@@ -6,7 +6,12 @@ class Profile:
         self.feed = [-1]
         self.count = 0
         self.config = config
+
+        if config.hostname:
+            self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
+        else:
             self.conn = db.Conn(config.Database)
+
         self.config.Profile = True
 
         verbose.Elastic(config)

twint/run.py

@@ -6,6 +6,10 @@ def run(x):
 
 def Favorites(config):
     config.Favorites = True
+    if config.Username:
+        config.search_name="Favourites"+str(config.Username) #to identify to which user is related to
+    else:
+        config.search_name="Favourites"+str(config.User_id) #to identify to which user is related to
     run(favorites.Favorites(config).main())
 
 def Followers(config):

twint/search.py

-from . import datelock, db, get, feed, output, verbose
+from . import datelock, db, get, feed, output, verbose, dbmysql
 from datetime import timedelta
 
 class Search:
@@ -7,6 +7,9 @@ class Search:
         self.feed = [-1]
         self.count = 0
         self.config = config
+        if config.hostname:
+            self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
+        else:
             self.conn = db.Conn(config.Database)
         self.d = datelock.Set(self.config.Until, self.config.Since)
         self.config.TwitterSearch = True

twint/url.py

@@ -46,7 +46,7 @@ async def Profile(username, init):
 async def Search(config, init):
     url = "{}/search/timeline?f=tweets&vertical=default&lang=en".format(base)
     url += "&include_available_features=1&include_entities=1&"
-    url += "reset_error_state=false&src=typd&max_position={}&q=".format(init)
+    url += "reset_error_state=false&src=typd&qf=off&max_position={}&q=".format(init)
 
     if config.Lang:
         url = url.replace("lang=en", "l={0.Lang}&lang=en".format(config))
@@ -79,6 +79,6 @@ async def Search(config, init):
     if config.Near:
         config.Near = config.Near.replace(" ", "%20")
         config.Near = config.Near.replace(",", "%2C")
-        url += "%20near%3A{0.Near}".format(config)
+        url += "%20near%3A%22{0.Near}%22".format(config)
 
     return url