Skip to content

T
Twint
Commit d7681f0b authored by Cody Zacharias's avatar Cody Zacharias Committed by GitHub
Browse files
Options

Merge to Dev (#157) 

* Delete twint_db_sqlite3

* Delete twint_db_mysql.sql

* Delete dbmysql.py

* Update output.py

* Update favorites.py

* Update follow.py

* Update profile.py

* Update search.py

* Removed dbmysql related stuff

* Deleted import

* Update

* Update

* Update favorites.py
parent 958c739e
Hide whitespace changes
Inline Side-by-side
Showing with 20 additions and 297 deletions
db-schema/twint_db_mysql.sql deleted 100644 → 0
View file @ 958c739e
CREATE DATABASE IF NOT EXISTS `twitterdata_v9` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
USE `twitterdata_v9`;
-- MySQL dump 10.13 Distrib 5.7.22, for Linux (x86_64)
--
-- Host: localhost Database: twitterdata_v9
-- ------------------------------------------------------
-- Server version 5.7.22-0ubuntu0.16.04.1
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `followers`
--
DROP TABLE IF EXISTS `followers`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `followers` (
`id` bigint(30) NOT NULL,
`name` mediumtext,
`username` text NOT NULL,
`bio` longtext,
`location` tinytext,
`url` longtext,
`join_date` tinytext NOT NULL,
`join_time` tinytext NOT NULL,
`tweets` int(11) DEFAULT NULL,
`following` int(11) DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`likes` int(11) DEFAULT NULL,
`media` int(11) DEFAULT NULL,
`private` tinytext NOT NULL,
`verified` tinytext NOT NULL,
`avatar` longtext NOT NULL,
`date_update` datetime NOT NULL,
`follower` text NOT NULL,
PRIMARY KEY (`follower`(255),`username`(255),`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `followers_names`
--
DROP TABLE IF EXISTS `followers_names`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `followers_names` (
`user` text NOT NULL,
`date_update` datetime NOT NULL,
`follower` text NOT NULL,
PRIMARY KEY (`user`(255),`follower`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `following`
--
DROP TABLE IF EXISTS `following`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `following` (
`id` bigint(30) NOT NULL,
`name` mediumtext,
`username` text NOT NULL,
`bio` longtext,
`location` tinytext,
`url` longtext,
`join_date` tinytext NOT NULL,
`join_time` tinytext NOT NULL,
`tweets` int(11) DEFAULT NULL,
`following` int(11) DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`likes` int(11) DEFAULT NULL,
`media` int(11) DEFAULT NULL,
`private` tinytext NOT NULL,
`verified` tinytext NOT NULL,
`avatar` longtext NOT NULL,
`date_update` datetime NOT NULL,
`follows` text NOT NULL,
PRIMARY KEY (`id`,`username`(255),`follows`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `following_names`
--
DROP TABLE IF EXISTS `following_names`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `following_names` (
`user` text NOT NULL,
`date_update` datetime NOT NULL,
`follows` text NOT NULL,
PRIMARY KEY (`user`(255),`follows`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `tweets`
--
DROP TABLE IF EXISTS `tweets`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `tweets` (
`id` bigint(30) NOT NULL,
`user_id` bigint(30) DEFAULT NULL,
`date` date NOT NULL,
`time` time NOT NULL,
`timezone` tinytext NOT NULL,
`location` tinytext NOT NULL,
`user` text NOT NULL,
`tweet` longtext NOT NULL,
`replies` int(11) DEFAULT NULL,
`likes` int(11) DEFAULT NULL,
`retweets` int(11) DEFAULT NULL,
`hashtags` longtext,
`link` longtext,
`retweet` int(1) DEFAULT NULL,
`user_rt` text,
`mentions` longtext,
`date_update` datetime NOT NULL,
`search_name` mediumtext NOT NULL COMMENT 'user can use this field to know from which search the info comes. max 255 chars. if the user do not especify, it must be set to "-" ',
PRIMARY KEY (`id`,`search_name`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2018-06-03 14:52:08
twint/dbmysql.py deleted 100644 → 0
View file @ 958c739e
from datetime import datetime
import MySQLdb
import sys
def Conn(hostname,Database,db_user,db_pwd):
if Database:
print("[+] Inserting into Database: " + str(Database))
conn = init(hostname,Database,db_user,db_pwd)
if isinstance(conn, str):
print(str)
sys.exit(1)
else:
conn = ""
return conn
def init(hostname,Database,db_user,db_pwd):
try:
conn = MySQLdb.connect(host=hostname, # your host, usually localhost
user=db_user, # your username
passwd=db_pwd, # your password
db=Database,# name of the data base
charset='utf8mb4',
use_unicode=True)
cursor = conn.cursor()
#here would be the code for creating the tables if them don't exist
return conn
except Exception as e:
return str(e)
def fTable(Followers):
if Followers:
table = "followers_names"
else:
table = "following_names"
return table
def uTable(Followers):
if Followers:
table = "followers"
else:
table = "following"
return table
def follow(conn, Username, Followers, User):
try:
date_time = str(datetime.now())
cursor = conn.cursor()
entry = (User, date_time, Username,)
query = 'INSERT INTO {} VALUES(%s,%s,%s)'.format(fTable(Followers))
cursor.execute(query, entry)
conn.commit()
except MySQLdb.IntegrityError:
pass
def user(conn, Username, Followers, User):
try:
date_time = str(datetime.now())
cursor = conn.cursor()
entry = (User.id,
User.name,
User.username,
User.bio,
User.location,
User.url,
User.join_date,
User.join_time,
User.tweets,
User.following,
User.followers,
User.likes,
User.media_count,
User.is_private,
User.is_verified,
User.avatar,
date_time,
Username,)
query = 'INSERT INTO {} VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(uTable(Followers))
cursor.execute(query, entry)
conn.commit()
except MySQLdb.IntegrityError:
pass
def tweets(conn, Tweet, config):
try:
date_time = str(datetime.now())
cursor = conn.cursor()
entry = (Tweet.id,
Tweet.user_id,
Tweet.datestamp,
Tweet.timestamp,
Tweet.timezone,
Tweet.location,
Tweet.username,
Tweet.tweet,
Tweet.replies,
Tweet.likes,
Tweet.retweets,
",".join(Tweet.hashtags),
Tweet.link,
Tweet.retweet,
Tweet.user_rt,
",".join(Tweet.mentions),
date_time,
config.search_name,)
cursor.execute('INSERT INTO tweets VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', entry)
conn.commit()
except MySQLdb.IntegrityError:
pass
twint/favorites.py
View file @ d7681f0b
from . import feed, get, db, output, verbose, dbmysql from . import feed, get, db, output, verbose
class Favorites: class Favorites:
def __init__(self, config): def __init__(self, config):
...@@ -6,10 +6,7 @@ class Favorites: ...@@ -6,10 +6,7 @@ class Favorites:
self.feed = [-1] self.feed = [-1]
self.count = 0 self.count = 0
self.config = config self.config = config
if config.hostname: self.conn = db.Conn(config.Database)
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
self.config.Favorites = True self.config.Favorites = True
verbose.Elastic(config) verbose.Elastic(config)
......
twint/follow.py
View file @ d7681f0b
from . import feed, get, db, output, verbose, dbmysql from . import feed, get, db, output, verbose
class Follow: class Follow:
def __init__(self, config): def __init__(self, config):
...@@ -6,10 +6,7 @@ class Follow: ...@@ -6,10 +6,7 @@ class Follow:
self.feed = [-1] self.feed = [-1]
self.count = 0 self.count = 0
self.config = config self.config = config
if config.hostname: self.conn = db.Conn(config.Database)
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
verbose.Elastic(config) verbose.Elastic(config)
async def Feed(self): async def Feed(self):
...@@ -27,7 +24,7 @@ class Follow: ...@@ -27,7 +24,7 @@ class Follow:
else: else:
for user in self.feed: for user in self.feed:
self.count += 1 self.count += 1
username = user.find("a")["name"] username = user.find("a")["name"]
await output.Username(username, self.config, self.conn) await output.Username(username, self.config, self.conn)
async def main(self): async def main(self):
...@@ -39,8 +36,8 @@ class Follow: ...@@ -39,8 +36,8 @@ class Follow:
await self.follow() await self.follow()
else: else:
break break
if get.Limit(self.config.Limit, self.count): if get.Limit(self.config.Limit, self.count):
break break
verbose.Count(self.config, self.count) verbose.Count(self.config, self.count)
twint/output.py
View file @ d7681f0b
from datetime import datetime from datetime import datetime
from . import db, elasticsearch, format, write, Pandas, dbmysql from . import db, elasticsearch, format, write, Pandas
from .tweet import Tweet from .tweet import Tweet
from .user import User from .user import User
...@@ -54,9 +54,7 @@ async def Tweets(tw, location, config, conn): ...@@ -54,9 +54,7 @@ async def Tweets(tw, location, config, conn):
if datecheck(tweet.datestamp, config): if datecheck(tweet.datestamp, config):
output = format.Tweet(config, tweet) output = format.Tweet(config, tweet)
if config.hostname: if config.Database:
dbmysql.tweets(conn, tweet, config)
elif config.Database:
db.tweets(conn, tweet, config) db.tweets(conn, tweet, config)
if config.Elasticsearch: if config.Elasticsearch:
elasticsearch.Tweet(tweet, config) elasticsearch.Tweet(tweet, config)
...@@ -66,9 +64,7 @@ async def Users(u, config, conn): ...@@ -66,9 +64,7 @@ async def Users(u, config, conn):
user = User(u) user = User(u)
output = format.User(config.Format, user) output = format.User(config.Format, user)
if config.hostname: if config.Database:
dbmysql.user(conn, config.Username, config.Followers, user)
elif config.Database:
db.user(conn, config.Username, config.Followers, user) db.user(conn, config.Username, config.Followers, user)
if config.Elasticsearch: if config.Elasticsearch:
...@@ -83,9 +79,7 @@ async def Users(u, config, conn): ...@@ -83,9 +79,7 @@ async def Users(u, config, conn):
_output(user, output, config) _output(user, output, config)
async def Username(username, config, conn): async def Username(username, config, conn):
if config.hostname: if config.Database:
dbmysql.follow(conn, config.Username, config.Followers, username)
elif config.Database:
db.follow(conn, config.Username, config.Followers, username) db.follow(conn, config.Username, config.Followers, username)
if config.Elasticsearch: if config.Elasticsearch:
......
twint/profile.py
View file @ d7681f0b
from . import db, get, feed, output, verbose, dbmysql from . import db, get, feed, output, verbose
class Profile: class Profile:
def __init__(self, config): def __init__(self, config):
...@@ -7,15 +7,12 @@ class Profile: ...@@ -7,15 +7,12 @@ class Profile:
self.count = 0 self.count = 0
self.config = config self.config = config
if config.hostname: self.conn = db.Conn(config.Database)
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
self.config.Profile = True self.config.Profile = True
verbose.Elastic(config) verbose.Elastic(config)
async def Feed(self): async def Feed(self):
response = await get.RequestUrl(self.config, self.init) response = await get.RequestUrl(self.config, self.init)
self.feed = [] self.feed = []
...@@ -45,7 +42,7 @@ class Profile: ...@@ -45,7 +42,7 @@ class Profile:
await self.tweets() await self.tweets()
else: else:
break break
if get.Limit(self.config.Limit, self.count): if get.Limit(self.config.Limit, self.count):
break break
......
twint/search.py
View file @ d7681f0b
from . import datelock, db, get, feed, output, verbose, dbmysql from . import datelock, db, get, feed, output, verbose
from datetime import timedelta from datetime import timedelta
class Search: class Search:
...@@ -7,10 +7,7 @@ class Search: ...@@ -7,10 +7,7 @@ class Search:
self.feed = [-1] self.feed = [-1]
self.count = 0 self.count = 0
self.config = config self.config = config
if config.hostname: self.conn = db.Conn(config.Database)
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
self.d = datelock.Set(self.config.Until, self.config.Since) self.d = datelock.Set(self.config.Until, self.config.Since)
self.config.TwitterSearch = True self.config.TwitterSearch = True
...@@ -53,7 +50,7 @@ class Search: ...@@ -53,7 +50,7 @@ class Search:
else: else:
self.d._until = self.d._until - _days self.d._until = self.d._until - _days
self.feed = [-1] self.feed = [-1]
if get.Limit(self.config.Limit, self.count): if get.Limit(self.config.Limit, self.count):
self.d._until = self.d._until - _days self.d._until = self.d._until - _days
self.feed = [-1] self.feed = [-1]
...@@ -63,8 +60,8 @@ class Search: ...@@ -63,8 +60,8 @@ class Search:
await self.tweets() await self.tweets()
else: else:
break break
if get.Limit(self.config.Limit, self.count): if get.Limit(self.config.Limit, self.count):
break break
verbose.Count(self.config, self.count) verbose.Count(self.config, self.count)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment