Skip to content

T
Twint
Commit 92741b19 authored by Cody Zacharias's avatar Cody Zacharias Committed by GitHub
Browse files
Options

Merge to Dev (#157) 

* Delete twint_db_sqlite3

* Delete twint_db_mysql.sql

* Delete dbmysql.py

* Update output.py

* Update favorites.py

* Update follow.py

* Update profile.py

* Update search.py

* Removed dbmysql related stuff

* Deleted import

* Update

* Update

* Update favorites.py
parent 88a22f51
Show whitespace changes
Inline Side-by-side
Showing with 20 additions and 297 deletions
db-schema/twint_db_mysql.sql deleted 100644 → 0
View file @ 88a22f51
CREATE DATABASE IF NOT EXISTS `twitterdata_v9` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
USE `twitterdata_v9`;
-- MySQL dump 10.13 Distrib 5.7.22, for Linux (x86_64)
--
-- Host: localhost Database: twitterdata_v9
-- ------------------------------------------------------
-- Server version 5.7.22-0ubuntu0.16.04.1
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `followers`
--
DROP TABLE IF EXISTS `followers`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `followers` (
`id` bigint(30) NOT NULL,
`name` mediumtext,
`username` text NOT NULL,
`bio` longtext,
`location` tinytext,
`url` longtext,
`join_date` tinytext NOT NULL,
`join_time` tinytext NOT NULL,
`tweets` int(11) DEFAULT NULL,
`following` int(11) DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`likes` int(11) DEFAULT NULL,
`media` int(11) DEFAULT NULL,
`private` tinytext NOT NULL,
`verified` tinytext NOT NULL,
`avatar` longtext NOT NULL,
`date_update` datetime NOT NULL,
`follower` text NOT NULL,
PRIMARY KEY (`follower`(255),`username`(255),`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `followers_names`
--
DROP TABLE IF EXISTS `followers_names`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `followers_names` (
`user` text NOT NULL,
`date_update` datetime NOT NULL,
`follower` text NOT NULL,
PRIMARY KEY (`user`(255),`follower`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `following`
--
DROP TABLE IF EXISTS `following`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `following` (
`id` bigint(30) NOT NULL,
`name` mediumtext,
`username` text NOT NULL,
`bio` longtext,
`location` tinytext,
`url` longtext,
`join_date` tinytext NOT NULL,
`join_time` tinytext NOT NULL,
`tweets` int(11) DEFAULT NULL,
`following` int(11) DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`likes` int(11) DEFAULT NULL,
`media` int(11) DEFAULT NULL,
`private` tinytext NOT NULL,
`verified` tinytext NOT NULL,
`avatar` longtext NOT NULL,
`date_update` datetime NOT NULL,
`follows` text NOT NULL,
PRIMARY KEY (`id`,`username`(255),`follows`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `following_names`
--
DROP TABLE IF EXISTS `following_names`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `following_names` (
`user` text NOT NULL,
`date_update` datetime NOT NULL,
`follows` text NOT NULL,
PRIMARY KEY (`user`(255),`follows`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `tweets`
--
DROP TABLE IF EXISTS `tweets`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `tweets` (
`id` bigint(30) NOT NULL,
`user_id` bigint(30) DEFAULT NULL,
`date` date NOT NULL,
`time` time NOT NULL,
`timezone` tinytext NOT NULL,
`location` tinytext NOT NULL,
`user` text NOT NULL,
`tweet` longtext NOT NULL,
`replies` int(11) DEFAULT NULL,
`likes` int(11) DEFAULT NULL,
`retweets` int(11) DEFAULT NULL,
`hashtags` longtext,
`link` longtext,
`retweet` int(1) DEFAULT NULL,
`user_rt` text,
`mentions` longtext,
`date_update` datetime NOT NULL,
`search_name` mediumtext NOT NULL COMMENT 'user can use this field to know from which search the info comes. max 255 chars. if the user do not especify, it must be set to "-" ',
PRIMARY KEY (`id`,`search_name`(255))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2018-06-03 14:52:08
twint/dbmysql.py deleted 100644 → 0
View file @ 88a22f51
from datetime import datetime
import MySQLdb
import sys
def Conn(hostname,Database,db_user,db_pwd):
if Database:
print("[+] Inserting into Database: " + str(Database))
conn = init(hostname,Database,db_user,db_pwd)
if isinstance(conn, str):
print(str)
sys.exit(1)
else:
conn = ""
return conn
def init(hostname,Database,db_user,db_pwd):
try:
conn = MySQLdb.connect(host=hostname, # your host, usually localhost
user=db_user, # your username
passwd=db_pwd, # your password
db=Database,# name of the data base
charset='utf8mb4',
use_unicode=True)
cursor = conn.cursor()
#here would be the code for creating the tables if them don't exist
return conn
except Exception as e:
return str(e)
def fTable(Followers):
if Followers:
table = "followers_names"
else:
table = "following_names"
return table
def uTable(Followers):
if Followers:
table = "followers"
else:
table = "following"
return table
def follow(conn, Username, Followers, User):
try:
date_time = str(datetime.now())
cursor = conn.cursor()
entry = (User, date_time, Username,)
query = 'INSERT INTO {} VALUES(%s,%s,%s)'.format(fTable(Followers))
cursor.execute(query, entry)
conn.commit()
except MySQLdb.IntegrityError:
pass
def user(conn, Username, Followers, User):
try:
date_time = str(datetime.now())
cursor = conn.cursor()
entry = (User.id,
User.name,
User.username,
User.bio,
User.location,
User.url,
User.join_date,
User.join_time,
User.tweets,
User.following,
User.followers,
User.likes,
User.media_count,
User.is_private,
User.is_verified,
User.avatar,
date_time,
Username,)
query = 'INSERT INTO {} VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(uTable(Followers))
cursor.execute(query, entry)
conn.commit()
except MySQLdb.IntegrityError:
pass
def tweets(conn, Tweet, config):
try:
date_time = str(datetime.now())
cursor = conn.cursor()
entry = (Tweet.id,
Tweet.user_id,
Tweet.datestamp,
Tweet.timestamp,
Tweet.timezone,
Tweet.location,
Tweet.username,
Tweet.tweet,
Tweet.replies,
Tweet.likes,
Tweet.retweets,
",".join(Tweet.hashtags),
Tweet.link,
Tweet.retweet,
Tweet.user_rt,
",".join(Tweet.mentions),
date_time,
config.search_name,)
cursor.execute('INSERT INTO tweets VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', entry)
conn.commit()
except MySQLdb.IntegrityError:
pass
twint/favorites.py
View file @ 92741b19
from . import feed, get, db, output, verbose, dbmysql
from . import feed, get, db, output, verbose
class Favorites:
def __init__(self, config):
......@@ -6,9 +6,6 @@ class Favorites:
self.feed = [-1]
self.count = 0
self.config = config
if config.hostname:
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
self.config.Favorites = True
......
twint/follow.py
View file @ 92741b19
from . import feed, get, db, output, verbose, dbmysql
from . import feed, get, db, output, verbose
class Follow:
def __init__(self, config):
......@@ -6,9 +6,6 @@ class Follow:
self.feed = [-1]
self.count = 0
self.config = config
if config.hostname:
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
verbose.Elastic(config)
......
twint/output.py
View file @ 92741b19
from datetime import datetime
from . import db, elasticsearch, format, write, Pandas, dbmysql
from . import db, elasticsearch, format, write, Pandas
from .tweet import Tweet
from .user import User
......@@ -54,9 +54,7 @@ async def Tweets(tw, location, config, conn):
if datecheck(tweet.datestamp, config):
output = format.Tweet(config, tweet)
if config.hostname:
dbmysql.tweets(conn, tweet, config)
elif config.Database:
if config.Database:
db.tweets(conn, tweet, config)
if config.Elasticsearch:
elasticsearch.Tweet(tweet, config)
......@@ -66,9 +64,7 @@ async def Users(u, config, conn):
user = User(u)
output = format.User(config.Format, user)
if config.hostname:
dbmysql.user(conn, config.Username, config.Followers, user)
elif config.Database:
if config.Database:
db.user(conn, config.Username, config.Followers, user)
if config.Elasticsearch:
......@@ -83,9 +79,7 @@ async def Users(u, config, conn):
_output(user, output, config)
async def Username(username, config, conn):
if config.hostname:
dbmysql.follow(conn, config.Username, config.Followers, username)
elif config.Database:
if config.Database:
db.follow(conn, config.Username, config.Followers, username)
if config.Elasticsearch:
......
twint/profile.py
View file @ 92741b19
from . import db, get, feed, output, verbose, dbmysql
from . import db, get, feed, output, verbose
class Profile:
def __init__(self, config):
......@@ -7,9 +7,6 @@ class Profile:
self.count = 0
self.config = config
if config.hostname:
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
self.config.Profile = True
......
twint/search.py
View file @ 92741b19
from . import datelock, db, get, feed, output, verbose, dbmysql
from . import datelock, db, get, feed, output, verbose
from datetime import timedelta
class Search:
......@@ -7,9 +7,6 @@ class Search:
self.feed = [-1]
self.count = 0
self.config = config
if config.hostname:
self.conn = dbmysql.Conn(config.hostname, config.Database, config.DB_user, config.DB_pwd)
else:
self.conn = db.Conn(config.Database)
self.d = datelock.Set(self.config.Until, self.config.Since)
self.config.TwitterSearch = True
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment