Skip to content

T
Twint
Commit bcf3d9b7 authored by Francesco Poldi's avatar Francesco Poldi
Browse files
Options

Fixed index creation for Elasticsearch 7.0.0 

#397
parent f6cc33c7
Hide whitespace changes
Inline Side-by-side
Showing with 55 additions and 64 deletions
twint/storage/elasticsearch.py
View file @ bcf3d9b7
...@@ -55,38 +55,36 @@ def createIndex(config, instance, **scope): ...@@ -55,38 +55,36 @@ def createIndex(config, instance, **scope):
if scope.get("scope") == "tweet": if scope.get("scope") == "tweet":
tweets_body = { tweets_body = {
"mappings": { "mappings": {
config.Index_type: { "properties": {
"properties": { "id": {"type": "long"},
"id": {"type": "long"}, "conversation_id": {"type": "long"},
"conversation_id": {"type": "long"}, "created_at": {"type": "long"},
"created_at": {"type": "long"}, "date": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
"date": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"}, "timezone": {"type": "keyword"},
"timezone": {"type": "keyword"}, "place": {"type": "keyword"},
"place": {"type": "keyword"}, "location": {"type": "keyword"},
"location": {"type": "keyword"}, "tweet": {"type": "text"},
"tweet": {"type": "text"}, "hashtags": {"type": "keyword"},
"hashtags": {"type": "keyword"}, "user_id": {"type": "long"},
"user_id": {"type": "long"}, "user_id_str": {"type": "keyword"},
"user_id_str": {"type": "keyword"}, "username": {"type": "keyword"},
"username": {"type": "keyword"}, "name": {"type": "text"},
"name": {"type": "text"}, "profile_image_url": {"type": "text"},
"profile_image_url": {"type": "text"}, "day": {"type": "integer"},
"day": {"type": "integer"}, "hour": {"type": "integer"},
"hour": {"type": "integer"}, "link": {"type": "text"},
"link": {"type": "text"}, "retweet": {"type": "text"},
"retweet": {"type": "text"}, "essid": {"type": "keyword"},
"essid": {"type": "keyword"}, "nlikes": {"type": "integer"},
"nlikes": {"type": "integer"}, "nreplies": {"type": "integer"},
"nreplies": {"type": "integer"}, "nretweets": {"type": "integer"},
"nretweets": {"type": "integer"}, "quote_url": {"type": "text"},
"quote_url": {"type": "text"}, "video": {"type":"integer"},
"video": {"type":"integer"}, "search": {"type": "text"},
"search": {"type": "text"}, "near": {"type": "text"},
"near": {"type": "text"}, "geo_near": {"type": "geo_point"},
"geo_near": {"type": "geo_point"}, "geo_tweet": {"type": "geo_point"},
"geo_tweet": {"type": "geo_point"}, "photos": {"type": "text"}
"photos": {"type": "text"}
}
} }
}, },
"settings": { "settings": {
...@@ -99,12 +97,10 @@ def createIndex(config, instance, **scope): ...@@ -99,12 +97,10 @@ def createIndex(config, instance, **scope):
elif scope.get("scope") == "follow": elif scope.get("scope") == "follow":
follow_body = { follow_body = {
"mappings": { "mappings": {
config.Index_type: { "properties": {
"properties": { "user": {"type": "keyword"},
"user": {"type": "keyword"}, "follow": {"type": "keyword"},
"follow": {"type": "keyword"}, "essid": {"type": "keyword"}
"essid": {"type": "keyword"}
}
} }
}, },
"settings": { "settings": {
...@@ -117,29 +113,27 @@ def createIndex(config, instance, **scope): ...@@ -117,29 +113,27 @@ def createIndex(config, instance, **scope):
elif scope.get("scope") == "user": elif scope.get("scope") == "user":
user_body = { user_body = {
"mappings": { "mappings": {
config.Index_type: { "properties": {
"properties": { "id": {"type": "keyword"},
"id": {"type": "keyword"}, "name": {"type": "keyword"},
"name": {"type": "keyword"}, "username": {"type": "keyword"},
"username": {"type": "keyword"}, "bio": {"type": "text"},
"bio": {"type": "text"}, "location": {"type": "keyword"},
"location": {"type": "keyword"}, "url": {"type": "text"},
"url": {"type": "text"}, "join_datetime": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
"join_datetime": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"}, "join_date": {"type": "date", "format": "yyyy-MM-dd"},
"join_date": {"type": "date", "format": "yyyy-MM-dd"}, "join_time": {"type": "date", "format": "HH:mm:ss"},
"join_time": {"type": "date", "format": "HH:mm:ss"}, "tweets": {"type": "integer"},
"tweets": {"type": "integer"}, "following": {"type": "integer"},
"following": {"type": "integer"}, "followers": {"type": "integer"},
"followers": {"type": "integer"}, "likes": {"type": "integer"},
"likes": {"type": "integer"}, "media": {"type": "integer"},
"media": {"type": "integer"}, "private": {"type": "integer"},
"private": {"type": "integer"}, "verified": {"type": "integer"},
"verified": {"type": "integer"}, "avatar": {"type": "text"},
"avatar": {"type": "text"}, "background_image": {"type": "text"},
"background_image": {"type": "text"}, "session": {"type": "keyword"},
"session": {"type": "keyword"}, "geo_user": {"type": "geo_point"}
"geo_user": {"type": "geo_point"}
}
} }
}, },
"settings": { "settings": {
...@@ -196,7 +190,6 @@ def Tweet(Tweet, config): ...@@ -196,7 +190,6 @@ def Tweet(Tweet, config):
j_data = { j_data = {
"_index": config.Index_tweets, "_index": config.Index_tweets,
"_type": config.Index_type,
"_id": str(Tweet.id) + "_raw_" + config.Essid, "_id": str(Tweet.id) + "_raw_" + config.Essid,
"_source": { "_source": {
"id": str(Tweet.id), "id": str(Tweet.id),
...@@ -273,7 +266,6 @@ def Follow(user, config): ...@@ -273,7 +266,6 @@ def Follow(user, config):
_follow = config.Username _follow = config.Username
j_data = { j_data = {
"_index": config.Index_follow, "_index": config.Index_follow,
"_type": config.Index_type,
"_id": _user + "_" + _follow + "_" + config.Essid, "_id": _user + "_" + _follow + "_" + config.Essid,
"_source": { "_source": {
"user": _user, "user": _user,
...@@ -297,7 +289,6 @@ def UserProfile(user, config): ...@@ -297,7 +289,6 @@ def UserProfile(user, config):
j_data = { j_data = {
"_index": config.Index_users, "_index": config.Index_users,
"_type": config.Index_type,
"_id": user.id + "_" + user.join_date + "_" + user.join_time + "_" + config.Essid, "_id": user.id + "_" + user.join_date + "_" + user.join_time + "_" + config.Essid,
"_source": { "_source": {
"id": user.id, "id": user.id,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment