Skip to content

T
Twint
Commit bcf3d9b7 authored by Francesco Poldi's avatar Francesco Poldi
Browse files
Options

Fixed index creation for Elasticsearch 7.0.0 

#397
parent f6cc33c7
Show whitespace changes
Inline Side-by-side
Showing with 55 additions and 64 deletions
twint/storage/elasticsearch.py
View file @ bcf3d9b7
...@@ -55,7 +55,6 @@ def createIndex(config, instance, **scope): ...@@ -55,7 +55,6 @@ def createIndex(config, instance, **scope):
if scope.get("scope") == "tweet": if scope.get("scope") == "tweet":
tweets_body = { tweets_body = {
"mappings": { "mappings": {
config.Index_type: {
"properties": { "properties": {
"id": {"type": "long"}, "id": {"type": "long"},
"conversation_id": {"type": "long"}, "conversation_id": {"type": "long"},
...@@ -87,7 +86,6 @@ def createIndex(config, instance, **scope): ...@@ -87,7 +86,6 @@ def createIndex(config, instance, **scope):
"geo_tweet": {"type": "geo_point"}, "geo_tweet": {"type": "geo_point"},
"photos": {"type": "text"} "photos": {"type": "text"}
} }
}
}, },
"settings": { "settings": {
"number_of_shards": 1 "number_of_shards": 1
...@@ -99,13 +97,11 @@ def createIndex(config, instance, **scope): ...@@ -99,13 +97,11 @@ def createIndex(config, instance, **scope):
elif scope.get("scope") == "follow": elif scope.get("scope") == "follow":
follow_body = { follow_body = {
"mappings": { "mappings": {
config.Index_type: {
"properties": { "properties": {
"user": {"type": "keyword"}, "user": {"type": "keyword"},
"follow": {"type": "keyword"}, "follow": {"type": "keyword"},
"essid": {"type": "keyword"} "essid": {"type": "keyword"}
} }
}
}, },
"settings": { "settings": {
"number_of_shards": 1 "number_of_shards": 1
...@@ -117,7 +113,6 @@ def createIndex(config, instance, **scope): ...@@ -117,7 +113,6 @@ def createIndex(config, instance, **scope):
elif scope.get("scope") == "user": elif scope.get("scope") == "user":
user_body = { user_body = {
"mappings": { "mappings": {
config.Index_type: {
"properties": { "properties": {
"id": {"type": "keyword"}, "id": {"type": "keyword"},
"name": {"type": "keyword"}, "name": {"type": "keyword"},
...@@ -140,7 +135,6 @@ def createIndex(config, instance, **scope): ...@@ -140,7 +135,6 @@ def createIndex(config, instance, **scope):
"session": {"type": "keyword"}, "session": {"type": "keyword"},
"geo_user": {"type": "geo_point"} "geo_user": {"type": "geo_point"}
} }
}
}, },
"settings": { "settings": {
"number_of_shards": 1 "number_of_shards": 1
...@@ -196,7 +190,6 @@ def Tweet(Tweet, config): ...@@ -196,7 +190,6 @@ def Tweet(Tweet, config):
j_data = { j_data = {
"_index": config.Index_tweets, "_index": config.Index_tweets,
"_type": config.Index_type,
"_id": str(Tweet.id) + "_raw_" + config.Essid, "_id": str(Tweet.id) + "_raw_" + config.Essid,
"_source": { "_source": {
"id": str(Tweet.id), "id": str(Tweet.id),
...@@ -273,7 +266,6 @@ def Follow(user, config): ...@@ -273,7 +266,6 @@ def Follow(user, config):
_follow = config.Username _follow = config.Username
j_data = { j_data = {
"_index": config.Index_follow, "_index": config.Index_follow,
"_type": config.Index_type,
"_id": _user + "_" + _follow + "_" + config.Essid, "_id": _user + "_" + _follow + "_" + config.Essid,
"_source": { "_source": {
"user": _user, "user": _user,
...@@ -297,7 +289,6 @@ def UserProfile(user, config): ...@@ -297,7 +289,6 @@ def UserProfile(user, config):
j_data = { j_data = {
"_index": config.Index_users, "_index": config.Index_users,
"_type": config.Index_type,
"_id": user.id + "_" + user.join_date + "_" + user.join_time + "_" + config.Essid, "_id": user.id + "_" + user.join_date + "_" + user.join_time + "_" + config.Essid,
"_source": { "_source": {
"id": user.id, "id": user.id,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment