Skip to content

T
Twint
Commit 06b00406 authored by Francesco Poldi's avatar Francesco Poldi
Browse files
Options

Added fields 

it seams that exporting dashboard and visualization does not work, so people will have to write new ones. Pretty easy, I'll add a short tutorial.
parent de2c427f
Hide whitespace changes
Inline Side-by-side
Showing with 27 additions and 163 deletions
dashboard.json deleted 100644 → 0
View file @ de2c427f
[
{
"_id": "b82b0210-2b04-11e8-b48f-dbf856df4d4c",
"_type": "dashboard",
"_source": {
"title": "[tweep] Overview",
"hits": 0,
"description": "",
"panelsJSON": "[\n {\n \"panelIndex\": \"1\",\n \"gridData\": {\n \"x\": 6,\n \"y\": 5,\n \"w\": 6,\n \"h\": 3,\n \"i\": \"1\"\n },\n \"id\": \"fcda84d0-2afa-11e8-b48f-dbf856df4d4c\",\n \"type\": \"visualization\",\n \"version\": \"6.2.2\"\n },\n {\n \"panelIndex\": \"2\",\n \"gridData\": {\n \"x\": 0,\n \"y\": 0,\n \"w\": 12,\n \"h\": 2,\n \"i\": \"2\"\n },\n \"id\": \"81446af0-2afc-11e8-b48f-dbf856df4d4c\",\n \"type\": \"visualization\",\n \"version\": \"6.2.2\"\n },\n {\n \"panelIndex\": \"3\",\n \"gridData\": {\n \"x\": 0,\n \"y\": 5,\n \"w\": 6,\n \"h\": 3,\n \"i\": \"3\"\n },\n \"id\": \"6683d0c0-2afc-11e8-b48f-dbf856df4d4c\",\n \"type\": \"visualization\",\n \"version\": \"6.2.2\"\n },\n {\n \"panelIndex\": \"4\",\n \"gridData\": {\n \"x\": 0,\n \"y\": 2,\n \"w\": 12,\n \"h\": 3,\n \"i\": \"4\"\n },\n \"id\": \"4f4f4380-2afc-11e8-b48f-dbf856df4d4c\",\n \"type\": \"visualization\",\n \"version\": \"6.2.2\"\n },\n {\n \"panelIndex\": \"5\",\n \"gridData\": {\n \"x\": 0,\n \"y\": 8,\n \"w\": 4,\n \"h\": 3,\n \"i\": \"5\"\n },\n \"version\": \"6.2.2\",\n \"type\": \"visualization\",\n \"id\": \"38bacd30-2b04-11e8-b48f-dbf856df4d4c\"\n },\n {\n \"panelIndex\": \"6\",\n \"gridData\": {\n \"x\": 8,\n \"y\": 8,\n \"w\": 4,\n \"h\": 3,\n \"i\": \"6\"\n },\n \"version\": \"6.2.2\",\n \"type\": \"visualization\",\n \"id\": \"45a0f4c0-2b04-11e8-b48f-dbf856df4d4c\"\n },\n {\n \"panelIndex\": \"7\",\n \"gridData\": {\n \"x\": 4,\n \"y\": 8,\n \"w\": 4,\n \"h\": 3,\n \"i\": \"7\"\n },\n \"version\": \"6.2.2\",\n \"type\": \"visualization\",\n \"id\": \"50ac41d0-2b04-11e8-b48f-dbf856df4d4c\"\n }\n]",
"optionsJSON": "{\n \"darkTheme\": false,\n \"hidePanelTitles\": false,\n \"useMargins\": true\n}",
"version": 1,
"timeRestore": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\n \"query\": {\n \"language\": \"lucene\",\n \"query\": \"\"\n },\n \"filter\": [],\n \"highlightAll\": true,\n \"version\": true\n}"
}
}
}
]
\ No newline at end of file
index.json
View file @ 06b00406
...@@ -2,7 +2,7 @@ PUT tweep ...@@ -2,7 +2,7 @@ PUT tweep
{ {
"mappings" : { "mappings" : {
"items": { "items": {
"properties": { "properties": {
"tweetid": {"type": "long"}, "tweetid": {"type": "long"},
"datestamp": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"}, "datestamp": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
"timezone": {"type": "text"}, "timezone": {"type": "text"},
...@@ -10,7 +10,9 @@ PUT tweep ...@@ -10,7 +10,9 @@ PUT tweep
"replies": {"type": "integer"}, "replies": {"type": "integer"},
"retweets": {"type": "integer"}, "retweets": {"type": "integer"},
"likes": {"type": "integer"}, "likes": {"type": "integer"},
"username": {"type": "keyword"} "username": {"type": "keyword"},
"day": {"type": "keyword"},
"hour": {"type": "keyword"}
} }
} }
} }
......
tweep.py
View file @ 06b00406
...@@ -204,6 +204,26 @@ async def outTweet(tweet): ...@@ -204,6 +204,26 @@ async def outTweet(tweet):
except sqlite3.IntegrityError: # this happens if the tweet is already in the db except sqlite3.IntegrityError: # this happens if the tweet is already in the db
return "" return ""
day = d.strftime("%A")
if day == "Monday":
_day = 1
elif day == "Tuesday":
_day = 2
elif day == "Wednesday":
_day = 3
elif day == "Thursday":
_day = 4
elif day == "Friday":
_day = 5
elif day == "Saturday":
_day = 6
elif day == "Sunday":
_day = 7
else:
print("[x] Something is going wrong!")
sys.exit(1)
if arg.elasticsearch: if arg.elasticsearch:
jObject = { jObject = {
"tweetid": tweetid, "tweetid": tweetid,
...@@ -214,7 +234,9 @@ async def outTweet(tweet): ...@@ -214,7 +234,9 @@ async def outTweet(tweet):
"replies": replies, "replies": replies,
"retweets": retweets, "retweets": retweets,
"likes": likes, "likes": likes,
"username": username "username": username,
"day": _day,
"hour": time.split(":")[0]
} }
es = Elasticsearch(arg.elasticsearch) es = Elasticsearch(arg.elasticsearch)
......
visualization.json deleted 100644 → 0
View file @ de2c427f
[
{
"_id": "4f4f4380-2afc-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Users activity over time [tweep]",
"visState": "{\"title\":\"Users activity over time [tweep]\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"datestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "81446af0-2afc-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top 5 users [tweep]",
"visState": "{\"title\":\"Top 5 users [tweep]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "df296b50-2b03-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top 5 users and top 10 reteewts [tweep]",
"visState": "{\"title\":\"Top 5 users and top 10 reteewts [tweep]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":true,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Users\",\"row\":true}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"retweets\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Likes\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "d31ab800-2b03-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top 5 users and top 10 replies [tweep]",
"visState": "{\"title\":\"Top 5 users and top 10 replies [tweep]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":true,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Users\",\"row\":true}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"replies\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Likes\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "45a0f4c0-2b04-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top user and top 10 replies [tweep]",
"visState": "{\"title\":\"Top user and top 10 replies [tweep]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":true,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Users\",\"row\":true}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"replies\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Likes\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "b85e6390-2b03-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top 5 users and top 10 likes [tweep]",
"visState": "{\"title\":\"Top 5 users and top 10 likes [tweep]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":true,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Users\",\"row\":true}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"likes\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Likes\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "38bacd30-2b04-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top user and top 10 likes [tweep]",
"visState": "{\"title\":\"Top user and top 10 likes [tweep]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":true,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Users\",\"row\":true}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"likes\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Likes\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "fcda84d0-2afa-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Most Active Users [tweep]",
"visState": "{\"title\":\"Most Active Users [tweep]\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}"
}
}
},
{
"_id": "6683d0c0-2afc-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Total tweets [tweep]",
"visState": "{\"title\":\"Total tweets [tweep]\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
},
{
"_id": "50ac41d0-2b04-11e8-b48f-dbf856df4d4c",
"_type": "visualization",
"_source": {
"title": "Top user and top 10 retweets [tweep]",
"visState": "{\"title\":\"Top user and top 10 retweets [tweep]\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":true,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"username\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Users\",\"row\":true}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"retweets\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Likes\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"7bfce7c0-26dd-11e8-9d16-855c7cbf79a1\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
}
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment