Fixed geo for user and tweet, updated tweet.place

elasticsearch/index-tweets.json

@@ -37,6 +37,7 @@ PUT twinttweets
           "quote_url": {"type": "text"},
           "search": {"type": "text"},
           "near":  {"type": "text"},
+          "geo_near": {"type": "geo_point"},
           "geo_tweet": {"type": "geo_point"}
       }
     }

twint/get.py

@@ -90,7 +90,7 @@ def ForceNewTorIdentity(config):
     except Exception as e:
         sys.stderr.write('Error connecting to Tor control port: {}\n'.format(repr(e)))
         sys.stderr.write('If you want to rotate Tor ports automatically - enable Tor control port\n')
-        
+
 async def Request(url, connector=None, params=[], headers=[]):
     #loggin.info("[<] " + str(datetime.now()) + ':: get+Request')
     if connector:

twint/run.py

@@ -68,7 +68,7 @@ class Twint:
                         continue
                 else:
                     print(str(e))
-                    break                
+                    break
             except Exception as e:
                 # Sometimes Twitter says there is no data. But it's a lie.
                 consecutive_errors_count += 1

twint/storage/elasticsearch.py

@@ -17,7 +17,10 @@ class RecycleObject(object):
 
 def getLocation(place):
     location = geolocator.geocode(place)
-    return {"lat": location.latitude, "lon": location.longitude}
+    if location:
+        return {"lat": location.latitude, "lon": location.longitude}
+    else:
+        return {}
 
 def handleIndexResponse(response):
     try:
@@ -76,6 +79,7 @@ def createIndex(config, instance, **scope):
                             "quote_url": {"type": "text"},
                             "search": {"type": "text"},
                             "near": {"type": "text"},
+                            "geo_near": {"type": "geopoint"},
                             "geo_tweet": {"type": "geopoint"}
                             }
                         }
@@ -226,7 +230,9 @@ def Tweet(Tweet, config):
                 }
             }
     if config.Near:
-        j_data["_source"].update({"geo_tweet": getLocation(config.Near)})
+        j_data["_source"].update({"geo_near": getLocation(config.Near)})
+    if Tweet.place:
+        j_data["_source"].update({"geo_tweet": getLocation(Tweet.place)})
     actions.append(j_data)
 
     es = Elasticsearch(config.Elasticsearch)
@@ -290,7 +296,9 @@ def UserProfile(user, config):
                 }
             }
     if config.Location:
-        j_data["_source"].update({"geo_user": getLocation(config.Location)})
+        _loc = getLocation(user.location)
+        if _loc:
+            j_data["_source"].update({"geo_user": _loc})
     actions.append(j_data)
 
     es = Elasticsearch(config.Elasticsearch)

twint/tweet.py

@@ -129,7 +129,7 @@ def Tweet(tw, location, config):
     t.username = tw["data-screen-name"]
     t.name = tw["data-name"]
     t.profile_image_url = tw.find("img", "js-action-profile-avatar").get('src').replace("_bigger","")
-    t.place = tw.find("a","js-geo-pivot-link").text.strip() if tw.find("a","js-geo-pivot-link") else None
+    t.place = tw.find("a",{'class':"u-textUserColor js-nav js-geo-pivot-link"}).text.strip() if tw.find("a","js-geo-pivot-link") else None
     t.timezone = strftime("%Z", localtime())
     for img in tw.findAll("img", "Emoji Emoji--forText"):
         img.replaceWith(img["alt"])
@@ -155,4 +155,4 @@ def Tweet(tw, location, config):
     t.in_reply_to_status_id_str = ""
     t.in_reply_to_user_id = 0
     t.in_reply_to_user_id_str = ""
-    return t
\ No newline at end of file
+    return t