Commit 53183006 authored by Cody Zacharias's avatar Cody Zacharias Committed by GitHub

Fix break from copyright Tweets

parent da5b7f70
...@@ -69,18 +69,7 @@ async def getFeed(init): ...@@ -69,18 +69,7 @@ async def getFeed(init):
return feed, init return feed, init
async def getPic(url): async def outTweet(tweet):
async with aiohttp.ClientSession() as session:
r = await fetch(session, url)
soup = BeautifulSoup(r, "html.parser")
picture = soup.find("div", "AdaptiveMedia-photoContainer js-adaptive-photo ")
if picture is not None:
return picture["data-image-url"].replace(" ", "")
async def getTweets(init):
tweets, init = await getFeed(init)
count = 0
for tweet in tweets:
tweetid = tweet["data-item-id"] tweetid = tweet["data-item-id"]
datestamp = tweet.find("a", "tweet-timestamp")["title"].rpartition(" - ")[-1] datestamp = tweet.find("a", "tweet-timestamp")["title"].rpartition(" - ")[-1]
d = datetime.datetime.strptime(datestamp, "%d %b %Y") d = datetime.datetime.strptime(datestamp, "%d %b %Y")
...@@ -90,7 +79,7 @@ async def getTweets(init): ...@@ -90,7 +79,7 @@ async def getTweets(init):
time = t.strftime("%H:%M:%S") time = t.strftime("%H:%M:%S")
username = tweet.find("span", "username").text.replace("@", "") username = tweet.find("span", "username").text.replace("@", "")
timezone = strftime("%Z", gmtime()) timezone = strftime("%Z", gmtime())
text = tweet.find("p", "tweet-text").text.replace("\n", " ").replace("http"," http").replace("pic.twitter"," pic.twitter") text = tweet.find("p", "tweet-text").text.replace("\n", "").replace("http", " http").replace("pic.twitter", " pic.twitter")
hashtags = ",".join(re.findall(r'(?i)\#\w+', text, flags=re.UNICODE)) hashtags = ",".join(re.findall(r'(?i)\#\w+', text, flags=re.UNICODE))
replies = tweet.find("span", "ProfileTweet-action--reply u-hiddenVisually").find("span")["data-tweet-stat-count"] replies = tweet.find("span", "ProfileTweet-action--reply u-hiddenVisually").find("span")["data-tweet-stat-count"]
retweets = tweet.find("span", "ProfileTweet-action--retweet u-hiddenVisually").find("span")["data-tweet-stat-count"] retweets = tweet.find("span", "ProfileTweet-action--retweet u-hiddenVisually").find("span")["data-tweet-stat-count"]
...@@ -117,15 +106,23 @@ async def getTweets(init): ...@@ -117,15 +106,23 @@ async def getTweets(init):
if arg.o != None: if arg.o != None:
if arg.csv: if arg.csv:
dat = [tweetid, date, time, timezone, username, text, hashtags, replies, retweets, likes] dat = [tweetid, date, time, timezone, username, text, replies, retweets, likes, hashtags]
with open(arg.o, "a", newline='') as csv_file: with open(arg.o, "a", newline='') as csv_file:
writer = csv.writer(csv_file, delimiter="|") writer = csv.writer(csv_file, delimiter="|")
writer.writerow(dat) writer.writerow(dat)
else: else:
print(output, file=open(arg.o, "a")) print(output, file=open(arg.o, "a"))
count += 1 return output
print(output)
async def getTweets(init):
tweets, init = await getFeed(init)
count = 0
for tweet in tweets:
copyright = tweet.find("div","StreamItemContent--withheld")
if copyright is None:
count +=1
print(await outTweet(tweet))
return tweets, init, count return tweets, init, count
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment