In [42]:
%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
# Make the graphs a bit prettier, and bigger
pd.set_option('display.mpl_style', 'default')
# This is necessary to show lots of columns in pandas 0.12.
# Not necessary in pandas 0.13.
pd.set_option('display.width', 5000)
pd.set_option('display.max_columns', 60)
plt.rcParams['figure.figsize'] = (15, 5)
In [43]:
dbase = pd.read_csv('/home/al/projects/twitter/jays.csv')
In [44]:
dbase.set_index('created_at', drop=False, inplace=True)
In [45]:
#dbase.index
dbase[:3]
Out[45]:
In [46]:
#dbase.head()
In [47]:
dbase.describe()
Out[47]:
In [48]:
total_tweets = pd.Series.count(dbase['created_at'])
total_tweets
Out[48]:
In [49]:
dbase1m = pd.Series.value_counts(dbase['created_at'], normalize=False, sort=True, ascending=False, bins=None)
dbase1m[:10].plot(kind='bar')
Out[49]:
In [50]:
#dbase1m.sort_index()
In [51]:
tweet_freq = dbase1m.sort_index()
tweet_freq.plot()
Out[51]:
In [52]:
avg = int(dbase1m.mean())
avg
Out[52]:
In [53]:
tweeted_from = pd.Series.value_counts(dbase['location'], normalize=False, sort=True, ascending=False, bins=None)
tweeted_from[:50].plot(kind='bar')
Out[53]:
In [54]:
top_tweeter = pd.Series.value_counts(dbase['name'], normalize=False, sort=True, ascending=False, bins=None)
top_tweeter_screen_name = pd.Series.value_counts(dbase['screen_name'], normalize=False, sort=True, ascending=False, bins=None)
top_tweeter_screen_name[:20].plot(kind='bar')
Out[54]:
In [55]:
source_tweet = pd.Series.value_counts(dbase['source'], normalize=False, sort=True, ascending=False, bins=None)
source_tweet[:10].plot(kind='bar')
Out[55]:
In [56]:
import nltk
from nltk.corpus import stopwords
from nltk import FreqDist
stop = stopwords.words('english')
stop2 = stopwords.words('spanish')
otherwords = ['rt', 'vs', '-', '&', '', '#bluejays']
text = dbase['text']
tokens = []
for txt in text.values:
tokens.extend([t.lower().strip(":,.") for t in txt.split()])
In [57]:
filtered_tokens = [w for w in tokens if not w in stop]
filtered_tokens_spanish = [w for w in filtered_tokens if not w in stop2]
other_filtered = [w for w in filtered_tokens_spanish if not w in otherwords]
freq_dist = nltk.FreqDist(other_filtered)
z = 20
print "The {} most common words tweeted;".format(z)
sorted(freq_dist, key=lambda key: (-freq_dist[key], key))[:z]
Out[57]:
In [58]:
freq_dist.plot(20)
In [59]:
dbase.created_at.value_counts()
Out[59]:
In [60]:
source_geo = pd.Series.value_counts(dbase['geo'], normalize=False, sort=True, ascending=False, bins=None)
In [61]:
#source_geo
In [62]:
import ast
for lines in dbase['geo']:
if type(lines) == str:
mylines = ast.literal_eval(lines)
games_geo_coordinates = mylines["coordinates"]
print games_geo_coordinates[0],
print ",",
print games_geo_coordinates[1]
In [63]:
import folium
from collections import namedtuple
filepath = "/home/al/projects/twitter/csv/jaysgeo.html"
na_coordinates = (43.641438,-79.389353) # Rogers Centre Toronto, Ontario
na_map = folium.Map(location=na_coordinates, zoom_start=4)
Location = namedtuple('Location', ['latitude', 'longitude'])
count_locations = 0
for lines in dbase['geo']:
if type(lines) == str:
count_locations += 1
mylines = ast.literal_eval(lines)
games_geo_coordinates = mylines["coordinates"]
location = Location(games_geo_coordinates[1], games_geo_coordinates[0])
na_map.circle_marker(location, popup=str(location))
na_map.create_map(filepath)
In [64]:
print str(count_locations) + " users reporting coordinates"
print str(total_tweets) + " total tweets"
percent = round(float(count_locations) / float(total_tweets) * 100, 2)
print str(percent) + "%"
In [84]:
from IPython.display import HTML
HTML('<iframe src=http://www.linuxnorth.org/jaysgeo.html width=960 height=530></iframe>')
Out[84]: