In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
In [2]:
df = pd.read_csv('../resources/owm09172017_09172018.csv')
Overview of the dataset
In [3]:
df.head()
Out[3]:
In [4]:
df.shape
Out[4]:
In [5]:
df.columns
Out[5]:
All data is, by design, from the Central Park station
Ideas
Cloud cover sunlight/present weather
Could I do something neat with wind direction and speed? Some kind of cartesian coordinates with vectors for wind?
Could choose daily data if hourly is too cluttered. Choose daily or monthly if displaying muliple years' worth of data
In [6]:
columns_keep = ['DATE', 'HOURLYSKYCONDITIONS', 'HOURLYVISIBILITY', 'HOURLYPRSENTWEATHERTYPE', 'HOURLYRelativeHumidity', 'HOURLYWindSpeed', 'HOURLYWindDirection',
'HOURLYWindGustSpeed', 'HOURLYPrecip', 'DAILYSunrise', 'DAILYSunset', 'DAILYWeather', 'DAILYPrecip', 'DAILYSnowfall']
In [7]:
df_thin = df[columns_keep]
In [8]:
df_thin.head()
Out[8]:
Missing data
In [9]:
df_thin.isna().sum()
Out[9]:
Available Data
In [10]:
df_thin.shape[0] - df_thin.isna().sum()
Out[10]:
Drop daily values for now
In [11]:
df_skinny = df_thin.drop(['DAILYWeather', 'DAILYPrecip', 'DAILYSnowfall'], axis=1)
In [12]:
df_skinny.shape[0] - df_skinny.isna().sum()
Out[12]:
Create features for DATE parts
In [13]:
df_skinny['DATE'].head()
Out[13]:
In [14]:
def get_year(row): # Get the year from DATE column
return int(row[:4])
df_skinny['Year'] = df_skinny['DATE'].apply(get_year)
In [15]:
def get_month(row): # Get the month from DATE column
return int(row[5:7])
df_skinny['Month'] = df_skinny['DATE'].apply(get_month)
In [16]:
def get_day(row): # Get the day from DATE column
return int(row[8:10])
df_skinny['Day'] = df_skinny['DATE'].apply(get_day)
In [17]:
def get_hour(row): # Get the hour from DATE column
return int(row[11:13])
df_skinny['Hour'] = df_skinny['DATE'].apply(get_hour)
Create matching features for sunrise and sunset
In [18]:
df_skinny['SunriseHour'] = df_skinny['DAILYSunrise'] // 100
In [19]:
df_skinny['SunriseMin'] = df_skinny['DAILYSunrise'] % 100
In [20]:
df_skinny['SunsetHour'] = df_skinny['DAILYSunset'] // 100
In [21]:
df_skinny['SunsetMin'] = df_skinny['DAILYSunset'] % 100
In [22]:
df_skinny['HOURLYSKYCONDITIONS'].head()
Out[22]:
In [23]:
def get_weather_type(row): #Get the weather type from HOURLYSKYCONDITIONS column
if type(row) == str:
search = re.findall(r'([A-Z]*)', row)
return search[0]
else:
return 'NaN'
In [24]:
df_skinny['HourlySkyWeather'] = df_skinny['HOURLYSKYCONDITIONS'].apply(get_weather_type)
In [25]:
df_skinny['HourlySkyWeather'].head()
Out[25]:
In [26]:
set(df_skinny['HourlySkyWeather'])
Out[26]:
'10' which was supposed to denote partly cloudy, does that entry exist in the original feature? No
In [27]:
df_skinny['HOURLYSKYCONDITIONS'][df_skinny['HOURLYSKYCONDITIONS'].str.startswith('10', na=False)]
Out[27]:
Could try to extract all weather types from HOURLYSKYCONDITIONS, but hold off until after looking at HOURLYPRSENTWEATHERTYPE
In [28]:
df_skinny['HOURLYVISIBILITY'].describe()
Out[28]:
In [29]:
set(df_skinny['HOURLYVISIBILITY'])
Out[29]:
Remove the trailing V because the documentation does not provide a justification for it
In [30]:
def get_visibility(row): # Create tidy hourly visibility feature
if type(row) == str:
search = re.search(r'[0-9\.]*', row)
return float(search[0])
else:
return 'NaN'
In [31]:
df_skinny['HourlyVisibility'] = df_skinny['HOURLYVISIBILITY'].apply(get_visibility)
In [32]:
set(df_skinny['HourlyVisibility'])
Out[32]:
In [33]:
df_skinny['HOURLYPRSENTWEATHERTYPE'].describe()
Out[33]:
HOURLYPRSENTWEATHERTYPE is formatted as AU|AW|MW where AU and AW are automatic weather type measurements and MW is a manual measurement
In [34]:
set(df_skinny['HOURLYPRSENTWEATHERTYPE'])
Out[34]:
Test regex
In [35]:
string = 'SN:03 FZ:8 FG:2 |FG:35 SN:72 |'
re.split(r'\|', string)
Out[35]:
In [36]:
def get_AU_weather(row): # Get the AU weather measurement
if type(row) == str:
search = re.split(r'\|', row)
return search[0]
else:
return 'NaN'
def get_AW_weather(row): # Get the AW weather measurement
if type(row) == str:
search = re.split(r'\|', row)
return search[1]
else:
return 'NaN'
def get_MW_weather(row): # Get the AU weather measurement
if type(row) == str:
search = re.split(r'\|', row)
return search[2]
else:
return 'NaN'
In [37]:
df_skinny['AUWeather'] = df_skinny['HOURLYPRSENTWEATHERTYPE'].apply(get_AU_weather)
df_skinny['AWWeather'] = df_skinny['HOURLYPRSENTWEATHERTYPE'].apply(get_AW_weather)
df_skinny['MWWeather'] = df_skinny['HOURLYPRSENTWEATHERTYPE'].apply(get_MW_weather)
In [38]:
set(df_skinny['AUWeather'])
Out[38]:
Refine these weather type features if they are to be used
In [39]:
df_skinny['HOURLYRelativeHumidity'].describe()
Out[39]:
In [40]:
fig = plt.figure()
axs = fig.add_subplot(111)
axs.hist(df_skinny['HOURLYRelativeHumidity'][df_skinny['HOURLYRelativeHumidity'].isna() == False], bins=25)
axs.set_title('Hourly Relative Humidity')
Out[40]:
In [41]:
df_skinny['HOURLYWindSpeed'].describe()
Out[41]:
In [42]:
fig = plt.figure()
axs = fig.add_subplot(111)
axs.hist(df_skinny['HOURLYWindSpeed'][df_skinny['HOURLYWindSpeed'].isna() == False], bins=30)
axs.set_title('Hourly Wind Speed')
Out[42]:
Indication of rounding for wind speed at low values, eg. at 0, 3, 5 are used in place of 1, 2 and 4
Wind direction is given in degrees from true north, with 360 as North and 000 as calm conditions
No explanation in the documentation for VRB, so treat it as a missing value
In [43]:
df_skinny['HOURLYWindDirection'].describe()
Out[43]:
In [44]:
set(df_skinny['HOURLYWindDirection'])
Out[44]:
Make a numeric feature for Wind Direction
In [45]:
def get_numeric_wind(row): # Create a numeric Wind Direction feature and remove VRB encoding
if str(row).isnumeric() == True:
return int(row)
else:
return False
In [46]:
df_skinny['HourlyWindDirection'] = df_skinny['HOURLYWindDirection'].apply(get_numeric_wind)
In [47]:
set(df_skinny['HourlyWindDirection'])
Out[47]:
In [48]:
df_skinny['DATE'].groupby(by = df_skinny['HourlyWindDirection']).count()
Out[48]:
In [49]:
df_skinny['DATE'].groupby(by = df_skinny['HourlyWindDirection']).count() / df_skinny['DATE'][df_skinny['HourlyWindDirection'].isna() == False].count()
Out[49]:
In [50]:
df_wind = df_skinny['DATE'][df_skinny['HourlyWindDirection'] != 0].groupby(by = df_skinny['HourlyWindDirection']).count() / df_skinny['DATE'][df_skinny['HourlyWindDirection'].isna() == False][df_skinny['HourlyWindDirection'] != 0].count()
In [51]:
df_wind.sort_values(ascending = False)[:10]
Out[51]:
The most frequent nonzero wind direction values belong to east and west directions, ie 50-70 for west and 260-300 for east
In [52]:
df_skinny['HOURLYWindSpeed'].describe()
Out[52]:
In [53]:
fig = plt.figure()
axs = fig.add_subplot(111)
axs.hist(df_skinny['HOURLYWindSpeed'][df_skinny['HOURLYWindSpeed'].isna() == False], bins = 30)
axs.set_title('Hourly Wind Speed')
Out[53]:
Again, there is evidence of rounding at low integer values, ie at 0, 3, 5 instead of 1, 2 and 4
In [54]:
df_skinny['HOURLYWindGustSpeed'].describe()
Out[54]:
In [55]:
fig = plt.figure()
axs = fig.add_subplot(111)
axs.hist(df_skinny['HOURLYWindGustSpeed'][df_skinny['HOURLYWindGustSpeed'].isna() == False], bins = 30)
axs.set_title('Hourly Wind Gust Speed')
Out[55]:
There was probably rounding at wind gust speeds approaching 20 mph
Make feature for minutesof sunlight
In [56]:
df_skinny[['SunriseHour', 'SunriseMin', 'SunsetHour', 'SunsetMin']].head()
Out[56]:
In [57]:
df_skinny['SunlightMin'] = (df_skinny['SunsetHour'] - df_skinny['SunriseHour']) * 60 + df_skinny['SunsetMin'] - df_skinny['SunriseMin']
In [58]:
df_skinny[['SunriseHour', 'SunriseMin', 'SunsetHour', 'SunsetMin', 'SunlightMin']].iloc[100]
# Spot checking the results
Out[58]:
Remove troublesome values the features to be used in visualization:
Year, Month, Day, Hour, SunlightMin, HourlySkyWeather, HourlyVisibility, HourlyWindDirection, HOURLYWindSpeed
In [59]:
keep = ['Year', 'Month', 'Day', 'Hour', 'SunlightMin', 'HourlySkyWeather', 'HourlyVisibility', 'HourlyWindDirection', 'HOURLYWindSpeed']
In [60]:
df_skinny[keep].count()
Out[60]:
Trear HOURLYWindSpeed for missing values
In [61]:
df_skinny['HOURLYWindSpeed'].head()
Out[61]:
In [62]:
def get_wind_speed(row): # Create hourly wind speed feature
if pd.isna(row) == False:
return row
else:
return 'NaN'
In [63]:
df_skinny['HourlyWindSpeed'] = df_skinny['HOURLYWindSpeed'].apply(get_wind_speed)
In [64]:
df_skinny['HourlyWindSpeed'].count()
Out[64]:
In [65]:
keep = ['Year', 'Month', 'Day', 'Hour', 'SunlightMin', 'HourlySkyWeather', 'HourlyVisibility', 'HourlyWindDirection', 'HourlyWindSpeed', 'SunriseHour', 'SunriseMin', 'SunsetHour', 'SunsetMin']
In [66]:
df_keep = df_skinny[keep]
In [67]:
df_keep.count()
Out[67]:
In [68]:
df_keep.columns
Out[68]:
In [69]:
df_keep.iloc[0]
Out[69]:
In [70]:
df_keep.iloc[-1]
Out[70]:
In [71]:
df_keep['Year'].groupby(by = df_keep['Hour']).count()
Out[71]:
Something weird might be going on with the time features...
In [72]:
df_keep[df_keep['Hour'] == 23].head()
Out[72]:
In [73]:
df_keep[df_keep['Hour'] == 22].head()
Out[73]:
There are duplicate rows!
In [74]:
df_keep['HourlySkyWeather'].groupby(by = [df_keep['Year'], df_keep['Month'], df_keep['Day'], df_keep['Hour']]).count()
Out[74]:
In [75]:
df_dedup = df_keep.drop_duplicates(subset = ['Year', 'Month', 'Day', 'Hour'], keep = 'first')
In [76]:
df_dedup['HourlySkyWeather'].groupby(by = [df_keep['Year'], df_keep['Month'], df_keep['Day'], df_keep['Hour']]).count()
Out[76]:
In [77]:
df_dedup.shape[0] / 24
Out[77]:
Might be missing some data but I'll try using df_dedup
In [78]:
df_dedup.to_csv('../resources/weatherData.csv', index = False)
In [79]:
df_dedup['SunlightMin'].describe()
Out[79]:
In [80]:
24 * 60 #Minutes in a day
Out[80]:
In [81]:
1440 - df_dedup['SunlightMin'].max() #Minimum minutes of darkness
Out[81]:
In [82]:
1440 - df_dedup['SunlightMin'].min() #Maximum minutes of darkness
Out[82]:
In [83]:
df_dedup['SunriseHour'].describe()
Out[83]:
In [84]:
df_dedup['SunsetHour'].describe()
Out[84]:
In [85]:
set(df_dedup['HourlyVisibility'])
Out[85]:
In [86]:
set(df_dedup['HourlySkyWeather'])
Out[86]: