import requests
from bs4 import BeautifulSoup
# soup = BeautifulSoup(open("/home/al/projects/webscraping/venv/weather_app/Timmins.html"), "html.parser")
url = 'https://weather.gc.ca/city/pages/on-127_metric_e.html'
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
#list(soup.children)
[type(item) for item in list(soup.children)]
html = list(soup.children)[6]
len(list(html.children))
Start of Work
Timmins, ON - 10:20 AM EDT Thursday 18 April 2019
Temp - 2C
Condition - Light Rain
Pressure - Rising
Humidity - 99%
Wind - N 18 km/h
Visibility - 24km
Tonight: Periods of rain changing to periods of light snow this evening. Wind north 20 km/h gusting to 40 becoming light late this evening. Low minus 2. Wind chill minus 7 overnight.
Fri, 19 Apr: Clearing in the morning. High 9. UV index 6 or high.
Night: Partly cloudy. Becoming clear near midnight. Wind up to 15 km/h. Low minus 4. Wind chill minus 5 overnight.
city_find = soup.find_all('h1')[0]
city = city_find.get_text()
print(city)
date_whole_div = soup.find_all('div', class_="col-sm-10 text-center")
todays_date_in_div = soup.find_all('dd', class_="mrgn-bttm-0")[1]
print(todays_date_in_div)
todays_date = todays_date_in_div.get_text()
todays_date
city
soup.find_all('h1')[0].get_text()
print('{}{}'.format(city, todays_date))
div_tables = soup.find_all('div', class_='div-table')
#div_tables
today_is_html = soup.find_all('strong')[0]
tomorrow_is_html = soup.find_all('strong')[1]
print(today_is_html)
print(tomorrow_is_html)
today_is = today_is_html.get_text()
tomorrow_is = tomorrow_is_html.get_text()
print('{} and {}'.format(today_is, tomorrow_is))
testing = soup.findall('div', class='div-row dir-row1 div-row-head') testing
i = 0
list = soup.find_all('strong')
# print(list[10].get_text())
list
find_temp = soup.find_all('span', class_='wxo-metric-hide')[0]
print(find_temp.get_text())
find_conditions_all = soup.find_all('dd', class_='mrgn-bttm-0')
conditions = find_conditions_all[2]
conditions.get_text()
barometric_pressure_dd = find_conditions_all[3]
barometric_pressure = barometric_pressure_dd.get_text()
barometric_pressure
#score, seperator, subject = title.partition('::')
beg, sep, end = barometric_pressure.partition("a")
beg
tendency_dd = find_conditions_all[5]
tendency = tendency_dd.get_text()
tendency
print('Barometric Pressure is {}a and {}'.format(beg, tendency))
humidity_dd = find_conditions_all[10]
humidity = humidity_dd.get_text()
print(humidity)
wind_dd = find_conditions_all[11]
wind = wind_dd.get_text()
wind.strip('\n')
p = find_conditions_all[3].get_text().replace('\n ','')
print('{} sdafj'.format(p))
extended_forecast_top_level = soup.find_all('table', class_='table mrgn-bttm-md mrgn-tp-sm textforecast')
len(extended_forecast_top_level)
extended_field0 = soup.select('tbody')
extended_field_td = soup.select('td')
extended_field_td
# Sunrise and sunset
sunrise = soup.select('td')[25].get_text()
sunset = soup.select('td')[26].get_text()
print('Sunrise: {} Sunset: {}'.format(sunrise, sunset))
# Sunsets
sunset = soup.select('td')[26].get_text()
print('Sunset: {}'.format(sunset))
# use this convention for the following field0_td , field1_td
soup.select('td')[0].get_text()
soup.select('td')[1].get_text()
soup.select('td')[2].get_text()
soup.select('td')[3].get_text()
soup.select('td')[4].get_text().strip(' ')
soup.select('td')[5].get_text()
soup.select('td')[6].get_text().strip(' ')
soup.select('td')[7].get_text()
soup.select('td')[8].get_text()
soup.select('td')[9].get_text()
print(soup.select('td')[10].get_text())
soup.select('td')[11].get_text()
soup.select('td')[12].get_text()
soup.select('td')[13].get_text()
soup.select('td')[14].get_text().strip(' ')
soup.select('td')[15].get_text()
soup.select('td')[16].get_text()
soup.select('td')[17].get_text()
soup.select('td')[18].get_text().strip(' ')
soup.select('td')[19].get_text()
soup.select('td')[20].get_text()
soup.select('td')[21].get_text()
soup.select('td')[22].get_text().strip(' ')
soup.select('td')[23].get_text()
soup.select('td')[24].get_text()
soup.select('td')[26]
table_wxo_normal = soup.find_all('table', class_='table wxo-normals')
min_max_field = soup.select('td span')
min_max_field
max = min_max_field[0]
min = min_max_field[2]
normals_min_max = max.get_text().strip('.'), min.get_text().strip('.')
print('Normals -- High: {} Low: {}'.format(normals_min_max[0], normals_min_max[1]))
sun = soup.select('td')
sun.pop(-1)
sun.pop(-1)
sun.pop()
sun
soup.select('div h2')[4].get_text() == 'No Alerts in effect'
soup.select('a div')
x = soup.select('a div')
warnings = soup.find_all('div', class_='col-xs-10 text-center')
warnings
alert1 = warnings.pop()
alert1.get_text().strip(' ')
alert2 = warnings.pop()
alert2.get_text().strip(' ')