Fran314/weather-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

started working on OpenMeteo

Browse Source
This commit is contained in:
Fran314
2024-10-27 16:21:19 +01:00
parent 4ddc0ba163
commit 1f8fcb3273
5 changed files with 188 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+14 -13
View File
@@ -21,17 +21,17 @@ The scraper collects data and arranges in the following scheme
{
"today": {
"[0..23]": {
"temp": {
"temperature": {
"type": "number",
"unitOfMeasurement": "degrees",
"description": "expected temperature at the given hour"
},
"prec": {
"precipitation": {
"type": "number",
"unitOfMeasurement": "mm",
"description": "expected precipitation at the given hour"
},
"code": {
"weatherCode": {
"type": "string",
"description": "weather code (sunny / cloudy / ...)"
}
@@ -39,17 +39,17 @@ The scraper collects data and arranges in the following scheme
},
"tomorrow": {
"[0..23]": {
"temp": {
"temperature": {
"type": "number",
"unitOfMeasurement": "degrees",
"description": "expected temperature at the given hour"
},
"prec": {
"precipitation": {
"type": "number",
"unitOfMeasurement": "mm",
"description": "expected precipitation at the given hour"
},
"code": {
"weatherCode": {
"type": "string",
"description": "weather code (sunny / cloudy / ...)"
}
@@ -57,17 +57,17 @@ The scraper collects data and arranges in the following scheme
},
"dayAfterTomorrow": {
"[0..23]": {
"temp": {
"temperature": {
"type": "number",
"unitOfMeasurement": "degrees",
"description": "expected temperature at the given hour"
},
"prec": {
"precipitation": {
"type": "number",
"unitOfMeasurement": "mm",
"description": "expected precipitation at the given hour"
},
"code": {
"weatherCode": {
"type": "string",
"description": "weather code (sunny / cloudy / ...)"
}
@@ -75,22 +75,22 @@ The scraper collects data and arranges in the following scheme
},
"week": {
"[0..6]": {
"minTemp": {
"minimumTemperature": {
"type": "number",
"unitOfMeasurement": "degrees",
"description": "minimum expected temperature for the day"
},
"maxTemp": {
"maximumTemperature": {
"type": "number",
"unitOfMeasurement": "degrees",
"description": "maximum expected temperature for the day"
},
"totPrec": {
"precipitationSum": {
"type": "number",
"unitOfMeasurement": "mm",
"description": "total expected precipitation for the day"
},
"code": {
"weatherCode": {
"type": "string",
"description": "weather code (sunny / cloudy / ...)"
}
@@ -119,4 +119,5 @@ eventually, together with the current level of implementation
| ---------------------------------------------------- | ------ | ---------------------------------------- |
| [iLMeteo](https://www.ilmeteo.it) | 🚧 | Weather Code not working |
| [3Bmeteo](https://www.3bmeteo.com/) | 🚧 | Precipitation might not work as intended |
| [OpenMeteo](https://open-meteo.com/) | 🚧 | Weather Code are given in WMO Code |
| [Meteo Aeronautica Militare](http://www.meteoam.it/) | ⛔️ | |
index.js
+6 -3
View File
@@ -3,6 +3,7 @@ import puppeteer from 'puppeteer'
import fetchILMeteo from './scrapers/iLMeteo.js'
import fetch3Bmeteo from './scrapers/3Bmeteo.js'
import fetchOpenMeteo from './scrapers/OpenMeteo.js'
const NIX_OPS = {
executablePath: process.env.NIX_CHROMIUM_PATH,
@@ -12,9 +13,10 @@ const opts = process.env.ON_NIX ? NIX_OPS : {}
const run = async () => {
const browser = await puppeteer.launch(opts)
const [iLMeteo, treBmeteo] = await Promise.all([
const [iLMeteo, treBmeteo, openMeteo] = await Promise.all([
fetchILMeteo(browser),
fetch3Bmeteo(browser),
fetchOpenMeteo(),
])
await browser.close()
@@ -22,8 +24,9 @@ const run = async () => {
return {
iLMeteo,
treBmeteo,
openMeteo,
}
}
console.dir(await run(), { depth: null })
// await run()
const result = await run()
console.dir(result, { depth: null })
scrapers/3Bmeteo.js
+57 -18
View File
@@ -9,6 +9,15 @@ const parsePrec = el => {
return parseFloat(text)
}
const getStartTime = isToday => {
if (!isToday) return 0
const d = new Date()
const h = d.getHours()
if (d.getMinutes() > 30) return h + 1
return h
}
const scrapePage = async (browser, url, isToday) => {
let result = {}
@@ -20,7 +29,8 @@ const scrapePage = async (browser, url, isToday) => {
.waitHandle()
const rows = await tablePrevisioni.$$('.row-table.noPad')
const startTime = isToday ? new Date().getHours() : 0
// const startTime = getStartTime(isToday)
const startTime = isToday ? new Date().getHours() + 1 : 0
const endTime = 23
for (let i = startTime; i <= endTime; i++) {
@@ -30,14 +40,30 @@ const scrapePage = async (browser, url, isToday) => {
const [rowLeft, rowRight] = await row.$$(':scope > div')
const codeDiv = (await (await rowLeft.$('.row-table')).$$('div'))[2]
const [tempDiv, precDiv] = await (
await rowRight.$('.row-table')
).$$('div')
const rightFields = await (await rowRight.$('.row-table')).$$('div')
const tempDiv = rightFields[0]
const precDiv = rightFields[1]
const appTempDiv = rightFields[4]
const code = await codeDiv.evaluate(el => el.textContent.trim())
const temp = await (await tempDiv.$('span')).evaluate(parseTemp)
const prec = await (await precDiv.$('span')).evaluate(parsePrec)
result[i] = { temp, prec, code }
const weatherCode = await codeDiv.evaluate(el =>
el.textContent.trim(),
)
const temperature = await (
await tempDiv.$('span')
).evaluate(parseTemp)
const precipitation = await (
await precDiv.$('span')
).evaluate(parsePrec)
const apparentTemperature = await (
await appTempDiv.$('span')
).evaluate(parseTemp)
result[i] = {
temperature,
precipitation,
apparentTemperature,
weatherCode,
}
} catch (error) {
result[i] = null
console.log(error)
@@ -50,18 +76,31 @@ const scrapePage = async (browser, url, isToday) => {
}
const getDaySummary = day => {
let minTemp = Number.MAX_VALUE
let maxTemp = Number.MIN_VALUE
let totPrec = 0
let minimumTemperature = Number.MAX_VALUE
let maximumTemperature = Number.MIN_VALUE
let minimumApparentTemperature = Number.MAX_VALUE
let maximumApparentTemperature = Number.MIN_VALUE
let precipitationSum = 0
for (const h in day) {
minTemp = Math.min(minTemp, day[h].temp)
maxTemp = Math.max(maxTemp, day[h].temp)
totPrec += day[h].prec
minimumTemperature = Math.min(minimumTemperature, day[h]?.temperature)
maximumTemperature = Math.max(maximumTemperature, day[h]?.temperature)
minimumApparentTemperature = Math.min(
minimumApparentTemperature,
day[h]?.apparentTemperature,
)
maximumApparentTemperature = Math.max(
maximumApparentTemperature,
day[h]?.apparentTemperature,
)
precipitationSum += day[h]?.precipitation
}
return {
minTemp,
maxTemp,
totPrec,
minimumTemperature,
maximumTemperature,
minimumApparentTemperature,
maximumApparentTemperature,
precipitationSum,
weatherCode: null,
}
}
@@ -82,6 +121,6 @@ export default async browser => {
today: results[0],
tomorrow: results[1],
dayAfterTomorrow: results[2],
week: results.map(getDaySummary),
week: { ...results.map(getDaySummary) },
}
}
scrapers/OpenMeteo.js
+71
View File
@@ -0,0 +1,71 @@
const getDailyData = async () => {
const response = await fetch(
'https://api.open-meteo.com/v1/forecast?latitude=43.7085&longitude=10.4036&hourly=temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,weather_code&timezone=Europe%2FBerlin&forecast_days=3',
)
const body = await response.json()
let today = {}
const startTime = new Date().getHours() + 1
for (let i = startTime; i <= 23; i++) {
today[i] = {
temperature: body.hourly.temperature_2m[i],
precipitation: body.hourly.precipitation[i],
apparentTemperature: body.hourly.apparent_temperature[i],
weatherCode: body.hourly.weather_code[i],
}
}
let tomorrow = {}
for (let i = 0; i <= 23; i++) {
tomorrow[i] = {
temperature: body.hourly.temperature_2m[24 + i],
precipitation: body.hourly.precipitation[24 + i],
apparentTemperature: body.hourly.apparent_temperature[24 + i],
weatherCode: body.hourly.weather_code[24 + i],
}
}
let dayAfterTomorrow = {}
for (let i = 0; i <= 23; i++) {
dayAfterTomorrow[i] = {
temperature: body.hourly.temperature_2m[48 + i],
precipitation: body.hourly.precipitation[48 + i],
apparentTemperature: body.hourly.apparent_temperature[48 + i],
weatherCode: body.hourly.weather_code[48 + i],
}
}
return {
today,
tomorrow,
dayAfterTomorrow,
}
}
const getWeekData = async () => {
const response = await fetch(
'https://api.open-meteo.com/v1/forecast?latitude=43.7085&longitude=10.4036&daily=weather_code,temperature_2m_max,temperature_2m_min,apparent_temperature_max,apparent_temperature_min,precipitation_sum&timezone=Europe%2FBerlin',
)
const body = await response.json()
let week = {}
for (let i = 0; i < 6; i++) {
week[i] = {
minimumTemperature: body.daily.temperature_2m_min[i],
maximumTemperature: body.daily.temperature_2m_max[i],
minimumApparentTemperature: body.daily.apparent_temperature_min[i],
maximumApparentTemperature: body.daily.apparent_temperature_max[i],
precipitationSum: body.daily.precipitation_sum[i],
weatherCode: body.daily.weather_code[i],
}
}
return week
}
export default async () => {
const [daily, week] = await Promise.all([getDailyData(), getWeekData()])
return {
...daily,
week,
}
}
scrapers/iLMeteo.js
+40 -16
View File
@@ -22,9 +22,14 @@ const scrapePage = async (browser, url, isToday) => {
try {
const row = await weatherTable.$$('.latest_detection')
const fields = await row[1]?.$$('td')
const temp = await fields[2].evaluate(parseTemp)
const prec = await fields[3].evaluate(parsePrec)
result[startTime - 1] = { temp, prec, code: null }
const temperature = await fields[2].evaluate(parseTemp)
const precipitation = await fields[3].evaluate(parsePrec)
result[startTime - 1] = {
temperature,
precipitation,
apparentTemperature: null,
weatherCode: null,
}
} catch (error) {
result[startTime - 1] = null
// console.log(error)
@@ -38,9 +43,15 @@ const scrapePage = async (browser, url, isToday) => {
const fields = await weatherTable.$$(selector)
const temp = await fields[2].evaluate(parseTemp)
const prec = await fields[3].evaluate(parsePrec)
result[i] = { temp, prec, code: null }
const temperature = await fields[2].evaluate(parseTemp)
const precipitation = await fields[3].evaluate(parsePrec)
const apparentTemperature = await fields[6].evaluate(parseTemp)
result[i] = {
temperature,
precipitation,
apparentTemperature,
weatherCode: null,
}
} catch (error) {
result[i] = null
// console.log(error)
@@ -64,18 +75,31 @@ const rearrangeResults = results => {
}
const getDaySummary = day => {
let minTemp = Number.MAX_VALUE
let maxTemp = Number.MIN_VALUE
let totPrec = 0
let minimumTemperature = Number.MAX_VALUE
let maximumTemperature = Number.MIN_VALUE
let minimumApparentTemperature = Number.MAX_VALUE
let maximumApparentTemperature = Number.MIN_VALUE
let precipitationSum = 0
for (const h in day) {
minTemp = Math.min(minTemp, day[h].temp)
maxTemp = Math.max(maxTemp, day[h].temp)
totPrec += day[h].prec
minimumTemperature = Math.min(minimumTemperature, day[h]?.temperature)
maximumTemperature = Math.max(maximumTemperature, day[h]?.temperature)
minimumApparentTemperature = Math.min(
minimumApparentTemperature,
day[h]?.apparentTemperature,
)
maximumApparentTemperature = Math.max(
maximumApparentTemperature,
day[h]?.apparentTemperature,
)
precipitationSum += day[h]?.precipitation
}
return {
minTemp,
maxTemp,
totPrec,
minimumTemperature,
maximumTemperature,
minimumApparentTemperature,
maximumApparentTemperature,
precipitationSum,
weatherCode: null,
}
}
@@ -98,6 +122,6 @@ export default async browser => {
today: results[0],
tomorrow: results[1],
dayAfterTomorrow: results[2],
week: results.map(getDaySummary),
week: { ...results.map(getDaySummary) },
}
}