From 1f8fcb32738956899059388261d6fc09f6c27d18 Mon Sep 17 00:00:00 2001 From: Fran314 Date: Sun, 27 Oct 2024 16:21:19 +0100 Subject: [PATCH] started working on OpenMeteo --- README.md | 27 +++++++-------- index.js | 9 +++-- scrapers/3Bmeteo.js | 77 ++++++++++++++++++++++++++++++++----------- scrapers/OpenMeteo.js | 71 +++++++++++++++++++++++++++++++++++++++ scrapers/iLMeteo.js | 56 ++++++++++++++++++++++--------- 5 files changed, 189 insertions(+), 51 deletions(-) create mode 100644 scrapers/OpenMeteo.js diff --git a/README.md b/README.md index 5beb934..ee142bb 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,17 @@ The scraper collects data and arranges in the following scheme { "today": { "[0..23]": { - "temp": { + "temperature": { "type": "number", "unitOfMeasurement": "degrees", "description": "expected temperature at the given hour" }, - "prec": { + "precipitation": { "type": "number", "unitOfMeasurement": "mm", "description": "expected precipitation at the given hour" }, - "code": { + "weatherCode": { "type": "string", "description": "weather code (sunny / cloudy / ...)" } @@ -39,17 +39,17 @@ The scraper collects data and arranges in the following scheme }, "tomorrow": { "[0..23]": { - "temp": { + "temperature": { "type": "number", "unitOfMeasurement": "degrees", "description": "expected temperature at the given hour" }, - "prec": { + "precipitation": { "type": "number", "unitOfMeasurement": "mm", "description": "expected precipitation at the given hour" }, - "code": { + "weatherCode": { "type": "string", "description": "weather code (sunny / cloudy / ...)" } @@ -57,17 +57,17 @@ The scraper collects data and arranges in the following scheme }, "dayAfterTomorrow": { "[0..23]": { - "temp": { + "temperature": { "type": "number", "unitOfMeasurement": "degrees", "description": "expected temperature at the given hour" }, - "prec": { + "precipitation": { "type": "number", "unitOfMeasurement": "mm", "description": "expected precipitation at the given hour" }, - "code": { + "weatherCode": { "type": "string", "description": "weather code (sunny / cloudy / ...)" } @@ -75,22 +75,22 @@ The scraper collects data and arranges in the following scheme }, "week": { "[0..6]": { - "minTemp": { + "minimumTemperature": { "type": "number", "unitOfMeasurement": "degrees", "description": "minimum expected temperature for the day" }, - "maxTemp": { + "maximumTemperature": { "type": "number", "unitOfMeasurement": "degrees", "description": "maximum expected temperature for the day" }, - "totPrec": { + "precipitationSum": { "type": "number", "unitOfMeasurement": "mm", "description": "total expected precipitation for the day" }, - "code": { + "weatherCode": { "type": "string", "description": "weather code (sunny / cloudy / ...)" } @@ -119,4 +119,5 @@ eventually, together with the current level of implementation | ---------------------------------------------------- | ------ | ---------------------------------------- | | [iLMeteo](https://www.ilmeteo.it) | 🚧 | Weather Code not working | | [3Bmeteo](https://www.3bmeteo.com/) | 🚧 | Precipitation might not work as intended | +| [OpenMeteo](https://open-meteo.com/) | 🚧 | Weather Code are given in WMO Code | | [Meteo Aeronautica Militare](http://www.meteoam.it/) | ⛔️ | | diff --git a/index.js b/index.js index baa1d19..297af52 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ import puppeteer from 'puppeteer' import fetchILMeteo from './scrapers/iLMeteo.js' import fetch3Bmeteo from './scrapers/3Bmeteo.js' +import fetchOpenMeteo from './scrapers/OpenMeteo.js' const NIX_OPS = { executablePath: process.env.NIX_CHROMIUM_PATH, @@ -12,9 +13,10 @@ const opts = process.env.ON_NIX ? NIX_OPS : {} const run = async () => { const browser = await puppeteer.launch(opts) - const [iLMeteo, treBmeteo] = await Promise.all([ + const [iLMeteo, treBmeteo, openMeteo] = await Promise.all([ fetchILMeteo(browser), fetch3Bmeteo(browser), + fetchOpenMeteo(), ]) await browser.close() @@ -22,8 +24,9 @@ const run = async () => { return { iLMeteo, treBmeteo, + openMeteo, } } -console.dir(await run(), { depth: null }) -// await run() +const result = await run() +console.dir(result, { depth: null }) diff --git a/scrapers/3Bmeteo.js b/scrapers/3Bmeteo.js index eace321..99b73c9 100644 --- a/scrapers/3Bmeteo.js +++ b/scrapers/3Bmeteo.js @@ -9,6 +9,15 @@ const parsePrec = el => { return parseFloat(text) } +const getStartTime = isToday => { + if (!isToday) return 0 + + const d = new Date() + const h = d.getHours() + + if (d.getMinutes() > 30) return h + 1 + return h +} const scrapePage = async (browser, url, isToday) => { let result = {} @@ -20,7 +29,8 @@ const scrapePage = async (browser, url, isToday) => { .waitHandle() const rows = await tablePrevisioni.$$('.row-table.noPad') - const startTime = isToday ? new Date().getHours() : 0 + // const startTime = getStartTime(isToday) + const startTime = isToday ? new Date().getHours() + 1 : 0 const endTime = 23 for (let i = startTime; i <= endTime; i++) { @@ -30,14 +40,30 @@ const scrapePage = async (browser, url, isToday) => { const [rowLeft, rowRight] = await row.$$(':scope > div') const codeDiv = (await (await rowLeft.$('.row-table')).$$('div'))[2] - const [tempDiv, precDiv] = await ( - await rowRight.$('.row-table') - ).$$('div') - - const code = await codeDiv.evaluate(el => el.textContent.trim()) - const temp = await (await tempDiv.$('span')).evaluate(parseTemp) - const prec = await (await precDiv.$('span')).evaluate(parsePrec) - result[i] = { temp, prec, code } + const rightFields = await (await rowRight.$('.row-table')).$$('div') + const tempDiv = rightFields[0] + const precDiv = rightFields[1] + const appTempDiv = rightFields[4] + + const weatherCode = await codeDiv.evaluate(el => + el.textContent.trim(), + ) + const temperature = await ( + await tempDiv.$('span') + ).evaluate(parseTemp) + const precipitation = await ( + await precDiv.$('span') + ).evaluate(parsePrec) + const apparentTemperature = await ( + await appTempDiv.$('span') + ).evaluate(parseTemp) + + result[i] = { + temperature, + precipitation, + apparentTemperature, + weatherCode, + } } catch (error) { result[i] = null console.log(error) @@ -50,18 +76,31 @@ const scrapePage = async (browser, url, isToday) => { } const getDaySummary = day => { - let minTemp = Number.MAX_VALUE - let maxTemp = Number.MIN_VALUE - let totPrec = 0 + let minimumTemperature = Number.MAX_VALUE + let maximumTemperature = Number.MIN_VALUE + let minimumApparentTemperature = Number.MAX_VALUE + let maximumApparentTemperature = Number.MIN_VALUE + let precipitationSum = 0 for (const h in day) { - minTemp = Math.min(minTemp, day[h].temp) - maxTemp = Math.max(maxTemp, day[h].temp) - totPrec += day[h].prec + minimumTemperature = Math.min(minimumTemperature, day[h]?.temperature) + maximumTemperature = Math.max(maximumTemperature, day[h]?.temperature) + minimumApparentTemperature = Math.min( + minimumApparentTemperature, + day[h]?.apparentTemperature, + ) + maximumApparentTemperature = Math.max( + maximumApparentTemperature, + day[h]?.apparentTemperature, + ) + precipitationSum += day[h]?.precipitation } return { - minTemp, - maxTemp, - totPrec, + minimumTemperature, + maximumTemperature, + minimumApparentTemperature, + maximumApparentTemperature, + precipitationSum, + weatherCode: null, } } @@ -82,6 +121,6 @@ export default async browser => { today: results[0], tomorrow: results[1], dayAfterTomorrow: results[2], - week: results.map(getDaySummary), + week: { ...results.map(getDaySummary) }, } } diff --git a/scrapers/OpenMeteo.js b/scrapers/OpenMeteo.js new file mode 100644 index 0000000..6677254 --- /dev/null +++ b/scrapers/OpenMeteo.js @@ -0,0 +1,71 @@ +const getDailyData = async () => { + const response = await fetch( + 'https://api.open-meteo.com/v1/forecast?latitude=43.7085&longitude=10.4036&hourly=temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,weather_code&timezone=Europe%2FBerlin&forecast_days=3', + ) + const body = await response.json() + + let today = {} + const startTime = new Date().getHours() + 1 + for (let i = startTime; i <= 23; i++) { + today[i] = { + temperature: body.hourly.temperature_2m[i], + precipitation: body.hourly.precipitation[i], + apparentTemperature: body.hourly.apparent_temperature[i], + weatherCode: body.hourly.weather_code[i], + } + } + + let tomorrow = {} + for (let i = 0; i <= 23; i++) { + tomorrow[i] = { + temperature: body.hourly.temperature_2m[24 + i], + precipitation: body.hourly.precipitation[24 + i], + apparentTemperature: body.hourly.apparent_temperature[24 + i], + weatherCode: body.hourly.weather_code[24 + i], + } + } + + let dayAfterTomorrow = {} + for (let i = 0; i <= 23; i++) { + dayAfterTomorrow[i] = { + temperature: body.hourly.temperature_2m[48 + i], + precipitation: body.hourly.precipitation[48 + i], + apparentTemperature: body.hourly.apparent_temperature[48 + i], + weatherCode: body.hourly.weather_code[48 + i], + } + } + return { + today, + tomorrow, + dayAfterTomorrow, + } +} + +const getWeekData = async () => { + const response = await fetch( + 'https://api.open-meteo.com/v1/forecast?latitude=43.7085&longitude=10.4036&daily=weather_code,temperature_2m_max,temperature_2m_min,apparent_temperature_max,apparent_temperature_min,precipitation_sum&timezone=Europe%2FBerlin', + ) + const body = await response.json() + + let week = {} + for (let i = 0; i < 6; i++) { + week[i] = { + minimumTemperature: body.daily.temperature_2m_min[i], + maximumTemperature: body.daily.temperature_2m_max[i], + minimumApparentTemperature: body.daily.apparent_temperature_min[i], + maximumApparentTemperature: body.daily.apparent_temperature_max[i], + precipitationSum: body.daily.precipitation_sum[i], + weatherCode: body.daily.weather_code[i], + } + } + + return week +} +export default async () => { + const [daily, week] = await Promise.all([getDailyData(), getWeekData()]) + + return { + ...daily, + week, + } +} diff --git a/scrapers/iLMeteo.js b/scrapers/iLMeteo.js index ce9518c..3f7db22 100644 --- a/scrapers/iLMeteo.js +++ b/scrapers/iLMeteo.js @@ -22,9 +22,14 @@ const scrapePage = async (browser, url, isToday) => { try { const row = await weatherTable.$$('.latest_detection') const fields = await row[1]?.$$('td') - const temp = await fields[2].evaluate(parseTemp) - const prec = await fields[3].evaluate(parsePrec) - result[startTime - 1] = { temp, prec, code: null } + const temperature = await fields[2].evaluate(parseTemp) + const precipitation = await fields[3].evaluate(parsePrec) + result[startTime - 1] = { + temperature, + precipitation, + apparentTemperature: null, + weatherCode: null, + } } catch (error) { result[startTime - 1] = null // console.log(error) @@ -38,9 +43,15 @@ const scrapePage = async (browser, url, isToday) => { const fields = await weatherTable.$$(selector) - const temp = await fields[2].evaluate(parseTemp) - const prec = await fields[3].evaluate(parsePrec) - result[i] = { temp, prec, code: null } + const temperature = await fields[2].evaluate(parseTemp) + const precipitation = await fields[3].evaluate(parsePrec) + const apparentTemperature = await fields[6].evaluate(parseTemp) + result[i] = { + temperature, + precipitation, + apparentTemperature, + weatherCode: null, + } } catch (error) { result[i] = null // console.log(error) @@ -64,18 +75,31 @@ const rearrangeResults = results => { } const getDaySummary = day => { - let minTemp = Number.MAX_VALUE - let maxTemp = Number.MIN_VALUE - let totPrec = 0 + let minimumTemperature = Number.MAX_VALUE + let maximumTemperature = Number.MIN_VALUE + let minimumApparentTemperature = Number.MAX_VALUE + let maximumApparentTemperature = Number.MIN_VALUE + let precipitationSum = 0 for (const h in day) { - minTemp = Math.min(minTemp, day[h].temp) - maxTemp = Math.max(maxTemp, day[h].temp) - totPrec += day[h].prec + minimumTemperature = Math.min(minimumTemperature, day[h]?.temperature) + maximumTemperature = Math.max(maximumTemperature, day[h]?.temperature) + minimumApparentTemperature = Math.min( + minimumApparentTemperature, + day[h]?.apparentTemperature, + ) + maximumApparentTemperature = Math.max( + maximumApparentTemperature, + day[h]?.apparentTemperature, + ) + precipitationSum += day[h]?.precipitation } return { - minTemp, - maxTemp, - totPrec, + minimumTemperature, + maximumTemperature, + minimumApparentTemperature, + maximumApparentTemperature, + precipitationSum, + weatherCode: null, } } @@ -98,6 +122,6 @@ export default async browser => { today: results[0], tomorrow: results[1], dayAfterTomorrow: results[2], - week: results.map(getDaySummary), + week: { ...results.map(getDaySummary) }, } }