diff --git a/README.md b/README.md index b34efca..5beb934 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ The scraper collects data and arranges in the following scheme "type": "number", "unitOfMeasurement": "mm", "description": "expected precipitation at the given hour" + }, + "code": { + "type": "string", + "description": "weather code (sunny / cloudy / ...)" } } }, @@ -44,6 +48,10 @@ The scraper collects data and arranges in the following scheme "type": "number", "unitOfMeasurement": "mm", "description": "expected precipitation at the given hour" + }, + "code": { + "type": "string", + "description": "weather code (sunny / cloudy / ...)" } } }, @@ -58,6 +66,10 @@ The scraper collects data and arranges in the following scheme "type": "number", "unitOfMeasurement": "mm", "description": "expected precipitation at the given hour" + }, + "code": { + "type": "string", + "description": "weather code (sunny / cloudy / ...)" } } }, @@ -77,6 +89,10 @@ The scraper collects data and arranges in the following scheme "type": "number", "unitOfMeasurement": "mm", "description": "total expected precipitation for the day" + }, + "code": { + "type": "string", + "description": "weather code (sunny / cloudy / ...)" } } } @@ -99,8 +115,8 @@ eventually, together with the current level of implementation ⛔️ = Not implemented ``` -| Source | Status | Comments | -| ---------------------------------------------------- | ------ | -------- | -| [iLMeteo](https://www.ilmeteo.it) | ✅ | | -| [3BMeteo](https://www.3bmeteo.com/) | ⛔️ | | -| [Meteo Aeronautica Militare](http://www.meteoam.it/) | ⛔️ | | +| Source | Status | Comments | +| ---------------------------------------------------- | ------ | ---------------------------------------- | +| [iLMeteo](https://www.ilmeteo.it) | 🚧 | Weather Code not working | +| [3Bmeteo](https://www.3bmeteo.com/) | 🚧 | Precipitation might not work as intended | +| [Meteo Aeronautica Militare](http://www.meteoam.it/) | ⛔️ | | diff --git a/index.js b/index.js index a8dea31..baa1d19 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ import 'dotenv/config' import puppeteer from 'puppeteer' import fetchILMeteo from './scrapers/iLMeteo.js' +import fetch3Bmeteo from './scrapers/3Bmeteo.js' const NIX_OPS = { executablePath: process.env.NIX_CHROMIUM_PATH, @@ -11,11 +12,17 @@ const opts = process.env.ON_NIX ? NIX_OPS : {} const run = async () => { const browser = await puppeteer.launch(opts) - const [iLMeteo] = await Promise.all([fetchILMeteo(browser)]) + const [iLMeteo, treBmeteo] = await Promise.all([ + fetchILMeteo(browser), + fetch3Bmeteo(browser), + ]) await browser.close() - return { iLMeteo } + return { + iLMeteo, + treBmeteo, + } } console.dir(await run(), { depth: null }) diff --git a/scrapers/3Bmeteo.js b/scrapers/3Bmeteo.js new file mode 100644 index 0000000..eace321 --- /dev/null +++ b/scrapers/3Bmeteo.js @@ -0,0 +1,87 @@ +const parseTemp = el => { + return parseFloat(el.textContent) +} +const parsePrec = el => { + const text = el.textContent + if (text === ' assenti\n ') return 0 + + //TODO not 100% sure that this would work as it was never tested + return parseFloat(text) +} + +const scrapePage = async (browser, url, isToday) => { + let result = {} + + const page = await browser.newPage() + await page.goto(url) + + const tablePrevisioni = await page + .locator('.table-previsioni-ora') + .waitHandle() + const rows = await tablePrevisioni.$$('.row-table.noPad') + + const startTime = isToday ? new Date().getHours() : 0 + const endTime = 23 + + for (let i = startTime; i <= endTime; i++) { + try { + const row = rows[rows.length - 1 - (endTime - i)] + // console.log(await row.evaluate(el => el.children)) + const [rowLeft, rowRight] = await row.$$(':scope > div') + + const codeDiv = (await (await rowLeft.$('.row-table')).$$('div'))[2] + const [tempDiv, precDiv] = await ( + await rowRight.$('.row-table') + ).$$('div') + + const code = await codeDiv.evaluate(el => el.textContent.trim()) + const temp = await (await tempDiv.$('span')).evaluate(parseTemp) + const prec = await (await precDiv.$('span')).evaluate(parsePrec) + result[i] = { temp, prec, code } + } catch (error) { + result[i] = null + console.log(error) + } + } + + await page.close() + + return result +} + +const getDaySummary = day => { + let minTemp = Number.MAX_VALUE + let maxTemp = Number.MIN_VALUE + let totPrec = 0 + for (const h in day) { + minTemp = Math.min(minTemp, day[h].temp) + maxTemp = Math.max(maxTemp, day[h].temp) + totPrec += day[h].prec + } + return { + minTemp, + maxTemp, + totPrec, + } +} + +export default async browser => { + const url = text => `https://www.3bmeteo.com/meteo/pisa/${text}` + + const results = await Promise.all([ + scrapePage(browser, url(''), true), + scrapePage(browser, url('1'), false), + scrapePage(browser, url('2'), false), + scrapePage(browser, url('3'), false), + scrapePage(browser, url('4'), false), + scrapePage(browser, url('5'), false), + scrapePage(browser, url('6'), false), + ]) + + return { + today: results[0], + tomorrow: results[1], + dayAfterTomorrow: results[2], + week: results.map(getDaySummary), + } +} diff --git a/scrapers/iLMeteo.js b/scrapers/iLMeteo.js index b1a7ab4..ce9518c 100644 --- a/scrapers/iLMeteo.js +++ b/scrapers/iLMeteo.js @@ -24,9 +24,9 @@ const scrapePage = async (browser, url, isToday) => { const fields = await row[1]?.$$('td') const temp = await fields[2].evaluate(parseTemp) const prec = await fields[3].evaluate(parsePrec) - result[startTime - 1] = { temp, prec } + result[startTime - 1] = { temp, prec, code: null } } catch (error) { - retult[startTime - 1] = null + result[startTime - 1] = null // console.log(error) } } @@ -40,9 +40,9 @@ const scrapePage = async (browser, url, isToday) => { const temp = await fields[2].evaluate(parseTemp) const prec = await fields[3].evaluate(parsePrec) - result[i] = { temp, prec } + result[i] = { temp, prec, code: null } } catch (error) { - retult[i] = null + result[i] = null // console.log(error) } }