weather-scraper/scrapers/iLMeteo.js

const FORMAT = {
    hourly: {
        temperature: {
            type: 'number',
            unit: '°C',
        },
        precipitation: {
            type: 'number',
            unit: 'mm',
        },
        apparentTemperature: {
            type: 'number',
            unit: '°C',
        },
    },
    daily: {
        minimumTemperature: {
            type: 'number',
            unit: '°C',
        },
        maximumTemperature: {
            type: 'number',
            unit: '°C',
        },
        minimumApparentTemperature: {
            type: 'number',
            unit: '°C',
        },
        maximumApparentTemperature: {
            type: 'number',
            unit: '°C',
        },
        precipitationSum: {
            type: 'number',
            unit: 'mm',
        },
    },
}

const parseTemp = el => {
    return parseFloat(el.textContent)
}
const parsePrec = el => {
    const text = el.textContent
    if (text === '- assenti -') return 0
    return parseFloat(text)
}

const scrapePage = async (browser, url, isToday) => {
    let result = {}

    const page = await browser.newPage()
    await page.goto(url)

    const weatherTable = await page.locator('.weather_table').waitHandle()

    const startTime = isToday ? new Date().getHours() + 1 : 0
    const endTime = 24

    // if (isToday) {
    //     try {
    //         const row = await weatherTable.$$('.latest_detection')
    //         const fields = await row[1]?.$$('td')
    //         const temperature = await fields[2].evaluate(parseTemp)
    //         const precipitation = await fields[3].evaluate(parsePrec)
    //         result[startTime - 1] = {
    //             temperature,
    //             precipitation,
    //             apparentTemperature: null,
    //             weatherCode: null,
    //         }
    //     } catch (error) {
    //         result[startTime - 1] = null
    //         // console.log(error)
    //     }
    // }
    for (let i = startTime; i < endTime; i++) {
        try {
            const selector = isToday
                ? `.forecast_1h[data-dialogid="${i}"] td`
                : `.forecast_1h[data-dialogid="${1000 + i}"] td`

            const fields = await weatherTable.$$(selector)

            const temperature = await fields[2].evaluate(parseTemp)
            const precipitation = await fields[3].evaluate(parsePrec)
            const apparentTemperature = await fields[6].evaluate(parseTemp)
            result[i] = {
                temperature,
                precipitation,
                apparentTemperature,
            }
        } catch (error) {
            result[i] = null
            // console.log(error)
        }
    }

    await page.close()

    return result
}

const rearrangeResults = results => {
    /// I don't know why but for a couple of days iLMeteo had the weather table
    /// starting at 1am instead of 0am, so I had to grab the midnight (0am)
    /// from the day before, delete it and put it at the beginning of the
    /// following day
    ///
    /// EXCEPT now it doesn't do this anymore (like it didn't use to do it
    /// before) and I have to skip this rearrangement
    ///
    /// I do NOT understand why weather websites need to change this often
    return results

    // for (let i = 0; i < 6; i++) {
    //     const midnight = results[i][24]
    //     delete results[i][24]
    //     results[i + 1][0] = midnight
    // }
    // delete results[6][24]
    //
    // return results
}

const getDaySummary = day => {
    let minimumTemperature = Number.MAX_VALUE
    let maximumTemperature = Number.MIN_VALUE
    let minimumApparentTemperature = Number.MAX_VALUE
    let maximumApparentTemperature = Number.MIN_VALUE
    let precipitationSum = 0
    for (const h in day) {
        minimumTemperature = Math.min(minimumTemperature, day[h]?.temperature)
        maximumTemperature = Math.max(maximumTemperature, day[h]?.temperature)
        minimumApparentTemperature = Math.min(
            minimumApparentTemperature,
            day[h]?.apparentTemperature,
        )
        maximumApparentTemperature = Math.max(
            maximumApparentTemperature,
            day[h]?.apparentTemperature,
        )
        precipitationSum += day[h]?.precipitation
    }
    return {
        minimumTemperature,
        maximumTemperature,
        minimumApparentTemperature,
        maximumApparentTemperature,
        precipitationSum,
    }
}

export default async browser => {
    const url = text => `https://www.ilmeteo.it/meteo/Pisa/${text}`

    const results = rearrangeResults(
        await Promise.all([
            scrapePage(browser, url(''), true),
            scrapePage(browser, url('domani'), false),
            scrapePage(browser, url('dopodomani'), false),
            scrapePage(browser, url('3'), false),
            scrapePage(browser, url('4'), false),
            scrapePage(browser, url('5'), false),
            scrapePage(browser, url('6'), false),
        ]),
    )

    return {
        format: FORMAT,
        today: results[0],
        tomorrow: results[1],
        dayAfterTomorrow: results[2],
        week: { ...results.map(getDaySummary) },
    }
}