started working on 3Bmeteo

README.md

@@ -30,6 +30,10 @@ The scraper collects data and arranges in the following scheme
                 "type": "number",
                 "unitOfMeasurement": "mm",
                 "description": "expected precipitation at the given hour"
+            },
+            "code": {
+                "type": "string",
+                "description": "weather code (sunny / cloudy / ...)"
             }
         }
     },
@@ -44,6 +48,10 @@ The scraper collects data and arranges in the following scheme
                 "type": "number",
                 "unitOfMeasurement": "mm",
                 "description": "expected precipitation at the given hour"
+            },
+            "code": {
+                "type": "string",
+                "description": "weather code (sunny / cloudy / ...)"
             }
         }
     },
@@ -58,6 +66,10 @@ The scraper collects data and arranges in the following scheme
                 "type": "number",
                 "unitOfMeasurement": "mm",
                 "description": "expected precipitation at the given hour"
+            },
+            "code": {
+                "type": "string",
+                "description": "weather code (sunny / cloudy / ...)"
             }
         }
     },
@@ -77,6 +89,10 @@ The scraper collects data and arranges in the following scheme
                 "type": "number",
                 "unitOfMeasurement": "mm",
                 "description": "total expected precipitation for the day"
+            },
+            "code": {
+                "type": "string",
+                "description": "weather code (sunny / cloudy / ...)"
             }
         }
     }
@@ -100,7 +116,7 @@ eventually, together with the current level of implementation
 ```
 
 | Source                                               | Status | Comments                                 |
-| ---------------------------------------------------- | ------ | -------- |
-| [iLMeteo](https://www.ilmeteo.it)                    | ✅     |          |
-| [3BMeteo](https://www.3bmeteo.com/)                  | ⛔️    |          |
+| ---------------------------------------------------- | ------ | ---------------------------------------- |
+| [iLMeteo](https://www.ilmeteo.it)                    | 🚧     | Weather Code not working                 |
+| [3Bmeteo](https://www.3bmeteo.com/)                  | 🚧     | Precipitation might not work as intended |
 | [Meteo Aeronautica Militare](http://www.meteoam.it/) | ⛔️    |                                          |

index.js

@@ -2,6 +2,7 @@ import 'dotenv/config'
 import puppeteer from 'puppeteer'
 
 import fetchILMeteo from './scrapers/iLMeteo.js'
+import fetch3Bmeteo from './scrapers/3Bmeteo.js'
 
 const NIX_OPS = {
     executablePath: process.env.NIX_CHROMIUM_PATH,
@@ -11,11 +12,17 @@ const opts = process.env.ON_NIX ? NIX_OPS : {}
 const run = async () => {
     const browser = await puppeteer.launch(opts)
 
-    const [iLMeteo] = await Promise.all([fetchILMeteo(browser)])
+    const [iLMeteo, treBmeteo] = await Promise.all([
+        fetchILMeteo(browser),
+        fetch3Bmeteo(browser),
+    ])
 
     await browser.close()
 
-    return { iLMeteo }
+    return {
+        iLMeteo,
+        treBmeteo,
+    }
 }
 
 console.dir(await run(), { depth: null })

scrapers/3Bmeteo.js

@@ -0,0 +1,87 @@
+const parseTemp = el => {
+    return parseFloat(el.textContent)
+}
+const parsePrec = el => {
+    const text = el.textContent
+    if (text === ' assenti\n ') return 0
+
+    //TODO not 100% sure that this would work as it was never tested
+    return parseFloat(text)
+}
+
+const scrapePage = async (browser, url, isToday) => {
+    let result = {}
+
+    const page = await browser.newPage()
+    await page.goto(url)
+
+    const tablePrevisioni = await page
+        .locator('.table-previsioni-ora')
+        .waitHandle()
+    const rows = await tablePrevisioni.$$('.row-table.noPad')
+
+    const startTime = isToday ? new Date().getHours() : 0
+    const endTime = 23
+
+    for (let i = startTime; i <= endTime; i++) {
+        try {
+            const row = rows[rows.length - 1 - (endTime - i)]
+            // console.log(await row.evaluate(el => el.children))
+            const [rowLeft, rowRight] = await row.$$(':scope > div')
+
+            const codeDiv = (await (await rowLeft.$('.row-table')).$$('div'))[2]
+            const [tempDiv, precDiv] = await (
+                await rowRight.$('.row-table')
+            ).$$('div')
+
+            const code = await codeDiv.evaluate(el => el.textContent.trim())
+            const temp = await (await tempDiv.$('span')).evaluate(parseTemp)
+            const prec = await (await precDiv.$('span')).evaluate(parsePrec)
+            result[i] = { temp, prec, code }
+        } catch (error) {
+            result[i] = null
+            console.log(error)
+        }
+    }
+
+    await page.close()
+
+    return result
+}
+
+const getDaySummary = day => {
+    let minTemp = Number.MAX_VALUE
+    let maxTemp = Number.MIN_VALUE
+    let totPrec = 0
+    for (const h in day) {
+        minTemp = Math.min(minTemp, day[h].temp)
+        maxTemp = Math.max(maxTemp, day[h].temp)
+        totPrec += day[h].prec
+    }
+    return {
+        minTemp,
+        maxTemp,
+        totPrec,
+    }
+}
+
+export default async browser => {
+    const url = text => `https://www.3bmeteo.com/meteo/pisa/${text}`
+
+    const results = await Promise.all([
+        scrapePage(browser, url(''), true),
+        scrapePage(browser, url('1'), false),
+        scrapePage(browser, url('2'), false),
+        scrapePage(browser, url('3'), false),
+        scrapePage(browser, url('4'), false),
+        scrapePage(browser, url('5'), false),
+        scrapePage(browser, url('6'), false),
+    ])
+
+    return {
+        today: results[0],
+        tomorrow: results[1],
+        dayAfterTomorrow: results[2],
+        week: results.map(getDaySummary),
+    }
+}

scrapers/iLMeteo.js

@@ -24,9 +24,9 @@ const scrapePage = async (browser, url, isToday) => {
             const fields = await row[1]?.$$('td')
             const temp = await fields[2].evaluate(parseTemp)
             const prec = await fields[3].evaluate(parsePrec)
-            result[startTime - 1] = { temp, prec }
+            result[startTime - 1] = { temp, prec, code: null }
         } catch (error) {
-            retult[startTime - 1] = null
+            result[startTime - 1] = null
             // console.log(error)
         }
     }
@@ -40,9 +40,9 @@ const scrapePage = async (browser, url, isToday) => {
 
             const temp = await fields[2].evaluate(parseTemp)
             const prec = await fields[3].evaluate(parsePrec)
-            result[i] = { temp, prec }
+            result[i] = { temp, prec, code: null }
         } catch (error) {
-            retult[i] = null
+            result[i] = null
             // console.log(error)
         }
     }