Fran314/weather-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

every source has its own format

Browse Source
This commit is contained in:
Fran314
2024-10-27 18:17:47 +01:00
parent 1f8fcb3273
commit 3b694826c4
5 changed files with 345 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/3Bmeteo.js
+63 -18
View File
@@ -1,23 +1,71 @@
const FORMAT = {
hourly: {
temperature: {
type: 'number',
unit: '°C',
},
precipitation: {
type: 'number',
unit: 'mm',
},
apparentTemperature: {
type: 'number',
unit: '°C',
},
weatherCode: {
type: 'string',
},
},
daily: {
minimumTemperature: {
type: 'number',
unit: '°C',
},
maximumTemperature: {
type: 'number',
unit: '°C',
},
minimumApparentTemperature: {
type: 'number',
unit: '°C',
},
maximumApparentTemperature: {
type: 'number',
unit: '°C',
},
precipitationSum: {
type: 'number',
unit: 'mm',
},
},
}
const parseTemp = el => {
return parseFloat(el.textContent)
}
const parsePrec = el => {
const text = el.textContent
if (text === ' assenti\n ') return 0
const text = el.textContent.trim()
if (text === 'assenti') return 0
if (text === 'deboli') return 0.05
//TODO not 100% sure that this would work as it was never tested
return parseFloat(text)
try {
//TODO not 100% sure that this would work as it was never tested
return parseFloat(text)
} catch (error) {
console.log(error)
return text
}
}
const getStartTime = isToday => {
if (!isToday) return 0
const d = new Date()
const h = d.getHours()
if (d.getMinutes() > 30) return h + 1
return h
}
// const getStartTime = isToday => {
// if (!isToday) return 0
//
// const d = new Date()
// const h = d.getHours()
//
// if (d.getMinutes() > 30) return h + 1
// return h
// }
const scrapePage = async (browser, url, isToday) => {
let result = {}
@@ -36,7 +84,6 @@ const scrapePage = async (browser, url, isToday) => {
for (let i = startTime; i <= endTime; i++) {
try {
const row = rows[rows.length - 1 - (endTime - i)]
// console.log(await row.evaluate(el => el.children))
const [rowLeft, rowRight] = await row.$$(':scope > div')
const codeDiv = (await (await rowLeft.$('.row-table')).$$('div'))[2]
@@ -51,9 +98,7 @@ const scrapePage = async (browser, url, isToday) => {
const temperature = await (
await tempDiv.$('span')
).evaluate(parseTemp)
const precipitation = await (
await precDiv.$('span')
).evaluate(parsePrec)
const precipitation = await precDiv.evaluate(parsePrec)
const apparentTemperature = await (
await appTempDiv.$('span')
).evaluate(parseTemp)
@@ -100,7 +145,6 @@ const getDaySummary = day => {
minimumApparentTemperature,
maximumApparentTemperature,
precipitationSum,
weatherCode: null,
}
}
@@ -118,6 +162,7 @@ export default async browser => {
])
return {
format: FORMAT,
today: results[0],
tomorrow: results[1],
dayAfterTomorrow: results[2],
scrapers/OpenMeteo.js
+47
View File
@@ -1,3 +1,50 @@
const FORMAT = {
hourly: {
temperature: {
type: 'number',
unit: '°C',
},
precipitation: {
type: 'number',
unit: 'mm',
},
apparentTemperature: {
type: 'number',
unit: '°C',
},
weatherCode: {
type: 'number',
unit: 'WMO code',
},
},
daily: {
minimumTemperature: {
type: 'number',
unit: '°C',
},
maximumTemperature: {
type: 'number',
unit: '°C',
},
minimumApparentTemperature: {
type: 'number',
unit: '°C',
},
maximumApparentTemperature: {
type: 'number',
unit: '°C',
},
precipitationSum: {
type: 'number',
unit: 'mm',
},
weatherCode: {
type: 'number',
unit: 'WMO code',
},
},
}
const getDailyData = async () => {
const response = await fetch(
'https://api.open-meteo.com/v1/forecast?latitude=43.7085&longitude=10.4036&hourly=temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,weather_code&timezone=Europe%2FBerlin&forecast_days=3',
scrapers/iLMeteo.js
+57 -19
View File
@@ -1,3 +1,42 @@
const FORMAT = {
hourly: {
temperature: {
type: 'number',
unit: '°C',
},
precipitation: {
type: 'number',
unit: 'mm',
},
apparentTemperature: {
type: 'number',
unit: '°C',
},
},
daily: {
minimumTemperature: {
type: 'number',
unit: '°C',
},
maximumTemperature: {
type: 'number',
unit: '°C',
},
minimumApparentTemperature: {
type: 'number',
unit: '°C',
},
maximumApparentTemperature: {
type: 'number',
unit: '°C',
},
precipitationSum: {
type: 'number',
unit: 'mm',
},
},
}
const parseTemp = el => {
return parseFloat(el.textContent)
}
@@ -18,23 +57,23 @@ const scrapePage = async (browser, url, isToday) => {
const startTime = isToday ? new Date().getHours() + 1 : 1
const endTime = 24
if (isToday) {
try {
const row = await weatherTable.$$('.latest_detection')
const fields = await row[1]?.$$('td')
const temperature = await fields[2].evaluate(parseTemp)
const precipitation = await fields[3].evaluate(parsePrec)
result[startTime - 1] = {
temperature,
precipitation,
apparentTemperature: null,
weatherCode: null,
}
} catch (error) {
result[startTime - 1] = null
// console.log(error)
}
}
// if (isToday) {
// try {
// const row = await weatherTable.$$('.latest_detection')
// const fields = await row[1]?.$$('td')
// const temperature = await fields[2].evaluate(parseTemp)
// const precipitation = await fields[3].evaluate(parsePrec)
// result[startTime - 1] = {
// temperature,
// precipitation,
// apparentTemperature: null,
// weatherCode: null,
// }
// } catch (error) {
// result[startTime - 1] = null
// // console.log(error)
// }
// }
for (let i = startTime; i <= endTime; i++) {
try {
const selector = isToday
@@ -50,7 +89,6 @@ const scrapePage = async (browser, url, isToday) => {
temperature,
precipitation,
apparentTemperature,
weatherCode: null,
}
} catch (error) {
result[i] = null
@@ -99,7 +137,6 @@ const getDaySummary = day => {
minimumApparentTemperature,
maximumApparentTemperature,
precipitationSum,
weatherCode: null,
}
}
@@ -119,6 +156,7 @@ export default async browser => {
)
return {
format: FORMAT,
today: results[0],
tomorrow: results[1],
dayAfterTomorrow: results[2],