chore: minor enhancements
This commit is contained in:
@@ -1,6 +1,9 @@
|
|||||||
# Local files
|
# Local files
|
||||||
*.local*
|
*.local*
|
||||||
|
|
||||||
|
# Output files
|
||||||
|
*.json
|
||||||
|
|
||||||
# Python
|
# Python
|
||||||
venv/
|
venv/
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ $ pip install -r requirements.txt
|
|||||||
|
|
||||||
## Launch
|
## Launch
|
||||||
|
|
||||||
The following command will crawl the conferences from `https://www.dm.unipi.it/research/past-conferences/` (pages 1 to 5) and save the results in `conferences.json`:
|
The following command will crawl all the conferences from `https://www.dm.unipi.it/research/past-conferences/` (all pages) and save the results in `conferences.json` as a list of json objects, one per line.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
$ python main.py
|
$ python main.py
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ from bs4 import BeautifulSoup
|
|||||||
import textwrap
|
import textwrap
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
OUTPUT_FILE = "conferences.json"
|
||||||
|
|
||||||
LLM_EXAMPLE = (
|
LLM_EXAMPLE = (
|
||||||
"INPUT:\n"
|
"INPUT:\n"
|
||||||
'<p><a href="http://www.crm.sns.it/event/507/" target="_blank" rel="noreferrer noopener">Statistical'
|
'<p><a href="http://www.crm.sns.it/event/507/" target="_blank" rel="noreferrer noopener">Statistical'
|
||||||
@@ -99,10 +101,10 @@ llm = Llama(model_path="./mistral-7b-instruct-v0.2.Q4_K_M.gguf", chat_format="ll
|
|||||||
|
|
||||||
|
|
||||||
# clear the result file
|
# clear the result file
|
||||||
open("conferences.json", "w").close()
|
open(OUTPUT_FILE, "w").close()
|
||||||
|
|
||||||
# the result file is a sequence of json objects, one per line
|
# the result file is a sequence of json objects, one per line
|
||||||
results_file = open("conferences.json", "a")
|
results_file = open(OUTPUT_FILE, "a")
|
||||||
|
|
||||||
for conference_html in conference_html_snippets:
|
for conference_html in conference_html_snippets:
|
||||||
print("Translating:")
|
print("Translating:")
|
||||||
|
|||||||
Reference in New Issue
Block a user