75 lines
2.1 KiB
Python
75 lines
2.1 KiB
Python
from pathlib import Path
|
|
import logging
|
|
from datetime import datetime
|
|
import schedule
|
|
from playwright.sync_api import sync_playwright
|
|
from bs4 import BeautifulSoup
|
|
import json
|
|
from typing import Any
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(message)s",
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def job() -> None:
|
|
try:
|
|
logger.info("Parsing started")
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
result: list[dict[str, Any]] = []
|
|
|
|
page.goto("https://quotes.toscrape.com/js/")
|
|
page.wait_for_selector(".quote")
|
|
|
|
html = page.content()
|
|
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
quotes = soup.select(".quote")
|
|
|
|
for quote in quotes:
|
|
quote_text = quote.select_one(".text")
|
|
quote_author = quote.select_one(".author")
|
|
|
|
if quote_text and quote_author:
|
|
result.append(
|
|
{
|
|
"title": quote_text.get_text(strip=True),
|
|
"author": quote_author.get_text(strip=True),
|
|
}
|
|
)
|
|
else:
|
|
logger.warning("Quote without text or author, skipping")
|
|
|
|
logger.info("Parsing ended")
|
|
logger.info("Writing to the file...")
|
|
|
|
Path("./data/").mkdir(exist_ok=True)
|
|
filename = "./data/dump-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.json"
|
|
with open(
|
|
filename,
|
|
"w",
|
|
encoding="utf-8",
|
|
) as file:
|
|
json.dump(result, file, indent=2, ensure_ascii=False)
|
|
|
|
browser.close()
|
|
logger.info(f"Job is done. Parsed {len(result)} quotes")
|
|
except Exception as e:
|
|
logger.error(f"Job failed: {e}", exc_info=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logger.info("Program started")
|
|
try:
|
|
schedule.every(20).seconds.do(job)
|
|
|
|
while True:
|
|
schedule.run_pending()
|
|
except KeyboardInterrupt:
|
|
logger.info("Program stopped by user")
|