45 lines
1.3 KiB
Python
45 lines
1.3 KiB
Python
import httpx
|
|
import logging
|
|
from bs4 import BeautifulSoup
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s"
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def main(page: str) -> list[dict[str, str]]:
|
|
try:
|
|
response = httpx.get(page)
|
|
response.raise_for_status()
|
|
html = response.text
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
articles = soup.find_all("article")
|
|
|
|
result: list[dict[str, str]] = []
|
|
|
|
for article in articles:
|
|
title_element = article.select_one("h2")
|
|
link_element = article.select_one("a")
|
|
|
|
if title_element and link_element:
|
|
result.append(
|
|
{
|
|
"title": title_element.get_text(strip=True),
|
|
"link": link_element.get("href"), # type: ignore
|
|
}
|
|
)
|
|
else:
|
|
logger.warning("No h2 or a element found, skipping the article")
|
|
|
|
return result
|
|
|
|
except httpx.HTTPStatusError as e:
|
|
if e.response.status_code == 404:
|
|
logger.warning("Page not found")
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Something went wrong. Check the message: {e}")
|
|
return []
|