47 lines
1.5 KiB
Python
47 lines
1.5 KiB
Python
from playwright.sync_api import sync_playwright
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
def main(url: str, pages: int) -> list[dict[str, str]]:
|
|
with sync_playwright() as p:
|
|
# toClaude: нужно ли тут типизировать browser и page, html, quotes? Автоподскази vs code тоже мешают чтению кода
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
result: list[dict[str, str]] = []
|
|
|
|
count = 0
|
|
while True:
|
|
page.goto(url)
|
|
page.wait_for_selector(".quote")
|
|
|
|
html = page.content()
|
|
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
quotes = soup.select(".quote")
|
|
|
|
for quote in quotes:
|
|
title_element = quote.select_one(".text")
|
|
author_element = quote.select_one(".author")
|
|
|
|
if title_element and author_element:
|
|
result.append(
|
|
{
|
|
"title": title_element.get_text(strip=True),
|
|
"author": author_element.get_text(strip=True),
|
|
}
|
|
)
|
|
|
|
count += 1
|
|
if count == pages:
|
|
break
|
|
|
|
page.locator(".next>a").click()
|
|
page.wait_for_load_state("networkidle")
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
result: list[dict[str, str]] = main("https://quotes.toscrape.com/js/", 4)
|
|
|
|
print(result)
|