53 lines
2 KiB
Python
53 lines
2 KiB
Python
# Задание 5. Финал: мини-скрапер с aiofiles
|
||
# Объедини всё: стяни 10 постов с jsonplaceholder,
|
||
# ограничь конкурентность семафором,
|
||
# и асинхронно сохрани каждый в отдельный JSON-файл через aiofiles.
|
||
# Бонус: добавь обработку ошибок так, чтобы падение одного запроса не роняло остальные (подумай, gather или TaskGroup тут уместнее и почему).
|
||
|
||
|
||
import asyncio
|
||
import aiofiles
|
||
from datetime import datetime
|
||
import random
|
||
import httpx
|
||
import json
|
||
|
||
|
||
async def fetch(url: str, client: httpx.AsyncClient, sem: asyncio.Semaphore) -> dict:
|
||
async with sem:
|
||
try:
|
||
print(f"Fetching of {url} started at {datetime.today()}")
|
||
r = await client.get(url)
|
||
data = r.json()
|
||
print(f"Fetching of {url} ended at {datetime.today()}")
|
||
return {f"Post_{url.split("/")[-1]}.json": data}
|
||
except httpx.HTTPStatusError as e:
|
||
print(f"HTTP error: {e}")
|
||
except httpx.RequestError as e:
|
||
print(f"Request failed: {e}")
|
||
|
||
|
||
async def save_file(file_name: str, data: str):
|
||
try:
|
||
async with aiofiles.open(file_name, "w") as file:
|
||
await file.write(json.dumps(data))
|
||
print(f"Written {file_name}")
|
||
except IOError:
|
||
print(f"Can't write file {file_name}...")
|
||
|
||
|
||
async def main():
|
||
urls = [f"https://jsonplaceholder.typicode.com/posts/{n}" for n in range(1, 11)]
|
||
semaphor = asyncio.Semaphore(2)
|
||
|
||
async with httpx.AsyncClient(timeout=10) as client:
|
||
results = await asyncio.gather(
|
||
*[fetch(url, client, semaphor) for url in urls], return_exceptions=True
|
||
)
|
||
|
||
for result in results:
|
||
for file_name, json in result.items():
|
||
await save_file(file_name, json)
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|