feat: fetch views manually and worker

shared/http_parser_base.py

@@ -0,0 +1,111 @@
+"""Base class for HTTP-based HTML/content parsing."""
+
+import logging
+from typing import Any
+
+import httpx
+import pydantic
+
+log = logging.getLogger(__name__)
+
+
+class HttpParserConfig(pydantic.BaseModel):
+    """Configuration for HTTP parser."""
+
+    TIMEOUT: int = 10
+    USER_AGENT: str | None = None
+    FOLLOW_REDIRECTS: bool = True
+
+
+class HttpParserBase:
+    """Base class for HTTP requests and HTML parsing.
+
+    Provides reusable HTTP client functionality with error handling.
+    Subclasses implement domain-specific parsing logic.
+    """
+
+    def __init__(self, config: HttpParserConfig) -> None:
+        self.config = config
+
+    def get_headers(self, **extra_headers: str) -> dict[str, str]:
+        """Build HTTP headers with default User-Agent.
+
+        Args:
+            **extra_headers: Additional headers to merge
+
+        Returns:
+            Dictionary of HTTP headers
+        """
+        headers = {
+            'User-Agent': self.config.USER_AGENT or 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+        }
+        headers.update(extra_headers)
+        return headers
+
+    async def fetch_html(
+        self,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        follow_redirects: bool | None = None,
+        timeout: int | None = None,
+    ) -> tuple[int, str]:
+        """Fetch HTML content from URL.
+
+        Args:
+            url: Target URL to fetch
+            headers: Optional custom headers (uses get_headers() if None)
+            follow_redirects: Whether to follow HTTP redirects (uses config default if None)
+            timeout: Optional custom timeout (uses config default if None)
+
+        Returns:
+            Tuple of (status_code, html_content)
+
+        Raises:
+            httpx.TimeoutException: Request timed out
+            httpx.RequestError: Network error occurred
+        """
+        if headers is None:
+            headers = self.get_headers()
+
+        async with httpx.AsyncClient(
+            timeout=timeout or self.config.TIMEOUT,
+            follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
+        ) as client:
+            response = await client.get(url, headers=headers)
+            return response.status_code, response.text
+
+    async def fetch_json(
+        self,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        follow_redirects: bool | None = None,
+        timeout: int | None = None,
+    ) -> tuple[int, Any]:
+        """Fetch JSON content from URL.
+
+        Args:
+            url: Target URL to fetch
+            headers: Optional custom headers (uses get_headers() if None)
+            follow_redirects: Whether to follow HTTP redirects (uses config default if None)
+            timeout: Optional custom timeout (uses config default if None)
+
+        Returns:
+            Tuple of (status_code, json_data)
+
+        Raises:
+            httpx.TimeoutException: Request timed out
+            httpx.RequestError: Network error occurred
+        """
+        if headers is None:
+            headers = self.get_headers()
+
+        async with httpx.AsyncClient(
+            timeout=timeout or self.config.TIMEOUT,
+            follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
+        ) as client:
+            response = await client.get(url, headers=headers)
+            return response.status_code, response.json()

shared/worker_base.py

@@ -0,0 +1,65 @@
+import asyncio
+import logging
+
+import pydantic
+
+log = logging.getLogger(__name__)
+
+
+class WorkerConfig(pydantic.BaseModel):
+    INTERVAL_SECONDS: int = 60
+
+
+class WorkerBase:
+    def __init__(self, config: WorkerConfig) -> None:
+        self.config = config
+        self._task: asyncio.Task[None] | None = None
+        self._stop_event = asyncio.Event()
+
+        log.info('Worker initialized (interval=%ds)', config.INTERVAL_SECONDS)
+
+    async def _cycle_func(self) -> None:
+        raise NotImplementedError
+
+    async def start(self) -> None:
+        if self._task is not None:
+            log.warning('Worker already running')
+            return
+
+        self._stop_event.clear()
+        self._task = asyncio.create_task(self._run())
+        log.info('Worker started')
+
+    async def stop(self) -> None:
+        if self._task is None:
+            log.warning('Worker not running')
+            return
+
+        self._stop_event.set()
+
+        try:
+            await asyncio.wait_for(self._task, timeout=5.0)
+        except TimeoutError:
+            log.warning('Worker did not stop in time, cancelling')
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+        except asyncio.CancelledError:
+            log.info('Worker task cancelled')
+
+        self._task = None
+        log.info('Worker stopped')
+
+    async def _run(self) -> None:
+        while not self._stop_event.is_set():
+            try:
+                await self._cycle_func()
+            except Exception:
+                log.exception('Error in worker cycle')
+
+            try:
+                await asyncio.wait_for(self._stop_event.wait(), timeout=self.config.INTERVAL_SECONDS)
+            except TimeoutError:
+                pass  # Время вышло, продолжаем