feat: fetch views manually and worker

shared/http_parser_base.py

@@ -0,0 +1,111 @@
+"""Base class for HTTP-based HTML/content parsing."""
+
+import logging
+from typing import Any
+
+import httpx
+import pydantic
+
+log = logging.getLogger(__name__)
+
+
+class HttpParserConfig(pydantic.BaseModel):
+    """Configuration for HTTP parser."""
+
+    TIMEOUT: int = 10
+    USER_AGENT: str | None = None
+    FOLLOW_REDIRECTS: bool = True
+
+
+class HttpParserBase:
+    """Base class for HTTP requests and HTML parsing.
+
+    Provides reusable HTTP client functionality with error handling.
+    Subclasses implement domain-specific parsing logic.
+    """
+
+    def __init__(self, config: HttpParserConfig) -> None:
+        self.config = config
+
+    def get_headers(self, **extra_headers: str) -> dict[str, str]:
+        """Build HTTP headers with default User-Agent.
+
+        Args:
+            **extra_headers: Additional headers to merge
+
+        Returns:
+            Dictionary of HTTP headers
+        """
+        headers = {
+            'User-Agent': self.config.USER_AGENT or 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+        }
+        headers.update(extra_headers)
+        return headers
+
+    async def fetch_html(
+        self,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        follow_redirects: bool | None = None,
+        timeout: int | None = None,
+    ) -> tuple[int, str]:
+        """Fetch HTML content from URL.
+
+        Args:
+            url: Target URL to fetch
+            headers: Optional custom headers (uses get_headers() if None)
+            follow_redirects: Whether to follow HTTP redirects (uses config default if None)
+            timeout: Optional custom timeout (uses config default if None)
+
+        Returns:
+            Tuple of (status_code, html_content)
+
+        Raises:
+            httpx.TimeoutException: Request timed out
+            httpx.RequestError: Network error occurred
+        """
+        if headers is None:
+            headers = self.get_headers()
+
+        async with httpx.AsyncClient(
+            timeout=timeout or self.config.TIMEOUT,
+            follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
+        ) as client:
+            response = await client.get(url, headers=headers)
+            return response.status_code, response.text
+
+    async def fetch_json(
+        self,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        follow_redirects: bool | None = None,
+        timeout: int | None = None,
+    ) -> tuple[int, Any]:
+        """Fetch JSON content from URL.
+
+        Args:
+            url: Target URL to fetch
+            headers: Optional custom headers (uses get_headers() if None)
+            follow_redirects: Whether to follow HTTP redirects (uses config default if None)
+            timeout: Optional custom timeout (uses config default if None)
+
+        Returns:
+            Tuple of (status_code, json_data)
+
+        Raises:
+            httpx.TimeoutException: Request timed out
+            httpx.RequestError: Network error occurred
+        """
+        if headers is None:
+            headers = self.get_headers()
+
+        async with httpx.AsyncClient(
+            timeout=timeout or self.config.TIMEOUT,
+            follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
+        ) as client:
+            response = await client.get(url, headers=headers)
+            return response.status_code, response.json()