112 lines
3.5 KiB
Python
112 lines
3.5 KiB
Python
"""Base class for HTTP-based HTML/content parsing."""
|
|
|
|
import logging
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pydantic
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class HttpParserConfig(pydantic.BaseModel):
|
|
"""Configuration for HTTP parser."""
|
|
|
|
TIMEOUT: int = 10
|
|
USER_AGENT: str | None = None
|
|
FOLLOW_REDIRECTS: bool = True
|
|
|
|
|
|
class HttpParserBase:
|
|
"""Base class for HTTP requests and HTML parsing.
|
|
|
|
Provides reusable HTTP client functionality with error handling.
|
|
Subclasses implement domain-specific parsing logic.
|
|
"""
|
|
|
|
def __init__(self, config: HttpParserConfig) -> None:
|
|
self.config = config
|
|
|
|
def get_headers(self, **extra_headers: str) -> dict[str, str]:
|
|
"""Build HTTP headers with default User-Agent.
|
|
|
|
Args:
|
|
**extra_headers: Additional headers to merge
|
|
|
|
Returns:
|
|
Dictionary of HTTP headers
|
|
"""
|
|
headers = {
|
|
'User-Agent': self.config.USER_AGENT or 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
}
|
|
headers.update(extra_headers)
|
|
return headers
|
|
|
|
async def fetch_html(
|
|
self,
|
|
url: str,
|
|
*,
|
|
headers: dict[str, str] | None = None,
|
|
follow_redirects: bool | None = None,
|
|
timeout: int | None = None,
|
|
) -> tuple[int, str]:
|
|
"""Fetch HTML content from URL.
|
|
|
|
Args:
|
|
url: Target URL to fetch
|
|
headers: Optional custom headers (uses get_headers() if None)
|
|
follow_redirects: Whether to follow HTTP redirects (uses config default if None)
|
|
timeout: Optional custom timeout (uses config default if None)
|
|
|
|
Returns:
|
|
Tuple of (status_code, html_content)
|
|
|
|
Raises:
|
|
httpx.TimeoutException: Request timed out
|
|
httpx.RequestError: Network error occurred
|
|
"""
|
|
if headers is None:
|
|
headers = self.get_headers()
|
|
|
|
async with httpx.AsyncClient(
|
|
timeout=timeout or self.config.TIMEOUT,
|
|
follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
|
|
) as client:
|
|
response = await client.get(url, headers=headers)
|
|
return response.status_code, response.text
|
|
|
|
async def fetch_json(
|
|
self,
|
|
url: str,
|
|
*,
|
|
headers: dict[str, str] | None = None,
|
|
follow_redirects: bool | None = None,
|
|
timeout: int | None = None,
|
|
) -> tuple[int, Any]:
|
|
"""Fetch JSON content from URL.
|
|
|
|
Args:
|
|
url: Target URL to fetch
|
|
headers: Optional custom headers (uses get_headers() if None)
|
|
follow_redirects: Whether to follow HTTP redirects (uses config default if None)
|
|
timeout: Optional custom timeout (uses config default if None)
|
|
|
|
Returns:
|
|
Tuple of (status_code, json_data)
|
|
|
|
Raises:
|
|
httpx.TimeoutException: Request timed out
|
|
httpx.RequestError: Network error occurred
|
|
"""
|
|
if headers is None:
|
|
headers = self.get_headers()
|
|
|
|
async with httpx.AsyncClient(
|
|
timeout=timeout or self.config.TIMEOUT,
|
|
follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
|
|
) as client:
|
|
response = await client.get(url, headers=headers)
|
|
return response.status_code, response.json()
|