feat: parser, refactoring
This commit is contained in:
Artem Tsyrulnikov
@@ -1,111 +0,0 @@
|
||||
"""Base class for HTTP-based HTML/content parsing."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pydantic
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HttpParserConfig(pydantic.BaseModel):
|
||||
"""Configuration for HTTP parser."""
|
||||
|
||||
TIMEOUT: int = 10
|
||||
USER_AGENT: str | None = None
|
||||
FOLLOW_REDIRECTS: bool = True
|
||||
|
||||
|
||||
class HttpParserBase:
|
||||
"""Base class for HTTP requests and HTML parsing.
|
||||
|
||||
Provides reusable HTTP client functionality with error handling.
|
||||
Subclasses implement domain-specific parsing logic.
|
||||
"""
|
||||
|
||||
def __init__(self, config: HttpParserConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
def get_headers(self, **extra_headers: str) -> dict[str, str]:
|
||||
"""Build HTTP headers with default User-Agent.
|
||||
|
||||
Args:
|
||||
**extra_headers: Additional headers to merge
|
||||
|
||||
Returns:
|
||||
Dictionary of HTTP headers
|
||||
"""
|
||||
headers = {
|
||||
'User-Agent': self.config.USER_AGENT or 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
}
|
||||
headers.update(extra_headers)
|
||||
return headers
|
||||
|
||||
async def fetch_html(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
follow_redirects: bool | None = None,
|
||||
timeout: int | None = None,
|
||||
) -> tuple[int, str]:
|
||||
"""Fetch HTML content from URL.
|
||||
|
||||
Args:
|
||||
url: Target URL to fetch
|
||||
headers: Optional custom headers (uses get_headers() if None)
|
||||
follow_redirects: Whether to follow HTTP redirects (uses config default if None)
|
||||
timeout: Optional custom timeout (uses config default if None)
|
||||
|
||||
Returns:
|
||||
Tuple of (status_code, html_content)
|
||||
|
||||
Raises:
|
||||
httpx.TimeoutException: Request timed out
|
||||
httpx.RequestError: Network error occurred
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.get_headers()
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout or self.config.TIMEOUT,
|
||||
follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
|
||||
) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
return response.status_code, response.text
|
||||
|
||||
async def fetch_json(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
follow_redirects: bool | None = None,
|
||||
timeout: int | None = None,
|
||||
) -> tuple[int, Any]:
|
||||
"""Fetch JSON content from URL.
|
||||
|
||||
Args:
|
||||
url: Target URL to fetch
|
||||
headers: Optional custom headers (uses get_headers() if None)
|
||||
follow_redirects: Whether to follow HTTP redirects (uses config default if None)
|
||||
timeout: Optional custom timeout (uses config default if None)
|
||||
|
||||
Returns:
|
||||
Tuple of (status_code, json_data)
|
||||
|
||||
Raises:
|
||||
httpx.TimeoutException: Request timed out
|
||||
httpx.RequestError: Network error occurred
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.get_headers()
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout or self.config.TIMEOUT,
|
||||
follow_redirects=follow_redirects if follow_redirects is not None else self.config.FOLLOW_REDIRECTS,
|
||||
) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
return response.status_code, response.json()
|
||||
112
shared/scripts/check_env.py
Executable file
112
shared/scripts/check_env.py
Executable file
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_env_file(file_path: Path) -> set[str]:
|
||||
if not file_path.exists():
|
||||
return set()
|
||||
|
||||
variables = set()
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
variables.add(line.split('=', 1)[0].strip())
|
||||
return variables
|
||||
|
||||
|
||||
def get_pydantic_settings_fields() -> tuple[set[str], set[str]]:
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
from pydantic import BaseModel
|
||||
|
||||
from src.config import Settings
|
||||
|
||||
all_fields, required_fields = set(), set()
|
||||
|
||||
def extract_fields(model: type[BaseModel], prefix: str = '') -> None:
|
||||
for field_name, field_info in model.model_fields.items():
|
||||
env_var_name = f'{prefix}{field_name}'.upper()
|
||||
field_type = field_info.annotation
|
||||
if hasattr(field_type, '__origin__'):
|
||||
field_type = field_type.__args__[0] if field_type.__args__ else field_type
|
||||
|
||||
try:
|
||||
if isinstance(field_type, type) and issubclass(field_type, BaseModel):
|
||||
extract_fields(field_type, prefix=f'{env_var_name}__')
|
||||
else:
|
||||
all_fields.add(env_var_name)
|
||||
if field_info.is_required():
|
||||
required_fields.add(env_var_name)
|
||||
except TypeError:
|
||||
all_fields.add(env_var_name)
|
||||
if field_info.is_required():
|
||||
required_fields.add(env_var_name)
|
||||
|
||||
extract_fields(Settings)
|
||||
return all_fields, required_fields
|
||||
|
||||
|
||||
def main() -> int:
|
||||
project_root = Path(__file__).parent.parent.parent
|
||||
env_file = project_root / '.env'
|
||||
env_example_file = project_root / '.env.example'
|
||||
|
||||
if not env_example_file.exists():
|
||||
print('❌ Файл .env.example не найден')
|
||||
return 1
|
||||
|
||||
env_vars = parse_env_file(env_file)
|
||||
env_example_vars = parse_env_file(env_example_file)
|
||||
all_pydantic_fields, required_pydantic_fields = get_pydantic_settings_fields()
|
||||
|
||||
print('🔍 Проверка соответствия .env и .env.example\n')
|
||||
has_errors = False
|
||||
|
||||
missing_in_env = env_example_vars - env_vars
|
||||
if missing_in_env:
|
||||
has_errors = True
|
||||
print('❌ Переменные из .env.example отсутствуют в .env:')
|
||||
for var in sorted(missing_in_env):
|
||||
print(f' - {var}')
|
||||
print()
|
||||
|
||||
extra_in_env = env_vars - env_example_vars
|
||||
if extra_in_env:
|
||||
has_errors = True
|
||||
print('⚠️ Переменные в .env, которых нет в .env.example:')
|
||||
for var in sorted(extra_in_env):
|
||||
print(f' - {var}')
|
||||
print()
|
||||
|
||||
missing_in_pydantic = env_example_vars - all_pydantic_fields
|
||||
if missing_in_pydantic:
|
||||
has_errors = True
|
||||
print('❌ Переменные из .env.example не определены в Pydantic Settings (src/config.py):')
|
||||
for var in sorted(missing_in_pydantic):
|
||||
print(f' - {var}')
|
||||
print()
|
||||
|
||||
missing_in_example = required_pydantic_fields - env_example_vars
|
||||
if missing_in_example:
|
||||
has_errors = True
|
||||
print('❌ Обязательные поля Pydantic Settings отсутствуют в .env.example:')
|
||||
for var in sorted(missing_in_example):
|
||||
print(f' - {var}')
|
||||
print()
|
||||
|
||||
if not has_errors:
|
||||
print('✅ Все проверки пройдены успешно!')
|
||||
print(f' - Переменных в .env.example: {len(env_example_vars)}')
|
||||
print(f' - Переменных в .env: {len(env_vars)}')
|
||||
print(
|
||||
f' - Полей в Pydantic Settings (всего/обязательных): {len(all_pydantic_fields)}/{len(required_pydantic_fields)}'
|
||||
)
|
||||
return 0
|
||||
|
||||
print('❌ Обнаружены несоответствия')
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,65 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pydantic
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WorkerConfig(pydantic.BaseModel):
|
||||
INTERVAL_SECONDS: int = 60
|
||||
|
||||
|
||||
class WorkerBase:
|
||||
def __init__(self, config: WorkerConfig) -> None:
|
||||
self.config = config
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._stop_event = asyncio.Event()
|
||||
|
||||
log.info('Worker initialized (interval=%ds)', config.INTERVAL_SECONDS)
|
||||
|
||||
async def _cycle_func(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._task is not None:
|
||||
log.warning('Worker already running')
|
||||
return
|
||||
|
||||
self._stop_event.clear()
|
||||
self._task = asyncio.create_task(self._run())
|
||||
log.info('Worker started')
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is None:
|
||||
log.warning('Worker not running')
|
||||
return
|
||||
|
||||
self._stop_event.set()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(self._task, timeout=5.0)
|
||||
except TimeoutError:
|
||||
log.warning('Worker did not stop in time, cancelling')
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except asyncio.CancelledError:
|
||||
log.info('Worker task cancelled')
|
||||
|
||||
self._task = None
|
||||
log.info('Worker stopped')
|
||||
|
||||
async def _run(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
await self._cycle_func()
|
||||
except Exception:
|
||||
log.exception('Error in worker cycle')
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(self._stop_event.wait(), timeout=self.config.INTERVAL_SECONDS)
|
||||
except TimeoutError:
|
||||
pass # Время вышло, продолжаем
|
||||
Reference in New Issue
Block a user