feat: fetch views manually and worker

This commit is contained in:
Artem Tsyrulnikov
2025-11-12 23:55:44 +03:00
parent 292bb0d49b
commit a48d5e67cb
28 changed files with 632 additions and 135 deletions

View File

@@ -358,3 +358,17 @@ class Postgres(DatabaseBase):
result = await self.session.execute(stmt)
return list(result.scalars().all())
async def get_active_purchases_with_urls(self) -> list[domain.Purchase]:
stmt = (
select(domain.Purchase)
.where(
domain.Purchase.status == domain.PurchaseStatus.ACTIVE,
domain.Purchase.ad_post_url.isnot(None),
domain.Purchase.views_availability != domain.PostViewsAvailability.MANUAL,
)
.order_by(domain.Purchase.created_at.asc())
)
result = await self.session.execute(stmt)
return list(result.scalars().all())

View File

@@ -1,59 +0,0 @@
import logging
from typing import Any
from aiogram.types import InlineKeyboardButton, InlineKeyboardMarkup
from shared.telegram_base import TelegramBase
log = logging.getLogger(__name__)
class Telegram(TelegramBase):
async def send_message(self, text: str, chat_id: int) -> None:
await self.bot.send_message(chat_id=chat_id, text=text)
async def send_message_with_button(self, text: str, chat_id: int, button_text: str, button_url: str) -> None:
keyboard = InlineKeyboardMarkup(inline_keyboard=[[InlineKeyboardButton(text=button_text, url=button_url)]])
await self.bot.send_message(chat_id=chat_id, text=text, reply_markup=keyboard)
async def get_chat_member(self, chat_id: int, user_id: int) -> dict[str, Any]:
member = await self.bot.get_chat_member(chat_id=chat_id, user_id=user_id)
return member.model_dump()
async def create_chat_invite_link(self, chat_id: int, requires_approval: bool = False) -> str:
invite_link = await self.bot.create_chat_invite_link(
chat_id=chat_id,
creates_join_request=requires_approval,
)
return invite_link.invite_link
async def get_post_views(self, post_url: str) -> int | None:
"""
Получить количество просмотров поста по URL.
ЗАГЛУШКА: В текущей реализации возвращает None.
Для реальной работы необходимо:
1. Интегрировать TDLib/MTProto клиент
2. Парсить post_url (формат: t.me/<username>/<message_id> или t.me/c/<chat_id>/<message_id>)
3. Вызывать channels.getMessages через TDLib
4. Извлекать поле views из ответа
Returns:
int | None: Количество просмотров или None если не удалось получить
"""
log.warning(
'get_post_views is not implemented yet (TDLib integration required). URL: %s',
post_url,
)
# TODO: Implement TDLib integration
# Example pseudo-code:
# try:
# username, message_id = parse_telegram_url(post_url)
# channel = await tdlib.resolve_username(username)
# message = await tdlib.get_message(channel.id, message_id)
# return message.views
# except Exception as e:
# log.error('Failed to get post views: %s', e)
# return None
return None

View File

@@ -0,0 +1,25 @@
import logging
from typing import Any
from aiogram.types import InlineKeyboardButton, InlineKeyboardMarkup
from shared.telegram_base import TelegramBase
log = logging.getLogger(__name__)
class TelegramBot(TelegramBase):
async def send_message(self, text: str, chat_id: int) -> None:
await self.bot.send_message(chat_id=chat_id, text=text)
async def send_message_with_button(self, text: str, chat_id: int, button_text: str, button_url: str) -> None:
keyboard = InlineKeyboardMarkup(inline_keyboard=[[InlineKeyboardButton(text=button_text, url=button_url)]])
await self.bot.send_message(chat_id=chat_id, text=text, reply_markup=keyboard)
async def get_chat_member(self, chat_id: int, user_id: int) -> dict[str, Any]:
member = await self.bot.get_chat_member(chat_id=chat_id, user_id=user_id)
return member.model_dump()
async def create_chat_invite_link(self, chat_id: int, requires_approval: bool = False) -> str:
invite_link = await self.bot.create_chat_invite_link(chat_id=chat_id, creates_join_request=requires_approval)
return invite_link.invite_link

View File

@@ -0,0 +1,99 @@
import logging
import re
from bs4 import BeautifulSoup
from shared.http_parser_base import HttpParserBase, HttpParserConfig
log = logging.getLogger(__name__)
class TelegramParser(HttpParserBase):
def __init__(self, config: HttpParserConfig) -> None:
super().__init__(config)
log.info('TelegramParser initialized (timeout=%s)', config.TIMEOUT)
def _parse_url(self, url: str) -> tuple[str, int] | None:
"""Parse Telegram post URL into username and message_id."""
match = re.match(r'(?:https?://)?t\.me/([\w]+)/(\d+)', url)
if not match:
return None
username, message_id = match.groups()
return username, int(message_id)
def _parse_view_count(self, text: str) -> int | None:
"""Convert view counter text (1.2K, 5.3M) to integer."""
if not text:
return None
text = text.strip().replace(',', '').replace(' ', '')
multipliers = {'K': 1_000, 'M': 1_000_000, 'B': 1_000_000_000}
try:
if text[-1].upper() in multipliers:
return int(float(text[:-1]) * multipliers[text[-1].upper()])
return int(text)
except (ValueError, IndexError):
return None
async def _fetch_from_preview(self, username: str, message_id: int) -> int | None:
url = f'https://t.me/s/{username}/{message_id}'
status_code, html = await self.fetch_html(url)
if status_code != 200:
return None
soup = BeautifulSoup(html, 'lxml')
messages = soup.find_all('div', class_='tgme_widget_message')
for message_div in messages:
data_post = message_div.get('data-post', '')
if isinstance(data_post, str) and data_post.endswith(f'/{message_id}'):
views_span = message_div.find('span', class_='tgme_widget_message_views')
if views_span:
views_text = views_span.get_text(strip=True)
return self._parse_view_count(views_text)
return None
async def _fetch_from_embed(self, username: str, message_id: int) -> int | None:
url = f'https://t.me/{username}/{message_id}?embed=1'
status_code, html = await self.fetch_html(url)
if status_code != 200:
return None
soup = BeautifulSoup(html, 'lxml')
views_span = soup.find('span', class_='tgme_widget_message_views')
if not views_span:
return None
views_text = views_span.get_text(strip=True)
return self._parse_view_count(views_text)
async def get_post_views(self, post_url: str) -> int | None:
parsed = self._parse_url(post_url)
if not parsed:
log.error('Invalid Telegram URL format: %s', post_url)
return None
username, message_id = parsed
try:
views = await self._fetch_from_preview(username, message_id)
if views is not None:
log.info('Fetched views for %s: %d', post_url, views)
return views
views = await self._fetch_from_embed(username, message_id)
if views is not None:
log.info('Fetched views for %s: %d', post_url, views)
return views
log.warning('Unable to extract views from %s', post_url)
return None
except Exception as exc:
log.exception('Failed to fetch %s: %s', post_url, exc.__class__.__name__)
return None

View File

@@ -3,8 +3,10 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
from shared.config_helper import load_settings
from shared.datebase_base import DatabaseConfig
from shared.http_parser_base import HttpParserConfig
from shared.logger import LoggerConfig
from shared.telegram_base import TelegramConfig
from shared.worker_base import WorkerConfig
from src.adapter.jwt import JWTConfig
@@ -19,7 +21,10 @@ class Settings(BaseSettings):
db: DatabaseConfig
logger: LoggerConfig
telegram: TelegramConfig
telegram_parser: HttpParserConfig
jwt: JWTConfig
fetch_views_worker: WorkerConfig
settings: Settings = load_settings(Settings)

View File

@@ -33,9 +33,9 @@ async def get_views_history(
async def fetch_views(
purchase_id: uuid.UUID,
current_user: Annotated[JWTPayload, Depends(dependencies.get_current_user)],
) -> dto.FetchViewsOutput:
) -> dto.FetchViewsManuallyOutput:
"""Получить актуальные просмотры через Telegram API."""
input_data = dto.FetchViewsInput(
input_data = dto.FetchViewsInputManually(
purchase_id=purchase_id,
user_id=current_user.user_id,
)

View File

@@ -0,0 +1,15 @@
import logging
from typing import TYPE_CHECKING
from shared.worker_base import WorkerBase
from src import dependencies
if TYPE_CHECKING:
pass
log = logging.getLogger(__name__)
class FetchViewsWorker(WorkerBase):
async def _cycle_func(self) -> None:
await dependencies.get_usecase().run_views_worker_cycle(self.config.INTERVAL_SECONDS)

View File

@@ -34,8 +34,8 @@ __all__ = (
'ViewsSnapshotOutput',
'GetViewsHistoryInput',
'GetViewsHistoryOutput',
'FetchViewsInput',
'FetchViewsOutput',
'FetchViewsInputManually',
'FetchViewsManuallyOutput',
'UpdateViewsManuallyInput',
)
@@ -79,8 +79,8 @@ from .target_channel import (
from .telegram_login import TelegramLoginInput
from .validate_login_token import ValidateLoginTokenInput, ValidateLoginTokenOutput
from .views import (
FetchViewsInput,
FetchViewsOutput,
FetchViewsInputManually,
FetchViewsManuallyOutput,
GetViewsHistoryInput,
GetViewsHistoryOutput,
UpdateViewsManuallyInput,

View File

@@ -31,16 +31,14 @@ class GetViewsHistoryOutput(pydantic.BaseModel):
snapshots: list[ViewsSnapshotOutput]
class FetchViewsInput(pydantic.BaseModel):
class FetchViewsInputManually(pydantic.BaseModel):
"""Запросить обновление просмотров для закупа."""
purchase_id: uuid.UUID
user_id: uuid.UUID
class FetchViewsOutput(pydantic.BaseModel):
"""Результат получения просмотров."""
class FetchViewsManuallyOutput(pydantic.BaseModel):
purchase_id: uuid.UUID
views_count: int | None # None если не удалось получить
views_availability: domain.PostViewsAvailability

View File

@@ -7,10 +7,12 @@ from shared import logger
from src import dependencies
from src.adapter.jwt import JWT
from src.adapter.postgres import Postgres
from src.adapter.telegram import Telegram
from src.adapter.telegram_bot import TelegramBot
from src.adapter.telegram_parser import TelegramParser
from src.config import settings
from src.controller.http import api_router
from src.controller.telegram_callback import telegram_callback_router
from src.controller.worker.views_worker import FetchViewsWorker
from src.usecase import Usecase
@@ -18,9 +20,11 @@ from src.usecase import Usecase
async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
await postgres.connect()
await telegram.start_polling()
await views_worker.start()
yield
await views_worker.stop()
await telegram.stop_polling()
await postgres.close()
@@ -29,15 +33,19 @@ logger.init(settings.logger)
# Dependencies
postgres = Postgres(settings.db)
telegram = Telegram(settings.telegram, telegram_callback_router)
telegram = TelegramBot(settings.telegram, telegram_callback_router)
telegram_views_fetcher = TelegramParser(settings.telegram_parser)
jwt_encoder = JWT(settings.jwt)
usecase = Usecase(
database=postgres,
telegram_writer=telegram,
telegram_bot=telegram,
telegram_parser=telegram_views_fetcher,
jwt_encoder=jwt_encoder,
)
dependencies.set_usecase(usecase)
views_worker = FetchViewsWorker(config=settings.fetch_views_worker)
app = FastAPI(lifespan=lifespan, version=settings.logger.APP_VERSION)
app.include_router(api_router)

View File

@@ -28,8 +28,9 @@ from .target_channel.get_user_target_chans import get_user_target_chans
from .target_channel.update_target_chan_permissions import update_target_chan_permissions
from .telegram_login import telegram_login
from .validate_login_token import validate_login_token
from .views.fetch_views import fetch_views
from .views.fetch_views_manually import fetch_views_manually
from .views.get_views_history import get_views_history
from .views.run_views_worker_cycle import run_views_worker_cycle
from .views.update_views_manually import update_views_manually
@@ -132,8 +133,10 @@ class Database(typing.Protocol):
to_date: datetime.datetime | None = None,
) -> list[domain.ViewsSnapshot]: ...
async def get_active_purchases_with_urls(self) -> list[domain.Purchase]: ... # Для worker
class TelegramWriter(typing.Protocol):
class TelegramBotWriter(typing.Protocol):
async def send_message(self, text: str, chat_id: int) -> None: ...
async def send_message_with_button(self, text: str, chat_id: int, button_text: str, button_url: str) -> None: ...
@@ -142,7 +145,9 @@ class TelegramWriter(typing.Protocol):
async def create_chat_invite_link(self, chat_id: int, requires_approval: bool = False) -> str: ...
async def get_post_views(self, post_url: str) -> int | None: ... # None если не удалось получить
class TelegramParser(typing.Protocol):
async def get_post_views(self, post_url: str) -> int | None: ...
class JWTEncoder(typing.Protocol):
@@ -152,7 +157,8 @@ class JWTEncoder(typing.Protocol):
@dataclass
class Usecase:
database: Database
telegram_writer: TelegramWriter
telegram_bot: TelegramBotWriter
telegram_parser: TelegramParser
jwt_encoder: JWTEncoder
validate_login_token = validate_login_token
@@ -178,6 +184,7 @@ class Usecase:
update_purchase = update_purchase
delete_purchase = delete_purchase
handle_subscription = handle_subscription
fetch_views = fetch_views
fetch_views = fetch_views_manually
run_views_worker_cycle = run_views_worker_cycle
get_views_history = get_views_history
update_views_manually = update_views_manually

View File

@@ -36,7 +36,7 @@ async def create_purchase(self: 'Usecase', input: dto.CreatePurchaseInput, user_
# Создаём уникальную пригласительную ссылку через Telegram Bot API
requires_approval = input.invite_link_type == domain.InviteLinkType.APPROVAL
invite_link = await self.telegram_writer.create_chat_invite_link(
invite_link = await self.telegram_bot.create_chat_invite_link(
chat_id=target_channel.telegram_id,
requires_approval=requires_approval,
)

View File

@@ -14,7 +14,7 @@ async def connect_target_chan(self: 'Usecase', input: dto.ConnectTargetChanInput
if not permissions.is_admin:
log.warning(f'Bot is not admin in channel {input.telegram_id}. Attempted by user {input.user_telegram_id}')
await self.telegram_writer.send_message(
await self.telegram_bot.send_message(
f'⚠️ Бот был добавлен в канал "{input.title}", но не является админом.\n\n'
'Пожалуйста, сделайте бота администратором канала.',
chat_id=input.user_telegram_id,
@@ -33,7 +33,7 @@ async def connect_target_chan(self: 'Usecase', input: dto.ConnectTargetChanInput
f'Attempted by user {input.user_telegram_id}'
)
permissions_text = '\n'.join(f'{p}' for p in missing_permissions)
await self.telegram_writer.send_message(
await self.telegram_bot.send_message(
f'⚠️ Бот был добавлен в канал "{input.title}", но не имеет необходимых прав.\n\n'
f'Отсутствующие права:\n{permissions_text}\n\n'
'Пожалуйста, предоставьте эти права боту в настройках канала.',
@@ -45,7 +45,7 @@ async def connect_target_chan(self: 'Usecase', input: dto.ConnectTargetChanInput
user = await self.database.get_user(telegram_id=input.user_telegram_id)
if not user:
log.warning(f'User {input.user_telegram_id} not found when trying to connect channel {input.telegram_id}')
await self.telegram_writer.send_message(
await self.telegram_bot.send_message(
f'⚠️ Канал "{input.title}" не может быть подключен.\n\n'
'Вы должны сначала авторизоваться в веб-панели перед подключением каналов.',
chat_id=input.user_telegram_id,
@@ -63,7 +63,7 @@ async def connect_target_chan(self: 'Usecase', input: dto.ConnectTargetChanInput
upserted_channel = await self.database.upsert_target_channel(channel)
log.info(f'Channel {input.telegram_id} connected/updated successfully by user {input.user_telegram_id}')
await self.telegram_writer.send_message(
await self.telegram_bot.send_message(
f'✅ Канал "{input.title}" успешно подключен!',
chat_id=input.user_telegram_id,
)

View File

@@ -33,7 +33,7 @@ async def update_target_chan_permissions(self: 'Usecase', input: dto.UpdateTarge
channel.is_active = True
await self.database.update_target_channel(channel)
await self.telegram_writer.send_message(
await self.telegram_bot.send_message(
f'✅ Канал "{input.chat_title}" был активирован.\n\nВсе необходимые права боту предоставлены!',
chat_id=input.user_telegram_id,
)
@@ -45,7 +45,7 @@ async def update_target_chan_permissions(self: 'Usecase', input: dto.UpdateTarge
await self.database.update_target_channel(channel)
missed_permissions = '\n'.join(f'{p}' for p in missing_permissions)
await self.telegram_writer.send_message(
await self.telegram_bot.send_message(
f'⚠️ Канал "{input.chat_title}" был деактивирован.\
\n\nБоту убрали необходимые права:\n{missed_permissions}',
chat_id=input.user_telegram_id,

View File

@@ -28,7 +28,7 @@ async def telegram_login(self: 'Usecase', input: dto.TelegramLoginInput) -> None
login_url = f'{settings.app.URL}/auth/complete?token={token}'
username_display = f'@{user.username}' if user.username else 'пользователь'
await self.telegram_writer.send_message_with_button(
await self.telegram_bot.send_message_with_button(
f'Привет, {username_display}!\n\nНажми кнопку ниже для входа на сайт.',
chat_id=input.chat_id,
button_text='Войти на сайт',

View File

@@ -10,21 +10,14 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
async def fetch_views(self: 'Usecase', input: dto.FetchViewsInput) -> dto.FetchViewsOutput:
"""
Получить просмотры для закупа через Telegram API.
Создаёт снимок просмотров и обновляет метрики закупа.
"""
async def fetch_views_manually(self: 'Usecase', input: dto.FetchViewsInputManually) -> dto.FetchViewsManuallyOutput:
async with self.database.transaction():
purchase = await self.database.get_purchase(input.user_id, input.purchase_id)
if not purchase:
log.warning('Purchase %s not found for user %s', input.purchase_id, input.user_id)
raise domain.PurchaseNotFound(input.purchase_id)
if not purchase.ad_post_url:
log.warning('Purchase %s has no ad_post_url', input.purchase_id)
return dto.FetchViewsOutput(
return dto.FetchViewsManuallyOutput(
purchase_id=purchase.id,
views_count=None,
views_availability=domain.PostViewsAvailability.UNAVAILABLE,
@@ -32,19 +25,15 @@ async def fetch_views(self: 'Usecase', input: dto.FetchViewsInput) -> dto.FetchV
error_message='No ad_post_url specified',
)
# Попытка получить просмотры через Telegram API
views_count = await self.telegram_writer.get_post_views(purchase.ad_post_url)
views_count = await self.telegram_parser.get_post_views(purchase.ad_post_url)
fetched_at = datetime.datetime.now(datetime.UTC)
if views_count is not None:
# Успешно получены просмотры
if views_count:
purchase.views_count = views_count
purchase.views_availability = domain.PostViewsAvailability.AVAILABLE
purchase.last_views_fetch_at = fetched_at
await self.database.update_purchase(purchase)
# Создаём снимок
snapshot = domain.ViewsSnapshot(
purchase_id=purchase.id,
views_count=views_count,
@@ -54,24 +43,23 @@ async def fetch_views(self: 'Usecase', input: dto.FetchViewsInput) -> dto.FetchV
log.info('Views fetched for purchase %s: %d', purchase.id, views_count)
return dto.FetchViewsOutput(
return dto.FetchViewsManuallyOutput(
purchase_id=purchase.id,
views_count=views_count,
views_availability=domain.PostViewsAvailability.AVAILABLE,
fetched_at=fetched_at,
)
else:
# Не удалось получить просмотры
purchase.views_availability = domain.PostViewsAvailability.UNAVAILABLE
purchase.last_views_fetch_at = fetched_at
await self.database.update_purchase(purchase)
log.warning('Failed to fetch views for purchase %s', purchase.id)
purchase.views_availability = domain.PostViewsAvailability.UNAVAILABLE
purchase.last_views_fetch_at = fetched_at
await self.database.update_purchase(purchase)
return dto.FetchViewsOutput(
purchase_id=purchase.id,
views_count=None,
views_availability=domain.PostViewsAvailability.UNAVAILABLE,
fetched_at=fetched_at,
error_message='Failed to fetch views from Telegram API',
)
log.warning('Failed to fetch views for purchase %s', purchase.id)
return dto.FetchViewsManuallyOutput(
purchase_id=purchase.id,
views_count=None,
views_availability=domain.PostViewsAvailability.UNAVAILABLE,
fetched_at=fetched_at,
error_message='Failed to fetch views from Telegram API',
)

View File

@@ -0,0 +1,65 @@
import asyncio
import datetime
import logging
from typing import TYPE_CHECKING, cast
from aiolimiter import AsyncLimiter
from src import domain
if TYPE_CHECKING:
from .. import Usecase
log = logging.getLogger(__name__)
async def _process_purchase(self: 'Usecase', purchase_id, user_id, ad_post_url: str):
views_count = await self.telegram_parser.get_post_views(ad_post_url)
fetched_at = datetime.datetime.now(datetime.UTC)
async with self.database.transaction():
p = await self.database.get_purchase(user_id=user_id, purchase_id=purchase_id)
if not p:
log.error('Purchase %s not found', purchase_id)
return
p.last_views_fetch_at = fetched_at
if not views_count:
p.views_availability = domain.PostViewsAvailability.UNAVAILABLE
await self.database.update_purchase(p)
log.warning('Failed to fetch views for %s', purchase_id)
return
p.views_count = views_count
p.views_availability = domain.PostViewsAvailability.AVAILABLE
await self.database.update_purchase(p)
snapshot = domain.ViewsSnapshot(
purchase_id=purchase_id,
views_count=views_count,
fetched_at=fetched_at,
)
await self.database.create_views_snapshot(snapshot)
log.info('Updated purchase %s: %d', purchase_id, views_count)
async def run_views_worker_cycle(self: 'Usecase', interval_seconds: int) -> None:
async with self.database.transaction():
purchases = await self.database.get_active_purchases_with_urls()
if not purchases:
log.debug('ViewsWorkerCycle: No purchases to process')
return
limiter = AsyncLimiter(max_rate=len(purchases), time_period=interval_seconds)
log.info('Processing %d purchases for %d seconds', len(purchases), interval_seconds)
async def process(p):
async with limiter:
try:
await _process_purchase(self, p.id, p.user_id, cast(str, p.ad_post_url))
except Exception:
log.exception('Error updating views for %s', p.id)
await asyncio.gather(*(process(p) for p in purchases))

View File

@@ -11,7 +11,6 @@ log = logging.getLogger(__name__)
async def update_views_manually(self: 'Usecase', input: dto.UpdateViewsManuallyInput) -> dto.PurchaseOutput:
"""Ручное обновление просмотров для закупа."""
async with self.database.transaction():
purchase = await self.database.get_purchase(input.user_id, input.purchase_id)
if not purchase:
@@ -20,13 +19,11 @@ async def update_views_manually(self: 'Usecase', input: dto.UpdateViewsManuallyI
fetched_at = datetime.datetime.now(datetime.UTC)
# Обновляем метрики закупа
purchase.views_count = input.views_count
purchase.views_availability = domain.PostViewsAvailability.MANUAL
purchase.last_views_fetch_at = fetched_at
await self.database.update_purchase(purchase)
# Создаём снимок
snapshot = domain.ViewsSnapshot(
purchase_id=purchase.id,
views_count=input.views_count,
@@ -36,7 +33,6 @@ async def update_views_manually(self: 'Usecase', input: dto.UpdateViewsManuallyI
log.info('Views manually updated for purchase %s: %d', purchase.id, input.views_count)
# Возвращаем обновлённый закуп
return dto.PurchaseOutput(
id=purchase.id,
target_channel_id=purchase.target_channel_id,