Initial commit

tg_parser/internal/adapter/telegram/get_active_views.go

@@ -0,0 +1,37 @@
+package telegram
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/TelegramExchange/tgex-backend/tg_parser/internal/domain"
+	"github.com/gotd/td/tg"
+)
+
+func (t *Telegram) UpdatePostsViews(ctx context.Context, channel domain.Channel, posts []domain.Post) error {
+	ids := make([]int, len(posts))
+	for i, p := range posts {
+		ids[i] = p.MessageID
+	}
+
+	req := &tg.MessagesGetMessagesViewsRequest{
+		Peer: &tg.InputPeerChannel{
+			ChannelID:  channel.ChannelID(),
+			AccessHash: channel.AccessHash,
+		},
+		ID:        ids,
+		Increment: false,
+	}
+
+	resp, err := t.API().MessagesGetMessagesViews(ctx, req)
+	if err != nil {
+		return fmt.Errorf("get messages views: %w", err)
+	}
+
+	// Telegram гарантирует, что resp.Views соответствует порядку ids
+	for i, v := range resp.Views {
+		posts[i].Views = v.Views
+	}
+
+	return nil
+}

tg_parser/internal/adapter/telegram/get_channel_diff.go

@@ -0,0 +1,125 @@
+package telegram
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/TelegramExchange/tgex-backend/tg_parser/internal/domain"
+	"github.com/gotd/td/tg"
+)
+
+func (t *Telegram) GetChannelDiff(ctx context.Context, channel domain.Channel, limit int) (domain.ChannelDiff, error) {
+	req := &tg.UpdatesGetChannelDifferenceRequest{
+		Channel: &tg.InputChannel{
+			ChannelID:  channel.ChannelID(),
+			AccessHash: channel.AccessHash,
+		},
+		Filter: &tg.ChannelMessagesFilterEmpty{},
+		Pts:    channel.Pts,
+		Limit:  limit,
+	}
+
+	rawDiff, err := t.API().UpdatesGetChannelDifference(ctx, req)
+	if err != nil {
+		return domain.ChannelDiff{}, fmt.Errorf("get difference: %w", err)
+	}
+
+	result := domain.ChannelDiff{}
+
+	switch d := rawDiff.(type) {
+	case *tg.UpdatesChannelDifferenceEmpty:
+		result.NewPts = d.Pts
+
+	case *tg.UpdatesChannelDifferenceTooLong:
+		if dialog, ok := d.Dialog.(*tg.Dialog); ok {
+			result.NewPts = dialog.Pts
+		}
+		result.NewPosts = extractPosts(channel, d.Messages)
+		result.UpdatedChannel = extractChannelMeta(channel, d.Chats)
+
+	case *tg.UpdatesChannelDifference:
+		result.NewPts = d.Pts
+		result.NewPosts = extractPosts(channel, d.NewMessages)
+		result.DeletedPosts = extractDeletedPosts(channel, d.OtherUpdates)
+		result.UpdatedChannel = extractChannelMeta(channel, d.Chats)
+
+	default:
+		return domain.ChannelDiff{}, fmt.Errorf("unexpected rawDiff type: %T", rawDiff)
+	}
+
+	return result, nil
+}
+
+func extractPosts(channel domain.Channel, msgs []tg.MessageClass) []domain.Post {
+	posts := make([]domain.Post, 0, len(msgs))
+
+	for _, raw := range msgs {
+		m, ok := raw.(*tg.Message)
+		if !ok {
+			continue
+		}
+
+		text := messageToHTML(m.Message, m.Entities)
+		publishedAt := time.Unix(int64(m.Date), 0).UTC()
+		p := domain.NewPost(channel, m.ID, text, m.Views, publishedAt)
+		posts = append(posts, p)
+	}
+
+	return posts
+}
+
+func extractDeletedPosts(channel domain.Channel, updates []tg.UpdateClass) []domain.Post {
+	var deleted []domain.Post
+
+	for _, upd := range updates {
+		if u, ok := upd.(*tg.UpdateDeleteChannelMessages); ok {
+			for _, msgID := range u.Messages {
+				p := domain.NewPost(channel, msgID, "", 0, time.Time{})
+				deleted = append(deleted, p)
+			}
+		}
+	}
+
+	return deleted
+}
+
+func extractChannelMeta(currentChannel domain.Channel, chats []tg.ChatClass) *domain.Channel {
+	for _, chat := range chats {
+		ch, ok := chat.(*tg.Channel)
+		if !ok {
+			continue
+		}
+
+		if ch.ID != currentChannel.ChannelID() {
+			continue
+		}
+
+		// Preserve username if Telegram doesn't provide a new one (private channels may not have username)
+		username := currentChannel.Username
+		if usernameVal, ok := ch.GetUsername(); ok && usernameVal != "" {
+			username = usernameVal
+		}
+
+		// Keep current access hash if new one is not provided or is zero
+		accessHash := currentChannel.AccessHash
+		if accessHashVal, ok := ch.GetAccessHash(); ok && accessHashVal != 0 {
+			accessHash = accessHashVal
+		}
+
+		updated := domain.Channel{
+			ID:           currentChannel.ID,
+			TelegramID:   domain.ChatIDFromChannelID(ch.ID),
+			Username:     username,
+			Title:        ch.Title,
+			AccessHash:   accessHash,
+			InviteLink:   currentChannel.InviteLink, // Preserve invite link (never returned in channel diff)
+			Pts:          currentChannel.Pts,
+			IsAccessible: currentChannel.IsAccessible,
+		}
+
+		return &updated
+	}
+
+	return nil
+}

tg_parser/internal/adapter/telegram/history.go

@@ -0,0 +1,49 @@
+package telegram
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/TelegramExchange/tgex-backend/tg_parser/internal/domain"
+	"github.com/gotd/td/tg"
+	"github.com/rs/zerolog/log"
+)
+
+func (t *Telegram) GetChannelHistory(ctx context.Context, channel domain.Channel, limit int) ([]domain.Post, error) {
+	log.Info().Msg(channel.String())
+
+	req := &tg.MessagesGetHistoryRequest{
+		Peer: &tg.InputPeerChannel{
+			ChannelID:  channel.ChannelID(),
+			AccessHash: channel.AccessHash,
+		},
+		Limit: limit,
+	}
+
+	resp, err := t.API().MessagesGetHistory(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("resp: %w", err)
+	}
+
+	messages, ok := resp.(*tg.MessagesChannelMessages)
+	if !ok {
+		return nil, nil
+	}
+
+	posts := make([]domain.Post, 0, len(messages.Messages))
+
+	for _, raw := range messages.Messages {
+		m, ok := raw.(*tg.Message)
+		if !ok {
+			continue
+		}
+
+		text := messageToHTML(m.Message, m.Entities)
+		publishedAt := time.Unix(int64(m.Date), 0).UTC()
+		p := domain.NewPost(channel, m.ID, text, m.Views, publishedAt)
+		posts = append(posts, p)
+	}
+
+	return posts, nil
+}

tg_parser/internal/adapter/telegram/message_html.go

@@ -0,0 +1,197 @@
+package telegram
+
+import (
+	"html"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/gotd/td/tg"
+)
+
+type htmlEntity struct {
+	start    int
+	end      int
+	openTag  string
+	closeTag string
+	length   int
+}
+
+type htmlEvent struct {
+	pos     int
+	tag     string
+	isStart bool
+	length  int
+}
+
+func messageToHTML(text string, entities []tg.MessageEntityClass) string {
+	if text == "" {
+		return ""
+	}
+
+	if len(entities) == 0 {
+		return html.EscapeString(text)
+	}
+
+	ranges := make([]htmlEntity, 0, len(entities))
+	needed := make([]int, 0, len(entities)*2)
+
+	for _, raw := range entities {
+		offset, length, openTag, closeTag, ok := htmlEntityMeta(raw)
+		if !ok {
+			continue
+		}
+
+		start := offset
+		end := offset + length
+		if length <= 0 {
+			continue
+		}
+
+		ranges = append(ranges, htmlEntity{
+			start:    start,
+			end:      end,
+			openTag:  openTag,
+			closeTag: closeTag,
+			length:   length,
+		})
+		needed = append(needed, start, end)
+	}
+
+	if len(ranges) == 0 {
+		return html.EscapeString(text)
+	}
+
+	positions := utf16PositionsToBytes(text, needed)
+	events := make([]htmlEvent, 0, len(ranges)*2)
+
+	for _, r := range ranges {
+		startByte, okStart := positions[r.start]
+		endByte, okEnd := positions[r.end]
+		if !okStart || !okEnd || startByte > endByte {
+			continue
+		}
+
+		events = append(events, htmlEvent{
+			pos:     startByte,
+			tag:     r.openTag,
+			isStart: true,
+			length:  r.length,
+		})
+		events = append(events, htmlEvent{
+			pos:     endByte,
+			tag:     r.closeTag,
+			isStart: false,
+			length:  r.length,
+		})
+	}
+
+	sort.SliceStable(events, func(i, j int) bool {
+		if events[i].pos != events[j].pos {
+			return events[i].pos < events[j].pos
+		}
+		if events[i].isStart != events[j].isStart {
+			return !events[i].isStart
+		}
+		if events[i].isStart {
+			return events[i].length > events[j].length
+		}
+		return events[i].length < events[j].length
+	})
+
+	var b strings.Builder
+	last := 0
+	for _, ev := range events {
+		if ev.pos > last {
+			b.WriteString(html.EscapeString(text[last:ev.pos]))
+		}
+		b.WriteString(ev.tag)
+		last = ev.pos
+	}
+	if last < len(text) {
+		b.WriteString(html.EscapeString(text[last:]))
+	}
+
+	return b.String()
+}
+
+func htmlEntityMeta(entity tg.MessageEntityClass) (offset int, length int, openTag string, closeTag string, ok bool) {
+	switch e := entity.(type) {
+	case *tg.MessageEntityBold:
+		return e.Offset, e.Length, "<b>", "</b>", true
+	case *tg.MessageEntityItalic:
+		return e.Offset, e.Length, "<i>", "</i>", true
+	case *tg.MessageEntityUnderline:
+		return e.Offset, e.Length, "<u>", "</u>", true
+	case *tg.MessageEntityStrike:
+		return e.Offset, e.Length, "<s>", "</s>", true
+	case *tg.MessageEntityCode:
+		return e.Offset, e.Length, "<code>", "</code>", true
+	case *tg.MessageEntityPre:
+		if e.Language != "" {
+			lang := html.EscapeString(e.Language)
+			return e.Offset, e.Length, `<pre><code class="language-` + lang + `">`, "</code></pre>", true
+		}
+		return e.Offset, e.Length, "<pre>", "</pre>", true
+	case *tg.MessageEntityTextURL:
+		url := html.EscapeString(e.URL)
+		return e.Offset, e.Length, `<a href="` + url + `">`, "</a>", true
+	case *tg.MessageEntityMentionName:
+		userID := html.EscapeString(strconv.FormatInt(e.UserID, 10))
+		return e.Offset, e.Length, `<a href="tg://user?id=` + userID + `">`, "</a>", true
+	case *tg.MessageEntitySpoiler:
+		return e.Offset, e.Length, `<span class="tg-spoiler">`, "</span>", true
+	case *tg.MessageEntityBlockquote:
+		if e.Collapsed {
+			return e.Offset, e.Length, `<blockquote expandable>`, "</blockquote>", true
+		}
+		return e.Offset, e.Length, "<blockquote>", "</blockquote>", true
+	case *tg.MessageEntityCustomEmoji:
+		id := html.EscapeString(strconv.FormatInt(e.DocumentID, 10))
+		return e.Offset, e.Length, `<tg-emoji emoji-id="` + id + `">`, "</tg-emoji>", true
+	default:
+		return 0, 0, "", "", false
+	}
+}
+
+func utf16PositionsToBytes(text string, needed []int) map[int]int {
+	result := make(map[int]int, len(needed))
+	needSet := make(map[int]struct{}, len(needed))
+	for _, n := range needed {
+		needSet[n] = struct{}{}
+	}
+
+	utf16Pos := 0
+	if _, ok := needSet[0]; ok {
+		result[0] = 0
+	}
+
+	for i, r := range text {
+		if _, ok := needSet[utf16Pos]; ok {
+			result[utf16Pos] = i
+		}
+
+		step := utf16RuneLen(r)
+		if step == 2 {
+			if _, ok := needSet[utf16Pos+1]; ok {
+				result[utf16Pos+1] = i
+			}
+		}
+		utf16Pos += step
+	}
+
+	if _, ok := needSet[utf16Pos]; ok {
+		result[utf16Pos] = len(text)
+	}
+
+	return result
+}
+
+func utf16RuneLen(r rune) int {
+	const surrSelf = 0x10000
+	if r >= surrSelf && r <= utf8.MaxRune {
+		return 2
+	}
+	return 1
+}

tg_parser/internal/adapter/telegram/pts.go

@@ -0,0 +1,33 @@
+package telegram
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/TelegramExchange/tgex-backend/tg_parser/internal/domain"
+	"github.com/gotd/td/tg"
+)
+
+func (t *Telegram) GetChannelPTS(ctx context.Context, channel domain.Channel) (int, error) {
+	req := []tg.InputDialogPeerClass{
+		&tg.InputDialogPeer{
+			Peer: &tg.InputPeerChannel{
+				ChannelID:  channel.ChannelID(),
+				AccessHash: channel.AccessHash,
+			},
+		},
+	}
+
+	dialogs, err := t.API().MessagesGetPeerDialogs(ctx, req)
+	if err != nil {
+		return 0, fmt.Errorf("peer dialogs: %w", err)
+	}
+
+	if len(dialogs.Dialogs) > 0 {
+		if d, ok := dialogs.Dialogs[0].(*tg.Dialog); ok {
+			return d.Pts, nil
+		}
+	}
+
+	return 0, nil
+}

tg_parser/internal/adapter/telegram/resolve.go

@@ -0,0 +1,58 @@
+package telegram
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/TelegramExchange/tgex-backend/tg_parser/internal/domain"
+	"github.com/gotd/td/tg"
+)
+
+func (t *Telegram) ParseChannelMeta(ctx context.Context, username string) (domain.Channel, error) {
+	resolved, err := t.API().ContactsResolveUsername(ctx, &tg.ContactsResolveUsernameRequest{
+		Username: username,
+	})
+	if err != nil {
+		return domain.Channel{}, fmt.Errorf("resolve username: %w", err)
+	}
+
+	if len(resolved.Chats) == 0 {
+		return domain.Channel{}, fmt.Errorf("no chats in resolve result")
+	}
+
+	ch, ok := resolved.Chats[0].(*tg.Channel)
+	if !ok {
+		return domain.Channel{}, fmt.Errorf("not a channel")
+	}
+
+	req := []tg.InputDialogPeerClass{
+		&tg.InputDialogPeer{
+			Peer: &tg.InputPeerChannel{
+				ChannelID:  ch.ID,
+				AccessHash: ch.AccessHash,
+			},
+		},
+	}
+
+	dialogs, err := t.API().MessagesGetPeerDialogs(ctx, req)
+	if err != nil {
+		return domain.Channel{}, fmt.Errorf("peer dialogs: %w", err)
+	}
+
+	pts := 0
+	if len(dialogs.Dialogs) > 0 {
+		d, ok := dialogs.Dialogs[0].(*tg.Dialog)
+		if ok {
+			pts = d.Pts
+		}
+	}
+
+	return domain.Channel{
+		TelegramID:   domain.ChatIDFromChannelID(ch.ID),
+		Username:     username,
+		Title:        ch.Title,
+		AccessHash:   ch.AccessHash,
+		Pts:          pts,
+		IsAccessible: true,
+	}, nil
+}

tg_parser/internal/adapter/telegram/resolve_invite.go

@@ -0,0 +1,133 @@
+package telegram
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/TelegramExchange/tgex-backend/tg_parser/internal/domain"
+	"github.com/gotd/td/telegram/deeplink"
+	"github.com/gotd/td/tg"
+)
+
+func (t *Telegram) ParseChannelMetaByInvite(ctx context.Context, inviteLink string) (domain.Channel, error) {
+	link, err := deeplink.Parse(inviteLink)
+	if err != nil {
+		return domain.Channel{}, fmt.Errorf("parse invite link: %w", err)
+	}
+	if link.Type != deeplink.Join {
+		return domain.Channel{}, fmt.Errorf("invite link is not a join link")
+	}
+	hash := link.Args.Get("invite")
+	if hash == "" {
+		return domain.Channel{}, fmt.Errorf("invite link missing hash")
+	}
+
+	info, err := t.API().MessagesCheckChatInvite(ctx, hash)
+	if err != nil {
+		return domain.Channel{}, fmt.Errorf("check invite: %w", err)
+	}
+
+	var channel *tg.Channel
+
+	switch v := info.(type) {
+	case *tg.ChatInviteAlready:
+		channel = extractChannelFromChat(v.Chat)
+	case *tg.ChatInvite:
+		updates, err := t.API().MessagesImportChatInvite(ctx, hash)
+		if err != nil {
+			return domain.Channel{}, fmt.Errorf("import invite: %w", err)
+		}
+		channel = extractChannelFromUpdates(updates)
+	case *tg.ChatInvitePeek:
+		// ChatInvitePeek means we can preview the channel without joining (public channels)
+		channel = extractChannelFromChat(v.Chat)
+	default:
+		return domain.Channel{}, fmt.Errorf("unexpected invite response: %T", v)
+	}
+
+	if channel == nil {
+		return domain.Channel{}, fmt.Errorf("no channel in invite response")
+	}
+	if channel.AccessHash == 0 {
+		return domain.Channel{}, fmt.Errorf("channel access hash missing")
+	}
+
+	pts, err := getChannelPTS(ctx, t.API(), channel)
+	if err != nil {
+		return domain.Channel{}, fmt.Errorf("get peer dialogs: %w", err)
+	}
+
+	username := ""
+	if usernameVal, ok := channel.GetUsername(); ok && usernameVal != "" {
+		username = usernameVal
+	}
+
+	return domain.Channel{
+		TelegramID:   domain.ChatIDFromChannelID(channel.ID),
+		Username:     username,
+		Title:        channel.Title,
+		AccessHash:   channel.AccessHash,
+		Pts:          pts,
+		InviteLink:   inviteLink,
+		IsAccessible: true,
+	}, nil
+}
+
+func extractChannelFromChat(chat tg.ChatClass) *tg.Channel {
+	switch v := chat.(type) {
+	case *tg.Channel:
+		return v
+	case *tg.ChannelForbidden:
+		return &tg.Channel{
+			ID:         v.ID,
+			AccessHash: v.AccessHash,
+			Title:      v.Title,
+		}
+	default:
+		return nil
+	}
+}
+
+func extractChannelFromUpdates(updates tg.UpdatesClass) *tg.Channel {
+	var chats []tg.ChatClass
+	switch v := updates.(type) {
+	case *tg.Updates:
+		chats = v.Chats
+	case *tg.UpdatesCombined:
+		chats = v.Chats
+	default:
+		return nil
+	}
+
+	for _, chat := range chats {
+		if channel := extractChannelFromChat(chat); channel != nil {
+			return channel
+		}
+	}
+
+	return nil
+}
+
+func getChannelPTS(ctx context.Context, api *tg.Client, channel *tg.Channel) (int, error) {
+	req := []tg.InputDialogPeerClass{
+		&tg.InputDialogPeer{
+			Peer: &tg.InputPeerChannel{
+				ChannelID:  channel.ID,
+				AccessHash: channel.AccessHash,
+			},
+		},
+	}
+
+	dialogs, err := api.MessagesGetPeerDialogs(ctx, req)
+	if err != nil {
+		return 0, err
+	}
+
+	if len(dialogs.Dialogs) > 0 {
+		if d, ok := dialogs.Dialogs[0].(*tg.Dialog); ok {
+			return d.Pts, nil
+		}
+	}
+
+	return 0, nil
+}

tg_parser/internal/adapter/telegram/telegram.go

@@ -0,0 +1,20 @@
+package telegram
+
+import (
+	"github.com/TelegramExchange/pkg/telegram"
+	"github.com/gotd/td/tg"
+)
+
+type Telegram struct {
+	client *telegram.Client
+}
+
+func New(client *telegram.Client) *Telegram {
+	return &Telegram{
+		client: client,
+	}
+}
+
+func (t *Telegram) API() *tg.Client {
+	return t.client.API()
+}