парсер

tg_parser/internal/adapter/telegram/get_channel_diff.go

@@ -59,7 +59,8 @@ func extractPosts(channel domain.Channel, msgs []tg.MessageClass) []domain.Post
 			continue
 		}
 
-		p := domain.NewPost(channel, m.ID, m.Message, m.Views)
+		text := messageToHTML(m.Message, m.Entities)
+		p := domain.NewPost(channel, m.ID, text, m.Views)
 		posts = append(posts, p)
 	}
 

tg_parser/internal/adapter/telegram/history.go

@@ -38,7 +38,8 @@ func (t *Telegram) GetChannelHistory(ctx context.Context, channel domain.Channel
 			continue
 		}
 
-		p := domain.NewPost(channel, m.ID, m.Message, m.Views)
+		text := messageToHTML(m.Message, m.Entities)
+		p := domain.NewPost(channel, m.ID, text, m.Views)
 		placements = append(placements, p)
 	}
 

tg_parser/internal/adapter/telegram/message_html.go

@@ -0,0 +1,197 @@
+package telegram
+
+import (
+	"html"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/gotd/td/tg"
+)
+
+type htmlEntity struct {
+	start    int
+	end      int
+	openTag  string
+	closeTag string
+	length   int
+}
+
+type htmlEvent struct {
+	pos     int
+	tag     string
+	isStart bool
+	length  int
+}
+
+func messageToHTML(text string, entities []tg.MessageEntityClass) string {
+	if text == "" {
+		return ""
+	}
+
+	if len(entities) == 0 {
+		return html.EscapeString(text)
+	}
+
+	ranges := make([]htmlEntity, 0, len(entities))
+	needed := make([]int, 0, len(entities)*2)
+
+	for _, raw := range entities {
+		offset, length, openTag, closeTag, ok := htmlEntityMeta(raw)
+		if !ok {
+			continue
+		}
+
+		start := offset
+		end := offset + length
+		if length <= 0 {
+			continue
+		}
+
+		ranges = append(ranges, htmlEntity{
+			start:    start,
+			end:      end,
+			openTag:  openTag,
+			closeTag: closeTag,
+			length:   length,
+		})
+		needed = append(needed, start, end)
+	}
+
+	if len(ranges) == 0 {
+		return html.EscapeString(text)
+	}
+
+	positions := utf16PositionsToBytes(text, needed)
+	events := make([]htmlEvent, 0, len(ranges)*2)
+
+	for _, r := range ranges {
+		startByte, okStart := positions[r.start]
+		endByte, okEnd := positions[r.end]
+		if !okStart || !okEnd || startByte > endByte {
+			continue
+		}
+
+		events = append(events, htmlEvent{
+			pos:     startByte,
+			tag:     r.openTag,
+			isStart: true,
+			length:  r.length,
+		})
+		events = append(events, htmlEvent{
+			pos:     endByte,
+			tag:     r.closeTag,
+			isStart: false,
+			length:  r.length,
+		})
+	}
+
+	sort.SliceStable(events, func(i, j int) bool {
+		if events[i].pos != events[j].pos {
+			return events[i].pos < events[j].pos
+		}
+		if events[i].isStart != events[j].isStart {
+			return !events[i].isStart
+		}
+		if events[i].isStart {
+			return events[i].length > events[j].length
+		}
+		return events[i].length < events[j].length
+	})
+
+	var b strings.Builder
+	last := 0
+	for _, ev := range events {
+		if ev.pos > last {
+			b.WriteString(html.EscapeString(text[last:ev.pos]))
+		}
+		b.WriteString(ev.tag)
+		last = ev.pos
+	}
+	if last < len(text) {
+		b.WriteString(html.EscapeString(text[last:]))
+	}
+
+	return b.String()
+}
+
+func htmlEntityMeta(entity tg.MessageEntityClass) (offset int, length int, openTag string, closeTag string, ok bool) {
+	switch e := entity.(type) {
+	case *tg.MessageEntityBold:
+		return e.Offset, e.Length, "<b>", "</b>", true
+	case *tg.MessageEntityItalic:
+		return e.Offset, e.Length, "<i>", "</i>", true
+	case *tg.MessageEntityUnderline:
+		return e.Offset, e.Length, "<u>", "</u>", true
+	case *tg.MessageEntityStrike:
+		return e.Offset, e.Length, "<s>", "</s>", true
+	case *tg.MessageEntityCode:
+		return e.Offset, e.Length, "<code>", "</code>", true
+	case *tg.MessageEntityPre:
+		if e.Language != "" {
+			lang := html.EscapeString(e.Language)
+			return e.Offset, e.Length, `<pre><code class="language-` + lang + `">`, "</code></pre>", true
+		}
+		return e.Offset, e.Length, "<pre>", "</pre>", true
+	case *tg.MessageEntityTextURL:
+		url := html.EscapeString(e.URL)
+		return e.Offset, e.Length, `<a href="` + url + `">`, "</a>", true
+	case *tg.MessageEntityMentionName:
+		userID := html.EscapeString(strconv.FormatInt(e.UserID, 10))
+		return e.Offset, e.Length, `<a href="tg://user?id=` + userID + `">`, "</a>", true
+	case *tg.MessageEntitySpoiler:
+		return e.Offset, e.Length, `<span class="tg-spoiler">`, "</span>", true
+	case *tg.MessageEntityBlockquote:
+		if e.Collapsed {
+			return e.Offset, e.Length, `<blockquote expandable>`, "</blockquote>", true
+		}
+		return e.Offset, e.Length, "<blockquote>", "</blockquote>", true
+	case *tg.MessageEntityCustomEmoji:
+		id := html.EscapeString(strconv.FormatInt(e.DocumentID, 10))
+		return e.Offset, e.Length, `<tg-emoji emoji-id="` + id + `">`, "</tg-emoji>", true
+	default:
+		return 0, 0, "", "", false
+	}
+}
+
+func utf16PositionsToBytes(text string, needed []int) map[int]int {
+	result := make(map[int]int, len(needed))
+	needSet := make(map[int]struct{}, len(needed))
+	for _, n := range needed {
+		needSet[n] = struct{}{}
+	}
+
+	utf16Pos := 0
+	if _, ok := needSet[0]; ok {
+		result[0] = 0
+	}
+
+	for i, r := range text {
+		if _, ok := needSet[utf16Pos]; ok {
+			result[utf16Pos] = i
+		}
+
+		step := utf16RuneLen(r)
+		if step == 2 {
+			if _, ok := needSet[utf16Pos+1]; ok {
+				result[utf16Pos+1] = i
+			}
+		}
+		utf16Pos += step
+	}
+
+	if _, ok := needSet[utf16Pos]; ok {
+		result[utf16Pos] = len(text)
+	}
+
+	return result
+}
+
+func utf16RuneLen(r rune) int {
+	const surrSelf = 0x10000
+	if r >= surrSelf && r <= utf8.MaxRune {
+		return 2
+	}
+	return 1
+}