diff --git a/README.md b/README.md index 062b393..c7d4bd2 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,31 @@ redirects `sys.stdout`/`sys.stderr` to the logcat tags `python.stdout`/`python.s This repo does not use the official Elyx/ElyxBuilder archive format — see `docs/sdk-notes.md` for why. +## What the plugin does + +**Summarize** (⋮ → Plugins → Summarize, or hold the unread button) runs in this order: + +1. A bulletin confirms the press, then the history is fetched. +2. The transcript streams to the configured OpenAI-compatible endpoint. +3. Only when the first chunk arrives is a summary card injected, directly after the newest + analyzed message. If that message is not loaded (an offset, or history scrolled away), + the chat is asked to load it first. The chat then jumps to the card once. +4. Reasoning and answer stream into the card. The finished summary ends with a native + `#aisummaries` hashtag and is saved to a local SQLite store, from which it is restored + whenever the chat is reopened. + +Any failure before step 3 (fetch error, empty history, unreachable host, bad key, HTTP +error) is reported as a bulletin only: the chat is not modified and the view does not move. +A failure mid-stream is written onto the card, and a summary saved earlier for the same +range is kept. + +Deleting a card like any other message removes its saved row. The settings page shows how +many summaries are saved and roughly how much space they take, and offers **Clear all +summaries** (with a confirmation step). + +Summaries exist only on this device, so Telegram's own search, which runs on the server, +does not find them, including a search for `#aisummaries`. + ## Repo layout ``` @@ -38,7 +63,8 @@ src/ the plugin, written as an ordinary Python package diagnostics.py bounded in-memory log, mirrored to a file (logcat is restricted here) providers/ endpoint settings, prompt assembly and SSE streaming services/ history fetch, anonymized transcript, markdown→RichMessage, injection, SQLite store - hooks/ Xposed hooks (unread-button entry points, summary restore on chat open) + hooks/ Xposed hooks (unread-button entry points, summary restore on chat open, + forgetting a summary when its card is deleted) ui/ summarize dialog, bulletin and theme helpers tools/ bundle.py compiles plugin.json + src/ into dist/.plugin diff --git a/docs/sdk-notes.md b/docs/sdk-notes.md index a0b909d..984c943 100644 --- a/docs/sdk-notes.md +++ b/docs/sdk-notes.md @@ -78,6 +78,41 @@ plugins that need to reach beyond the public SDK surface. - `intents.IntentsManager` — Android intent integration, **exteraGram 12.6.4+ only**. +## Host internals relied on by the summary card + +Verified against exteraGram's `ChatActivity`/`MessagesController` sources and the 12.5.x dex. + +- **Scrolling.** `ChatActivity.scrollToMessageId(id, 0, false, 0, true, 0)`: for an id that + is not loaded it requests history around it and jumps there when it arrives; for a loaded + one it only `smoothScrollBy`s from the current cell positions, which are stale right after + an insert. To land on a row, set the layout manager's pending position instead: + `chatLayoutManager.scrollToPositionWithOffset(chatAdapter.messagesStartRow + index, + offset, false)` (`bottom=false` counts `offset` from the top edge). `messagesStartRow` is + public and current after `notifyDataSetChanged()`, which runs `updateRowsInternal()`. +- **Deleting.** Every delete goes through `MessagesController.deleteMessages` (4 overloads, + all starting `(messages, randoms, encryptedChat, dialogId, …)`). Only ids `> 0` are sent to + the server; the rest are deleted from the local cache and posted in + `NotificationCenter.messagesDeleted`. `ChatActivity` removes deleted rows by looking them up + in `messagesDict`, which injected cards are never added to, so the plugin removes them. +- **Hashtags in rich text.** `TL_iv.textHashtag` (single field `text`) is rendered by + `RichMessageLayout.formatText` exactly like a `TL_messageEntityHashtag` in a normal + message (`URLSpanNoUnderline`). Long-pressing it offers Open / Copy hashtag. A `textUrl` + whose URL starts with `#` is not a hashtag: it goes through the external-URL path and does + nothing. +- **Search.** In-chat and hashtag search are server-side (`messages.search`); local results + exist only for Saved Messages tags. A locally injected message cannot appear in them. + +## SDK UI details used here + +- `ui.settings.Text(text, subtext=, icon=, accent=, red=, on_click=)`, where `on_click` + receives the clicked `View`. `set_setting(key, value, reload_settings=True)` re-renders + the settings page. +- `AlertDialogBuilder(activity)`; button listeners receive `(builder, which)`; + `make_button_red(AlertDialogBuilder.BUTTON_POSITIVE)` after `show()`. A dismissed builder + cannot be shown again. +- `BulletinHelper.show_*` post to the UI thread themselves. +- `android_utils.run_on_ui_thread(fn, delay)`, where `delay` is in milliseconds. + ## On-device dev server protocol diff --git a/plugin.json b/plugin.json index 090cbfa..8df7d54 100644 --- a/plugin.json +++ b/plugin.json @@ -3,7 +3,7 @@ "name": "Chat Summaries", "description": "Summarize the last N messages (with offset) in any chat, group, channel, or topic using AI.", "author": "@lyuksovannyy", - "version": "1.0.3", + "version": "1.1.0", "icon": "exteraPlugins/1", "app_version": ">=12.5.1", "sdk_version": ">=1.4.4.3", diff --git a/src/config.py b/src/config.py index 02665f4..c411e81 100644 --- a/src/config.py +++ b/src/config.py @@ -28,6 +28,9 @@ SETTING_DEFAULT_COUNT = "default_count" SETTING_DEFAULT_OFFSET = "default_offset" SETTING_GLOBAL_PROMPT = "global_prompt" SETTING_INCLUDE_ATTACHMENTS = "include_attachments" +# Not a user setting: writing it with reload_settings=True is how the +# settings page is re-rendered after the store changes. +SETTING_STORAGE_REVISION = "storage_revision" DEFAULT_PROMPT = ( "Summarize the following chat conversation. Focus on key points, " @@ -46,3 +49,14 @@ LONG_PRESS_SECONDS = 0.4 # rebuilds the RichMessage blocks and rebinds the cell, so pushing on every SSE # chunk would burn the UI thread for no visible gain. STREAM_UPDATE_INTERVAL = 0.25 + +# Appended to every finished summary. Lets a reader tell a local AI summary +# from a real message at a glance, and gives copied or forwarded text a +# searchable marker. +SUMMARY_HASHTAG = "#aisummaries" + +# A summary card can only be placed directly after its newest analyzed +# message, which with an offset may not be loaded yet. Placement asks the chat +# to load it, then retries this many times, this many ms apart. +PLACEMENT_RETRIES = 25 +PLACEMENT_RETRY_DELAY_MS = 120 diff --git a/src/hooks/delete_hook.py b/src/hooks/delete_hook.py new file mode 100644 index 0000000..9960bab --- /dev/null +++ b/src/hooks/delete_hook.py @@ -0,0 +1,84 @@ +"""Forgets a summary when the user deletes its card. + +Deleting goes through MessagesController.deleteMessages, whichever menu it +starts from. Injected cards have negative ids, which that method never sends +to the server (only ids > 0 are), so letting the call through is harmless; it +just never touches the plugin's own store. This hook does: any deleted id that +is a stored summary's virtual id removes that row, so the card does not come +back the next time the chat is opened. + +Every deleteMessages overload is hooked. They delegate to one another, so one +delete can be seen several times; handling is idempotent and only the call +that actually removed a row reports it. +""" +from typing import Any, List + +import diagnostics +from hooks.restore_hook import _find_methods_by_name + + +def _java_ints(value: Any) -> List[int]: + """ArrayList as Chaquopy exposes it: size()/get(), not iterable.""" + if value is None: + return [] + try: + return [int(value.get(i)) for i in range(value.size())] + except Exception: + try: + return [int(v) for v in value] + except Exception: + return [] + + +class DeleteHook: + def __init__(self, plugin): + self.plugin = plugin + self._unhooks: List[Any] = [] + + def install(self) -> None: + try: + from base_plugin import MethodHook + from hook_utils import find_class + except Exception as exc: + diagnostics.record_error("delete_hook.install:imports", exc) + return + + controller_class = find_class("org.telegram.messenger.MessagesController") + if not controller_class: + diagnostics.note("delete_hook.install: MessagesController not found") + return + methods = _find_methods_by_name(controller_class, "deleteMessages") + diagnostics.note(f"delete_hook.install: deleteMessages overloads found={len(methods)}") + + plugin = self.plugin + + class _OnDelete(MethodHook): + def before_hooked_method(self, param): + try: + args = list(getattr(param, "args", None) or []) + # Every overload starts (messages, randoms, encryptedChat, + # dialogId, ...). + if len(args) < 4: + return + ids = [i for i in _java_ints(args[0]) if i < 0] + if not ids: + return + plugin.on_cards_deleted(int(args[3] or 0), ids) + except Exception as exc: + diagnostics.record_error("delete_hook.before_hooked_method", exc) + + for method in methods: + try: + ref = self.plugin.hook_method(method, _OnDelete()) + if ref: + self._unhooks.append(ref) + except Exception as exc: + diagnostics.record_error("delete_hook.install:hook_method", exc) + + def uninstall(self) -> None: + for ref in self._unhooks: + try: + ref.unhook() + except Exception: + pass + self._unhooks = [] diff --git a/src/main.py b/src/main.py index ae405d5..1195861 100644 --- a/src/main.py +++ b/src/main.py @@ -21,8 +21,10 @@ from config import ( SETTING_DEFAULT_OFFSET, SETTING_GLOBAL_PROMPT, SETTING_INCLUDE_ATTACHMENTS, + SETTING_STORAGE_REVISION, STREAM_UPDATE_INTERVAL, ) +from hooks.delete_hook import DeleteHook from hooks.restore_hook import RestoreHook from hooks.unread_hook import UnreadHook from providers.dispatcher import Dispatcher @@ -37,6 +39,7 @@ class ChatSummariesPlugin(BasePlugin): self.dispatcher = Dispatcher(self) self.unread_hook = UnreadHook(self) self.restore_hook = RestoreHook(self) + self.delete_hook = DeleteHook(self) self.summary_db = None def _get_summary_db(self): @@ -83,10 +86,15 @@ class ChatSummariesPlugin(BasePlugin): self.restore_hook.install() except Exception as exc: diagnostics.record_error("on_plugin_load:restore_hook.install", exc) + try: + self.delete_hook.install() + except Exception as exc: + diagnostics.record_error("on_plugin_load:delete_hook.install", exc) def on_plugin_unload(self) -> None: self.unread_hook.uninstall() self.restore_hook.uninstall() + self.delete_hook.uninstall() log("chat_summaries unloaded") def create_settings(self): @@ -140,9 +148,22 @@ class ChatSummariesPlugin(BasePlugin): "summary can mention shared files. Message text is sent either way.", icon="msg_photos", ), + Divider(text="Storage"), + Text( + text="Saved summaries", + subtext=self._storage_summary(), + icon="msg_storage_usage", + ), + Text( + text="Clear all summaries", + icon="msg_delete", + red=True, + on_click=self._on_clear_all_click, + ), + Divider(), Text( text="About", - subtext="Summarize via ⋮ → Plugins → Summarize, or hold the unread button to auto-fill unread messages. Diagnostics are also available from that menu.", + subtext="Summarize via ⋮ → Plugins → Summarize, or hold the unread button to auto-fill unread messages. Diagnostics are also available from that menu. Delete a summary card like any message to forget it.", icon="msg_info", ), ] @@ -245,117 +266,93 @@ class ChatSummariesPlugin(BasePlugin): offset: int, prompt: str, ) -> None: - """Shows the summary card immediately, then drives it through every - stage: fetching history, contacting the endpoint, streaming reasoning, - streaming the answer. + """Fetches history, then streams the summary into a card. - The card is injected before the fetch so the click has instant visible - feedback. Its anchor is provisional until the fetch returns (the newest - analyzed message id is not known yet) and is corrected via reanchor(). + Nothing is injected into the chat until the endpoint has actually + started answering. Every failure before that point -- history fetch, + unreachable host, bad key, HTTP error, empty history -- is reported as a + bulletin and leaves the chat exactly as the user left it: no error + bubble to dismiss, no jump away from where they were reading. """ - from services.rich_injector import SummaryInjection, virtual_id_for - - account = self._get_account() - # With a non-zero offset the analyzed range ends partway up the - # history, so skip that many messages when guessing where the card - # goes. reanchor() corrects it exactly once the fetch returns. - provisional_anchor = self._newest_visible_message_id(fragment, skip=offset) - - injection = SummaryInjection( - fragment, - account, - dialog_id, - topic_id, - provisional_anchor, - virtual_id_for(0, provisional_anchor or 1), - ) - if not injection.insert(status="Fetching messages…", streaming=True): - BulletinHelper.show_error("Chat Summaries: cannot inject into this chat", fragment) - return - - def fail(message: str) -> None: - injection.update(answer_markdown=f"**Error:** {message}", streaming=False) - BulletinHelper.show_error(f"Chat Summaries: {message}", fragment) + BulletinHelper.show_info(f"Chat Summaries: reading {count} messages…", fragment) def on_fetched(messages, error): if error: - fail(str(error)) + self._report_error(str(error), fragment) return if not messages: - fail("no messages found") + self._report_error("no messages found", fragment) return try: - self._stream_into(injection, fragment, dialog_id, topic_id, prompt, messages) + self._stream_into(fragment, dialog_id, topic_id, prompt, messages) except Exception as exc: diagnostics.record_error("on_fetched:_stream_into", exc) - fail(str(exc)) + self._report_error(str(exc), fragment) self.fetcher.fetch_history(dialog_id, count, offset, topic_id, on_fetched) - def _newest_visible_message_id(self, chat_activity: Any, skip: int = 0) -> int: - """Provisional anchor: the newest real message the analysis will cover. - - `skip` mirrors the summarize offset, so with an offset the guess lands - where the analyzed range actually ends instead of at the bottom of the - chat. Injected cards carry negative ids and are ignored. - - When the offset reaches past the loaded window there is no message to - skip to, so fall back to the oldest loaded message rather than 0: the - card only needs somewhere valid to appear while "Fetching messages…" is - shown, and reanchor() moves it to the exact spot once the range is - known. Returning 0 would make the insert fail and the user would click - Summarize and see nothing happen. - """ - try: - from services.rich_injector import get_messages_list - - messages_list = get_messages_list(chat_activity) - if messages_list is None: - return 0 - remaining = max(0, int(skip)) - oldest = 0 - for i in range(messages_list.size()): - owner = getattr(messages_list.get(i), "messageOwner", None) - msg_id = int(getattr(owner, "id", 0) or 0) if owner is not None else 0 - if msg_id <= 0: - continue - oldest = msg_id - if remaining: - remaining -= 1 - continue - return msg_id - return oldest - except Exception as exc: - diagnostics.record_error("_newest_visible_message_id", exc) - return 0 + def _report_error(self, message: str, fragment: Any) -> None: + BulletinHelper.show_error(f"Chat Summaries: {message}", fragment) def _stream_into( self, - injection: Any, fragment: Any, dialog_id: int, topic_id: int, prompt: str, messages: list, ) -> None: - """Streams reasoning, then the answer, into the already-visible card.""" - from services.rich_injector import virtual_id_for + """Streams reasoning, then the answer, into a card created on the + first chunk. + + The card goes directly after the newest analyzed message and the chat + jumps to it once, when it is placed. + """ + from services.rich_injector import SummaryInjection, virtual_id_for anchor_msg_id = max(int(getattr(m, "id", 0) or 0) for m in messages) - db = self._get_summary_db() - # Reserve the row now so the virtual id is stable for the whole stream. - summary_id = 0 - if db is not None: - try: - summary_id = db.save(dialog_id, topic_id, anchor_msg_id, "", "", len(messages)) - except Exception as exc: - diagnostics.record_error("_stream_into:db.reserve", exc) + state = { + "answer": "", + "thinking": "", + "last_push": 0.0, + "mapping": None, + "injection": None, + "summary_id": 0, + "created_row": False, + } - injection.reanchor(anchor_msg_id, virtual_id_for(summary_id, anchor_msg_id)) - injection.update(status=f"Summarizing {len(messages)} messages…", streaming=True) - - state = {"answer": "", "thinking": "", "last_push": 0.0, "mapping": None} + def open_card(): + """Creates and places the card. Runs once, on the first chunk.""" + if state["injection"] is not None: + return state["injection"] + # Reserve the row now so the virtual id is stable for the whole + # stream, and matches the card the restore hook may already show + # for this range. + if db is not None: + try: + state["summary_id"], state["created_row"] = db.reserve( + dialog_id, topic_id, anchor_msg_id, len(messages) + ) + except Exception as exc: + diagnostics.record_error("_stream_into:db.reserve", exc) + injection = SummaryInjection( + fragment, + self._get_account(), + dialog_id, + topic_id, + anchor_msg_id, + virtual_id_for(state["summary_id"], anchor_msg_id), + ) + state["injection"] = injection + if not injection.insert( + answer_markdown=rendered("answer"), thinking=rendered("thinking"), streaming=True + ): + # The summary is still saved when it finishes, and the restore + # hook shows it once the chat can take it. + BulletinHelper.show_error("Chat Summaries: cannot show the summary in this chat", fragment) + state["last_push"] = _now() + return injection def rendered(key: str, final: bool = False) -> str: """Placeholder-substituted view of `answer` or `thinking`. @@ -378,18 +375,17 @@ class ChatSummariesPlugin(BasePlugin): return text return deanonymize(text, mapping, dialog_id) - def push(force: bool = False) -> None: + def push(force: bool = False, tagged: bool = False) -> None: """Rate-limits re-renders; each one rebuilds blocks and rebinds cells.""" - import time - - now = time.monotonic() + now = _now() if not force and now - state["last_push"] < STREAM_UPDATE_INTERVAL: return state["last_push"] = now - injection.update( + state["injection"].update( answer_markdown=rendered("answer", final=force), thinking=rendered("thinking", final=force), streaming=not force, + tagged=tagged, ) def on_mapping(mapping) -> None: @@ -400,18 +396,32 @@ class ChatSummariesPlugin(BasePlugin): state["thinking"] += text else: state["answer"] += text - push() + if state["injection"] is None: + open_card() + else: + push() def on_done(answer: Optional[str], thinking: Optional[str], error: Optional[str]) -> None: if error: + self._report_error(error, fragment) + if state["injection"] is None: + # Failed before the first chunk: the chat was never touched. + return state["answer"] = (state["answer"] or "") + f"\n\n**Error:** {error}" push(force=True) - BulletinHelper.show_error(f"Chat Summaries: {error}", fragment) + # A row created for this stream would never get text; don't + # leave it behind. A pre-existing summary keeps its text. + if db is not None and state["created_row"] and state["summary_id"]: + try: + db.delete(state["summary_id"]) + except Exception as exc: + diagnostics.record_error("_stream_into:db.delete", exc) return state["answer"] = answer or state["answer"] state["thinking"] = thinking or state["thinking"] - push(force=True) - if db is not None and summary_id: + open_card() + push(force=True, tagged=True) + if db is not None and state["summary_id"]: try: # Persist real names and links, not placeholders: the # mapping is per-request and would be gone by the time the @@ -425,10 +435,7 @@ class ChatSummariesPlugin(BasePlugin): # The restore gate skips scopes it has already handled, so # the newly saved row has to invalidate that cache or it # would not be restored until the chat was reopened. - try: - self.restore_hook.invalidate() - except Exception: - pass + self.restore_hook.invalidate() except Exception as exc: diagnostics.record_error("_stream_into:db.save", exc) @@ -442,6 +449,90 @@ class ChatSummariesPlugin(BasePlugin): topic_id=topic_id, ) + # ------------------------------------------------------------------ + # Deleting a card / clearing the store + # ------------------------------------------------------------------ + + def on_cards_deleted(self, dialog_id: int, deleted_ids: list) -> None: + """Called by DeleteHook with the negative ids being deleted in a chat.""" + from client_utils import get_last_fragment + from services.rich_injector import remove_cards, virtual_id_for + + db = self._get_summary_db() + if db is None or not dialog_id: + return + wanted = {int(i) for i in deleted_ids} + removed = [] + try: + for summary in db.get_for_dialog(dialog_id): + virtual_id = virtual_id_for(summary.id, summary.anchor_msg_id) + if virtual_id in wanted and db.delete(summary.id): + removed.append(virtual_id) + except Exception as exc: + diagnostics.record_error("on_cards_deleted", exc) + return + if not removed: + return + # The restore cache still holds the deleted rows; without this the + # card would be re-injected on the next frame. + self.restore_hook.invalidate() + fragment = get_last_fragment() + remove_cards(fragment, removed) + BulletinHelper.show_info( + "Summary deleted" if len(removed) == 1 else f"{len(removed)} summaries deleted", + fragment, + ) + + def _storage_summary(self) -> str: + db = self._get_summary_db() + if db is None: + return "Storage unavailable" + try: + count, size = db.stats() + except Exception as exc: + diagnostics.record_error("_storage_summary", exc) + return "Storage unavailable" + noun = "summary" if count == 1 else "summaries" + return f"{count} {noun} saved · about {_format_size(size)}" + + def _on_clear_all_click(self, view: Any = None) -> None: + try: + from client_utils import get_last_fragment + from ui.alert import AlertDialogBuilder + + fragment = get_last_fragment() + activity = fragment.getParentActivity() if fragment is not None else None + if activity is None: + return + builder = AlertDialogBuilder(activity) + builder.set_title("Clear all summaries?") + builder.set_message( + "Every saved summary in every chat will be deleted. This cannot be undone." + ) + builder.set_positive_button("Clear", lambda b, w: self._clear_all_summaries()) + builder.set_negative_button("Cancel", lambda b, w: b.dismiss()) + builder.show() + builder.make_button_red(AlertDialogBuilder.BUTTON_POSITIVE) + except Exception as exc: + diagnostics.record_error("_on_clear_all_click", exc) + + def _clear_all_summaries(self) -> None: + db = self._get_summary_db() + if db is None: + return + try: + removed = db.clear_all() + except Exception as exc: + diagnostics.record_error("_clear_all_summaries", exc) + BulletinHelper.show_error("Chat Summaries: could not clear summaries") + return + self.restore_hook.invalidate() + # Re-render the settings page so the size row shows the new total. + self.set_setting(SETTING_STORAGE_REVISION, str(_now()), reload_settings=True) + BulletinHelper.show_success( + "Deleted 1 summary" if removed == 1 else f"Deleted {removed} summaries" + ) + def _fetch_reply_context(self, messages: list) -> dict: """Reply parents that fall outside the analyzed range. @@ -519,3 +610,17 @@ class ChatSummariesPlugin(BasePlugin): def _save_prompt_for(self, dialog_id: int, topic_id: int, prompt: str) -> None: if prompt: self.set_setting(self._prompt_key(dialog_id, topic_id), prompt) + + +def _now() -> float: + import time + + return time.monotonic() + + +def _format_size(size: int) -> str: + if size < 1024: + return f"{size} B" + if size < 1024 * 1024: + return f"{size / 1024:.0f} KB" + return f"{size / (1024 * 1024):.1f} MB" diff --git a/src/services/rich_injector.py b/src/services/rich_injector.py index 292cc44..3f1065f 100644 --- a/src/services/rich_injector.py +++ b/src/services/rich_injector.py @@ -16,7 +16,8 @@ Verified against exteraGram 12.5.x (dex inspection): from typing import Any, Optional import diagnostics -from services.rich_markdown import build_rich_message +from config import PLACEMENT_RETRIES, PLACEMENT_RETRY_DELAY_MS, SUMMARY_HASHTAG +from services.rich_markdown import build_rich_message, hashtag_block, markdown_to_plain # Above Telegram's issued id range, so this user exists only locally. MOCK_SENDER_ID = 8000000000000001 @@ -108,8 +109,15 @@ def build_summary_message( thinking: str = "", streaming: bool = False, status: str = "", + tagged: bool = False, ) -> Optional[Any]: - """Builds the local MessageObject for a summary, or None on failure.""" + """Builds the local MessageObject for a summary, or None on failure. + + `tagged` marks a finished summary: SUMMARY_HASHTAG is appended to the + card as a native hashtag node (TL_iv.textHashtag), and to the message's + plain-text body with a TL_messageEntityHashtag. The tag is added here rather than stored, so every + summary restored from the DB carries it too. + """ try: import time @@ -124,7 +132,6 @@ def build_summary_message( msg.unread = False msg.post = False msg.flags = 0 - msg.message = "" msg.peer_id = peer_for_dialog(dialog_id) sender = TLRPC.TL_peerUser() @@ -143,7 +150,12 @@ def build_summary_message( except Exception as exc: diagnostics.record_error("injector.reply_header", exc) + tagged = tagged and bool((answer_markdown or "").strip()) + _set_plain_body(msg, answer_markdown, tagged, TLRPC) + msg.rich_message = build_rich_message(answer_markdown, thinking, streaming, status) + if tagged: + msg.rich_message.blocks.add(hashtag_block(SUMMARY_HASHTAG)) msg.flags2 = int(getattr(msg, "flags2", 0) or 0) | FLAGS2_RICH_MESSAGE message_object = MessageObject(account, msg, True, True) @@ -156,6 +168,33 @@ def build_summary_message( return None +def _utf16_len(text: str) -> int: + """Length in UTF-16 code units, which is what Telegram entity offsets count.""" + return len(text.encode("utf-16-le")) // 2 + + +def _set_plain_body(msg: Any, answer_markdown: str, tagged: bool, TLRPC: Any) -> None: + """Fills `message` with the summary as plain text. + + The card renders from rich_message, but copy, translate and any text + matching in the app read `message`. Left empty, a summary is invisible to + all of them. + """ + text = markdown_to_plain(answer_markdown) + if tagged: + text = (text + "\n\n" + SUMMARY_HASHTAG) if text else SUMMARY_HASHTAG + msg.message = text + if not tagged: + return + try: + entity = TLRPC.TL_messageEntityHashtag() + entity.offset = _utf16_len(text) - _utf16_len(SUMMARY_HASHTAG) + entity.length = _utf16_len(SUMMARY_HASHTAG) + msg.entities.add(entity) + except Exception as exc: + diagnostics.record_error("injector.hashtag_entity", exc) + + def get_messages_list(chat_activity: Any) -> Optional[Any]: """Returns the ChatActivity's live ArrayList, or None.""" try: @@ -260,7 +299,12 @@ def notify_adapter(chat_activity: Any) -> None: class SummaryInjection: - """A single injected summary message, updatable in place while streaming.""" + """A single injected summary message, updatable in place while streaming. + + Created only once the summary has started arriving, with its final anchor + and virtual id already known: nothing about it is provisional, so it is + placed exactly once, directly after the newest analyzed message. + """ def __init__(self, chat_activity: Any, account: int, dialog_id: int, topic_id: int, anchor_msg_id: int, virtual_id: int): self.chat_activity = chat_activity @@ -270,25 +314,13 @@ class SummaryInjection: self.anchor_msg_id = int(anchor_msg_id) self.virtual_id = int(virtual_id) self.message_object: Optional[Any] = None - # Every id this injection has ever placed in the list. reanchor() - # changes the id mid-flight, so an update must be able to find and - # remove the card under its previous id instead of adding a second one. - self._used_ids = {int(virtual_id)} + # True once the card is in the list. Until then renders only replace + # message_object, which placement reads when it finally lands. + self.placed = False + self._placing = False - def insert( - self, - answer_markdown: str = "", - thinking: str = "", - streaming: bool = True, - status: str = "", - ) -> bool: - messages_list = get_messages_list(self.chat_activity) - if messages_list is None: - diagnostics.note("injector: ChatActivity messages list not found") - return False - - register_mock_sender(self.account) - message_object = build_summary_message( + def _build(self, answer_markdown: str, thinking: str, streaming: bool, status: str, tagged: bool) -> Optional[Any]: + return build_summary_message( self.account, self.dialog_id, self.topic_id, @@ -298,81 +330,143 @@ class SummaryInjection: thinking, streaming, status, + tagged, ) + + def insert( + self, + answer_markdown: str = "", + thinking: str = "", + streaming: bool = True, + status: str = "", + tagged: bool = False, + ) -> bool: + """Places the card after its anchor and jumps the chat to it. + + Returns False only when the chat cannot take an injected card at all. + An anchor that is not loaded yet is not a failure: the chat is asked to + load it and placement retries until it arrives. + """ + if get_messages_list(self.chat_activity) is None: + diagnostics.note("injector: ChatActivity messages list not found") + return False + + register_mock_sender(self.account) + message_object = self._build(answer_markdown, thinking, streaming, status, tagged) if message_object is None: return False self.message_object = message_object - - # The index must be computed on the UI thread, immediately before the - # mutation that uses it. Computing it here would race: loading a large - # range takes long enough for paging or an incoming message to shift the - # list, and the stale index then lands the card after the wrong message. - def run(): - try: - messages_list_now = get_messages_list(self.chat_activity) - if messages_list_now is None: - return - existing = find_existing(messages_list_now, self.virtual_id) - if existing >= 0: - messages_list_now.set(existing, message_object) - else: - # A live card is a direct response to the user pressing - # Summarize, so it must become visible even when the - # provisional anchor is not loaded (a large offset reaches - # past the loaded window). Fall back to id ordering, which - # still keeps every newer message below the card, and let - # reanchor() move it to the exact spot once the analyzed - # range is known. Restoring saved summaries stays strict -- - # there is no follow-up correction there. - index = find_index_for_anchor(messages_list_now, self.anchor_msg_id) - if index < 0: - index = find_index_for_anchor( - messages_list_now, self.anchor_msg_id, require_anchor=False - ) - if index < 0: - # No anchor at all (nothing loaded yet, or the offset - # reached past everything). Show the card at the bottom, - # where the user is looking after pressing Summarize; - # reanchor() relocates it once the range is known. - diagnostics.note( - f"injector: anchor {self.anchor_msg_id} unresolved, " - "showing status card at the bottom" - ) - index = 0 - messages_list_now.add(index, message_object) - adapter = get_chat_adapter(self.chat_activity) - if adapter is not None: - adapter.notifyDataSetChanged() - # The analyzed range can end far up the history, so the card - # may be injected off-screen. Bring it into view once, on the - # first insert only -- scrolling on every streaming update - # would fight the user for control of the list. - if existing < 0: - self.scroll_into_view() - except Exception as exc: - diagnostics.record_error("injector.insert", exc) - - _on_ui(run) + self._placing = True + _on_ui(lambda: self._place(PLACEMENT_RETRIES)) return True - def scroll_into_view(self) -> bool: - """Scrolls the chat so the summary card is visible. - - Targets the anchor, not the card: the anchor is a real message the app - can resolve and load, and the card sits directly above it, so both end - up on screen. ChatActivity.scrollToMessageId(id, fromMessageId, - select, loadIndex, forceScroll, unreadCount) -- arity verified in the - APK. + def _place(self, retries_left: int) -> None: + """Runs on the UI thread. The index is computed immediately before the + mutation that uses it: computed earlier, paging or an incoming message + could shift the list and land the card after the wrong message. """ - if not self.anchor_msg_id: - return False try: - self.chat_activity.scrollToMessageId( - int(self.anchor_msg_id), 0, False, 0, True, 0 + messages_list = get_messages_list(self.chat_activity) + if messages_list is None: + self._placing = False + return + message_object = self.message_object + index = find_existing(messages_list, self.virtual_id) + if index >= 0: + # A card for this summary is already on screen -- the restore + # hook re-injected the saved row being re-summarized. Claim it. + messages_list.set(index, message_object) + else: + index = find_index_for_anchor(messages_list, self.anchor_msg_id) + if index < 0: + if retries_left == PLACEMENT_RETRIES: + # The analyzed range ends outside the loaded window + # (an offset, or history scrolled away). Have the chat + # load and show it, then place once it is in the list. + self._load_anchor() + if retries_left > 0: + _on_ui(lambda: self._place(retries_left - 1), PLACEMENT_RETRY_DELAY_MS) + else: + self._placing = False + diagnostics.note( + f"injector: anchor {self.anchor_msg_id} never loaded, card not placed" + ) + return + messages_list.add(index, message_object) + # A render may have landed while this was queued; show the newest. + if self.message_object is not message_object: + messages_list.set(index, self.message_object) + self.placed = True + self._placing = False + adapter = get_chat_adapter(self.chat_activity) + if adapter is not None: + adapter.notifyDataSetChanged() + # Once, on placement only -- scrolling on every streaming update + # would fight the user for control of the list. + self.scroll_into_view() + except Exception as exc: + self._placing = False + diagnostics.record_error("injector.place", exc) + + def _load_anchor(self) -> None: + """Asks the chat to load the anchor and scroll to it. + + ChatActivity.scrollToMessageId(id, fromMessageId, select, loadIndex, + forceScroll, unreadCount) -- arity verified in the APK. For an + unloaded id it requests history around it and jumps there on arrival. + """ + try: + self.chat_activity.scrollToMessageId(int(self.anchor_msg_id), 0, False, 0, True, 0) + except Exception as exc: + diagnostics.record_error("injector.load_anchor", exc) + + def scroll_into_view(self) -> bool: + """Jumps the chat so the card sits a third of the way down the screen, + with its anchor directly above it. + + A hard jump, not scrollToMessageId: for a message already in the list + that only does smoothScrollBy() from the current cell positions, which + are stale right after an insert (the layout pass has not run yet), so + it undershoots or does nothing. Setting the layout manager's pending + position is applied on the next layout, after the insert, and is what + ChatActivity itself does when it must land on a specific row. + """ + if self._jump_to_card(): + return True + self._load_anchor() + return False + + def _jump_to_card(self) -> bool: + try: + from hook_utils import get_private_field + + messages_list = get_messages_list(self.chat_activity) + adapter = get_chat_adapter(self.chat_activity) + layout_manager = getattr(self.chat_activity, "chatLayoutManager", None) or get_private_field( + self.chat_activity, "chatLayoutManager" ) + list_view = _chat_list_view(self.chat_activity) + if messages_list is None or adapter is None or layout_manager is None: + return False + index = find_existing(messages_list, self.virtual_id) + if index < 0: + return False + # Public on ChatActivityAdapter; already current, because + # notifyDataSetChanged() runs updateRowsInternal() before returning. + position = int(adapter.messagesStartRow) + index + offset = 0 + if list_view is not None: + try: + list_view.stopScroll() + except Exception: + pass + offset = int(list_view.getMeasuredHeight()) // 3 + # bottom=False: offset counts from the top edge, as in + # ChatActivity.scrollToMessageId's own scrollToPosition call. + layout_manager.scrollToPositionWithOffset(position, offset, False) return True except Exception as exc: - diagnostics.record_error("injector.scroll_into_view", exc) + diagnostics.record_error("injector.jump_to_card", exc) return False def update( @@ -381,6 +475,7 @@ class SummaryInjection: thinking: str = "", streaming: bool = True, status: str = "", + tagged: bool = False, ) -> None: """Re-renders the card in place, without rebuilding the whole list. @@ -397,32 +492,28 @@ class SummaryInjection: a fallback when the live cell cannot be found. """ if self.message_object is None: - self.insert(answer_markdown, thinking, streaming, status) + self.insert(answer_markdown, thinking, streaming, status, tagged) return - replacement = build_summary_message( - self.account, - self.dialog_id, - self.topic_id, - self.anchor_msg_id, - self.virtual_id, - answer_markdown, - thinking, - streaming, - status, - ) + replacement = self._build(answer_markdown, thinking, streaming, status, tagged) if replacement is None: return self.message_object = replacement + if not self.placed: + # Placement is still waiting for the anchor and will show this + # render. If it gave up, give it one more look now: the user may + # have scrolled the anchor in since. + if not self._placing: + self._placing = True + _on_ui(lambda: self._place(0)) + return + def run(): try: messages_list = get_messages_list(self.chat_activity) inserted = False if messages_list is not None: - # Replace at the current id; drop any card left behind - # under an id this injection used earlier, so a reanchor - # mid-stream cannot leave two cards on screen. index = find_existing(messages_list, self.virtual_id) if index >= 0: messages_list.set(index, replacement) @@ -434,16 +525,9 @@ class SummaryInjection: return messages_list.add(index, replacement) inserted = True - for stale_id in self._used_ids: - if stale_id == self.virtual_id: - continue - stale = find_existing(messages_list, stale_id) - if stale >= 0: - messages_list.remove(stale) - inserted = True - # A row was added or removed, so the adapter's item count - # changed and only a full rebind is correct. + # A row was added, so the adapter's item count changed and only + # a full rebind is correct. if inserted or not self._force_update_cell(replacement): adapter = get_chat_adapter(self.chat_activity) if adapter is not None: @@ -467,9 +551,7 @@ class SummaryInjection: delegate = getattr(self.chat_activity, "chatMessageCellDelegate", None) or get_private_field( self.chat_activity, "chatMessageCellDelegate" ) - list_view = getattr(self.chat_activity, "chatListView", None) or get_private_field( - self.chat_activity, "chatListView" - ) + list_view = _chat_list_view(self.chat_activity) if delegate is None or list_view is None: return False @@ -491,80 +573,56 @@ class SummaryInjection: diagnostics.record_error("injector._force_update_cell", exc) return False - def reanchor(self, anchor_msg_id: int, virtual_id: int) -> None: - """Re-targets an already-visible card once the analyzed range is known. - The card is injected on click, before any messages are fetched, so both - the anchor and the DB row id only become known later. Moving it keeps - the promise that a summary sits directly after its newest analyzed - message. - """ - anchor_msg_id = int(anchor_msg_id) - virtual_id = int(virtual_id) - if anchor_msg_id == self.anchor_msg_id and virtual_id == self.virtual_id: - return +def _chat_list_view(chat_activity: Any) -> Optional[Any]: + try: + from hook_utils import get_private_field - message_object = self.message_object - old_virtual_id = self.virtual_id - self.anchor_msg_id = anchor_msg_id - self.virtual_id = virtual_id - self._used_ids.add(virtual_id) - if message_object is None: - return + return getattr(chat_activity, "chatListView", None) or get_private_field(chat_activity, "chatListView") + except Exception: + return None - def run(): - try: - owner = message_object.messageOwner - messages_list = get_messages_list(self.chat_activity) - # Locate the card by its OLD id before rewriting it, otherwise - # the lookup misses and the card is never moved. - current = find_existing(messages_list, old_virtual_id) if messages_list is not None else -1 - owner.id = virtual_id - try: - header = _tlrpc().TL_messageReplyHeader() - header.reply_to_msg_id = anchor_msg_id - if self.topic_id: - header.reply_to_top_id = int(self.topic_id) - owner.reply_to = header - except Exception as exc: - diagnostics.record_error("injector.reanchor.reply_header", exc) +def remove_cards(chat_activity: Any, virtual_ids) -> None: + """Drops injected cards from an open chat's list. - if messages_list is not None and current >= 0: - messages_list.remove(current) - # A card may already sit under the new id -- e.g. the - # restore hook re-injected this summary's saved row. Claim - # that id rather than ending up with two cards. - duplicate = find_existing(messages_list, virtual_id) - if duplicate >= 0: - messages_list.remove(duplicate) - # The card was just removed, so it must go back somewhere: - # the new anchor came from the loaded list and is normally - # present, but if it paged out in between, fall back to id - # ordering rather than dropping the card entirely. - target = find_index_for_anchor(messages_list, anchor_msg_id) - if target < 0: - target = find_index_for_anchor( - messages_list, anchor_msg_id, require_anchor=False - ) - messages_list.add(max(target, 0), message_object) - adapter = get_chat_adapter(self.chat_activity) + ChatActivity removes deleted messages by looking them up in messagesDict, + which injected cards are never added to, so a deleted card would otherwise + stay on screen. + """ + wanted = {int(v) for v in virtual_ids} + if chat_activity is None or not wanted: + return + + def run(): + try: + messages_list = get_messages_list(chat_activity) + if messages_list is None: + return + removed = 0 + for i in range(messages_list.size() - 1, -1, -1): + owner = getattr(messages_list.get(i), "messageOwner", None) + if owner is not None and int(getattr(owner, "id", 0) or 0) in wanted: + messages_list.remove(i) + removed += 1 + if removed: + adapter = get_chat_adapter(chat_activity) if adapter is not None: adapter.notifyDataSetChanged() - # The card just moved to the real end of the analyzed range, - # which may be somewhere else entirely, so follow it there. - self.scroll_into_view() - except Exception as exc: - diagnostics.record_error("injector.reanchor", exc) + except Exception as exc: + diagnostics.record_error("injector.remove_cards", exc) - _on_ui(run) + _on_ui(run) -def _on_ui(fn) -> None: +def _on_ui(fn, delay_ms: int = 0) -> None: try: from android_utils import run_on_ui_thread - run_on_ui_thread(fn) + if delay_ms: + run_on_ui_thread(fn, delay_ms) + else: + run_on_ui_thread(fn) except Exception: fn() @@ -621,6 +679,7 @@ def restore_saved_summaries(chat_activity: Any, account: int, dialog_id: int, to summary.text, summary.thinking, False, + tagged=True, ) if message_object is not None: pending.append((summary.anchor_msg_id, virtual_id, message_object)) diff --git a/src/services/rich_markdown.py b/src/services/rich_markdown.py index 46e2d00..cef361b 100644 --- a/src/services/rich_markdown.py +++ b/src/services/rich_markdown.py @@ -296,3 +296,38 @@ def _thinking_details(TL_iv: Any, thinking: str) -> Any: if not details.blocks: details.blocks.add(_block(TL_iv, "pageBlockParagraph", build_rich_text(thinking, TL_iv))) return details + + +_PLAIN_BLOCK_RE = re.compile(r"^\s{0,3}(?:#{1,6}\s+|>\s?)", re.MULTILINE) +_FENCE_LINE_RE = re.compile(r"^\s*```+.*$\n?", re.MULTILINE) + + +def markdown_to_plain(markdown: str) -> str: + """Markdown reduced to the text a reader sees, for the message's plain body. + + Links keep their label, emphasis markers and heading/quote prefixes are + dropped. List bullets stay: they still read naturally as plain text. + """ + text = _FENCE_LINE_RE.sub("", markdown or "") + text = _PLAIN_BLOCK_RE.sub("", text) + + def inline(m: "re.Match") -> str: + for body in ("code_body", "link_text", "bold_body", "strike_body", "em_body"): + if m.group(body) is not None: + return markdown_to_plain(m.group(body)) if body != "code_body" else m.group(body) + return m.group(0) + + return _INLINE_RE.sub(inline, text).strip() + + +def hashtag_block(tag: str, TL_iv: Optional[Any] = None) -> Any: + """A paragraph holding one native hashtag node. + + TL_iv.textHashtag is the rich-text counterpart of a hashtag entity + (verified in the APK: it carries only `text`), so the layout styles it + and handles taps on it the way it does any other hashtag. + """ + TL_iv = TL_iv or _tl_iv() + node = TL_iv.textHashtag() + node.text = _plain(TL_iv, tag) + return _block(TL_iv, "pageBlockParagraph", node) diff --git a/src/services/summary_db.py b/src/services/summary_db.py index 3e2ce59..8399be1 100644 --- a/src/services/summary_db.py +++ b/src/services/summary_db.py @@ -8,7 +8,7 @@ message when the chat is reopened. import os import sqlite3 import time -from typing import List, Optional +from typing import List, Optional, Tuple DB_DIRNAME = "chat_summaries" DB_FILENAME = "summaries.db" @@ -107,6 +107,32 @@ class SummaryDb: ).fetchone() return int(row["id"]) + def reserve(self, dialog_id: int, topic_id: int, anchor_msg_id: int, msg_count: int = 0) -> Tuple[int, bool]: + """Claims a row id for a summary that is about to stream. + + Returns (row id, created). An existing summary for the same anchor is + left untouched, so re-summarizing a range does not wipe the saved text + before the new one has actually arrived; a stream that fails then keeps + the previous summary instead of leaving an empty row behind. + """ + now = int(time.time()) + with self._connect() as conn: + cur = conn.execute( + """ + INSERT INTO summaries + (dialog_id, topic_id, anchor_msg_id, text, thinking, msg_count, created_at) + VALUES (?, ?, ?, '', '', ?, ?) + ON CONFLICT (dialog_id, topic_id, anchor_msg_id) DO NOTHING + """, + (int(dialog_id), int(topic_id), int(anchor_msg_id), int(msg_count), now), + ) + created = cur.rowcount > 0 + row = conn.execute( + "SELECT id FROM summaries WHERE dialog_id = ? AND topic_id = ? AND anchor_msg_id = ?", + (int(dialog_id), int(topic_id), int(anchor_msg_id)), + ).fetchone() + return int(row["id"]), created + def get_for_scope(self, dialog_id: int, topic_id: int = 0) -> List[StoredSummary]: """All summaries for a chat/topic, oldest anchor first.""" with self._connect() as conn: @@ -120,6 +146,15 @@ class SummaryDb: ).fetchall() return [_row_to_summary(r) for r in rows] + def get_for_dialog(self, dialog_id: int) -> List[StoredSummary]: + """All summaries for a chat across every topic.""" + with self._connect() as conn: + rows = conn.execute( + "SELECT * FROM summaries WHERE dialog_id = ? ORDER BY anchor_msg_id ASC", + (int(dialog_id),), + ).fetchall() + return [_row_to_summary(r) for r in rows] + def get(self, summary_id: int) -> Optional[StoredSummary]: with self._connect() as conn: row = conn.execute("SELECT * FROM summaries WHERE id = ?", (int(summary_id),)).fetchone() @@ -138,6 +173,33 @@ class SummaryDb: ) return cur.rowcount + def clear_all(self) -> int: + """Deletes every stored summary and compacts the file. Returns rows removed.""" + with self._connect() as conn: + cur = conn.execute("DELETE FROM summaries") + removed = cur.rowcount + # VACUUM cannot run inside a transaction, and the `with` block above + # commits on exit, so it gets a connection of its own. + conn = self._connect() + try: + conn.isolation_level = None + conn.execute("VACUUM") + finally: + conn.close() + return removed + + def stats(self) -> Tuple[int, int]: + """(row count, approximate on-disk size in bytes).""" + with self._connect() as conn: + count = int(conn.execute("SELECT COUNT(*) FROM summaries").fetchone()[0]) + size = 0 + for suffix in ("", "-wal", "-journal"): + try: + size += os.path.getsize(self.db_path + suffix) + except OSError: + pass + return count, size + def _row_to_summary(row: sqlite3.Row) -> StoredSummary: return StoredSummary( diff --git a/tools/verify_bundle.py b/tools/verify_bundle.py index 7baee75..bdea76f 100755 --- a/tools/verify_bundle.py +++ b/tools/verify_bundle.py @@ -162,6 +162,7 @@ class _TLObj: self.blocks = _JavaList() self.items = _JavaList() self.texts = _JavaList() + self.entities = _JavaList() self.flags = 0 self.flags2 = 0 @@ -254,6 +255,7 @@ def _fake_send_request(request, callback, *, account=None): _KNOWN_METHODS = { "org.telegram.ui.LaunchActivity": ["dispatchTouchEvent"], + "org.telegram.messenger.MessagesController": ["deleteMessages", "deleteMessages"], "org.telegram.ui.ChatActivity": [ "onPageDownClicked", "updateMessagesVisiblePart", @@ -343,9 +345,16 @@ class FakeAlertDialogBuilder: BUTTON_POSITIVE = 1 BUTTON_NEGATIVE = 2 + built: List[Any] = [] + def __init__(self, activity, progress_style=0, resources_provider=None): self.activity = activity self._view = None + self._positive = None + self._negative = None + self.red_buttons = [] + self.shown = False + FakeAlertDialogBuilder.built.append(self) def set_title(self, *a, **k): return self @@ -372,8 +381,12 @@ class FakeAlertDialogBuilder: return self def show(self): + self.shown = True return self + def make_button_red(self, button_type): + self.red_buttons.append(button_type) + def dismiss(self): pass @@ -572,6 +585,7 @@ def install_fakes() -> None: "TL_peerChannel": staticmethod(lambda: _TLObj("TL_peerChannel")), "TL_messageReplyHeader": staticmethod(lambda: _TLObj("TL_messageReplyHeader")), "TL_emojiStatus": staticmethod(lambda: _TLObj("TL_emojiStatus")), + "TL_messageEntityHashtag": staticmethod(lambda: _TLObj("TL_messageEntityHashtag")), }, ) sys.modules["org.telegram.tgnet"] = tgnet_mod @@ -588,7 +602,7 @@ def install_fakes() -> None: "textFixed textUnderline textUrl pageBlockParagraph pageBlockHeading1 " "pageBlockHeading2 pageBlockHeading3 pageBlockPreformatted pageBlockBlockquote " "pageBlockDivider pageBlockThinking pageBlockList pageBlockOrderedList " - "pageBlockDetails TL_pageListItemText TL_pageListOrderedItemText" + "pageBlockDetails TL_pageListItemText TL_pageListOrderedItemText textHashtag" ).split() tgnet_tl_mod.TL_iv = type( "TL_iv", @@ -711,6 +725,242 @@ def expected_dunders(meta: dict) -> Dict[str, Any]: return result +def run_pipeline_checks(plugin, injector, summary_db_mod, chat_activity_cls, message_cls) -> None: + """Drives the whole Summarize pipeline: fetch -> stream -> card. + + The rule under test: the chat is not touched until the endpoint has + actually started answering. Every earlier failure is a bulletin only. + """ + if plugin is None or message_cls is None: + check("pipeline checks need the plugin class", False, "plugin not loaded") + return + + dialog_id = -4242 + requests_mod = sys.modules["requests"] + real_post = requests_mod.post + cu_mod = sys.modules["client_utils"] + real_last_fragment = cu_mod.get_last_fragment + plugin.set_setting("custom_base_url", "https://example.test/v1") + + def fresh_db(): + plugin.summary_db = summary_db_mod.SummaryDb(str(Path(tempfile.mkdtemp()) / "summaries.db")) + return plugin.summary_db + + def feed(messages=None, error=None): + plugin.fetcher.fetch_history = lambda d, c, o, t, cb: cb(messages, error) + + def run(activity): + BULLETINS.clear() + plugin._run_summary(None, activity, dialog_id, 0, 3, 0, "Summarize") + + analyzed = [ + message_cls("Alice", "a", id=500, sender_id=1), + message_cls("Bob", "b", id=555, sender_id=2), + message_cls("Alice", "c", id=520, sender_id=1), + ] + + class _ErrorResponse: + status_code = 500 + text = "upstream exploded" + + def close(self): + pass + + class _BreakingResponse: + status_code = 200 + text = "" + + def iter_lines(self): + yield b'data: {"choices":[{"delta":{"content":"Partial"}}]}' + raise IOError("connection reset") + + def close(self): + pass + + try: + # 1. History fetch fails: bulletin only, chat untouched. + db = fresh_db() + activity = chat_activity_cls([600, 555, 500]) + feed(error="flood wait") + run(activity) + check( + "a failed history fetch shows a bulletin and leaves the chat untouched", + activity.ids() == [600, 555, 500] + and activity.chatLayoutManager.jumps == [] + and activity.scrolled_to == [] + and ("error", "Chat Summaries: flood wait") in BULLETINS, + f"ids={activity.ids()} bulletins={BULLETINS}", + ) + + feed(messages=[]) + run(activity) + check( + "an empty history shows a bulletin and leaves the chat untouched", + activity.ids() == [600, 555, 500] and any("no messages" in b[1] for b in BULLETINS), + f"ids={activity.ids()} bulletins={BULLETINS}", + ) + + # 2. Endpoint fails before the first chunk (HTTP error, unreachable + # host): no card, no scroll, no DB row. + feed(messages=analyzed) + requests_mod.post = lambda *a, **k: _ErrorResponse() + run(activity) + check( + "an HTTP error before the first chunk injects nothing", + activity.ids() == [600, 555, 500] + and activity.chatLayoutManager.jumps == [] + and any("HTTP 500" in b[1] for b in BULLETINS if b[0] == "error"), + f"ids={activity.ids()} bulletins={BULLETINS}", + ) + + def unreachable(*a, **k): + raise ConnectionError("no route to host") + + requests_mod.post = unreachable + run(activity) + check( + "an unreachable endpoint injects nothing and reports the failure", + activity.ids() == [600, 555, 500] + and any("request failed" in b[1] for b in BULLETINS if b[0] == "error"), + f"ids={activity.ids()} bulletins={BULLETINS}", + ) + check( + "a failure before the first chunk leaves no row in the store", + db.stats()[0] == 0, + f"rows={db.stats()[0]}", + ) + + # 3. Success: the card appears right after the newest analyzed message + # (555, not the chat's newest 600), the chat jumps to it once, and the + # finished summary is tagged and persisted. + requests_mod.post = real_post + plugin.restore_hook._scope_state["done"].add(("stale",)) + run(activity) + ids = activity.ids() + cards = [i for i in ids if i < 0] + card_index = ids.index(cards[0]) if cards else -1 + check( + "a successful stream places exactly one card directly after the newest analyzed message", + len(cards) == 1 and card_index == ids.index(555) - 1, + f"ids={ids}", + ) + check( + "the chat hard-jumps to the card exactly once", + activity.chatLayoutManager.jumps == [(2 + card_index, 300, False)], + f"jumps={activity.chatLayoutManager.jumps}", + ) + final_owner = activity.messages.get(card_index).messageOwner if cards else None + check( + "the finished card is tagged #aisummaries and carries the answer", + final_owner is not None + and final_owner.message == "Fake summary.\n\n#aisummaries", + f"message={getattr(final_owner, 'message', None)!r}", + ) + rows = db.get_for_scope(dialog_id, 0) + check( + "the finished summary is saved under the analyzed anchor without the tag", + len(rows) == 1 and rows[0].anchor_msg_id == 555 and rows[0].text == "Fake summary." + and cards and cards[0] == injector.virtual_id_for(rows[0].id, 555), + f"rows={rows}", + ) + check( + "saving a summary invalidates the restore cache", + ("stale",) not in plugin.restore_hook._scope_state["done"], + ) + + # 4. Re-running a failing stream over a saved range keeps the saved text. + requests_mod.post = lambda *a, **k: _BreakingResponse() + run(activity) + rows = db.get_for_scope(dialog_id, 0) + check( + "a stream that breaks mid-way does not wipe the previously saved summary", + len(rows) == 1 and rows[0].text == "Fake summary.", + f"rows={rows}", + ) + check( + "a mid-stream failure reuses the existing card and reports the error on it", + len([i for i in activity.ids() if i < 0]) == 1 + and "Error:" in (activity.messages.get(card_index).messageOwner.message or "") + and any("stream failed" in b[1] for b in BULLETINS if b[0] == "error"), + f"ids={activity.ids()} bulletins={BULLETINS}", + ) + + # A fresh range that breaks mid-way leaves no empty row behind. + broken_activity = chat_activity_cls([700, 690]) + feed(messages=[message_cls("Alice", "x", id=690, sender_id=1)]) + run(broken_activity) + check( + "a fresh stream that breaks mid-way removes its reserved row", + [r.anchor_msg_id for r in db.get_for_dialog(dialog_id)] == [555], + f"rows={db.get_for_dialog(dialog_id)}", + ) + requests_mod.post = real_post + + # 5. Deleting the card through the app's own delete path forgets it. + card_id = cards[0] + cu_mod.get_last_fragment = lambda: activity + delete_hooks = [ + hook for method, hook in HOOKS_INSTALLED + if getattr(method, "class_name", "") == "org.telegram.messenger.MessagesController" + ] + BULLETINS.clear() + plugin.restore_hook._scope_state["done"].add(("stale",)) + param = types.SimpleNamespace( + args=[_JavaArrayList([card_id, 600]), None, None, dialog_id, False, 0] + ) + for hook in delete_hooks: + hook.before_hooked_method(param) + check( + "deleting a summary card removes its row, its card, and says so once", + db.get_for_dialog(dialog_id) == [] + and card_id not in activity.ids() + and BULLETINS == [("info", "Summary deleted")] + and ("stale",) not in plugin.restore_hook._scope_state["done"], + f"rows={db.get_for_dialog(dialog_id)} ids={activity.ids()} bulletins={BULLETINS}", + ) + BULLETINS.clear() + for hook in delete_hooks: + hook.before_hooked_method(types.SimpleNamespace(args=[_JavaArrayList([600]), None, None, dialog_id])) + check( + "deleting ordinary messages never touches the summary store", + BULLETINS == [], + f"bulletins={BULLETINS}", + ) + + # 6. Clear all: confirmed through a red-buttoned alert. + db.save(dialog_id, 0, 555, "one", "", 1) + db.save(-1, 0, 7, "two", "", 1) + count, size = db.stats() + check("stats report row count and a non-zero file size", count == 2 and size > 0, f"{count} {size}") + + class _Fragment: + def getParentActivity(self): + return object() + + cu_mod.get_last_fragment = lambda: _Fragment() + FakeAlertDialogBuilder.built.clear() + BULLETINS.clear() + plugin._on_clear_all_click(None) + dialog = FakeAlertDialogBuilder.built[-1] if FakeAlertDialogBuilder.built else None + check( + "Clear all asks for confirmation with a red button before deleting", + dialog is not None and dialog.shown + and dialog.red_buttons == [FakeAlertDialogBuilder.BUTTON_POSITIVE] + and db.stats()[0] == 2, + f"dialog={getattr(dialog, '__dict__', None)}", + ) + if dialog is not None and dialog._positive is not None: + dialog._positive(dialog, FakeAlertDialogBuilder.BUTTON_POSITIVE) + check( + "confirming Clear all wipes every summary and reports it", + db.stats()[0] == 0 and ("success", "Deleted 2 summaries") in BULLETINS, + f"rows={db.stats()[0]} bulletins={BULLETINS}", + ) + finally: + requests_mod.post = real_post + cu_mod.get_last_fragment = real_last_fragment + + def main(argv=None) -> int: argv = sys.argv[1:] if argv is None else argv if len(argv) != 1: @@ -1007,44 +1257,26 @@ def main(argv=None) -> int: f"page_results={page_results}", ) - # Provisional anchor, chosen before the fetch returns: with an offset - # the analyzed range ends partway up the history, so the guess has to - # skip that many messages instead of grabbing the newest. - class _AnchorList(list): - def size(self): - return len(self) - - def get(self, index): - return self[index] - - class _AnchorActivity: - def __init__(self, ids): - self.messages = _AnchorList() - for mid in ids: - owner_obj = _TLObj("TL_message") - owner_obj.id = mid - holder = types.SimpleNamespace(messageOwner=owner_obj) - self.messages.append(holder) - - anchor_activity = _AnchorActivity([900, 800, 700, 600]) check( - "with no offset the provisional anchor is the newest message", - plugin._newest_visible_message_id(anchor_activity) == 900, - f"got {plugin._newest_visible_message_id(anchor_activity)}", - ) - check( - "an offset moves the provisional anchor up the history", - plugin._newest_visible_message_id(anchor_activity, skip=2) == 700, - f"got {plugin._newest_visible_message_id(anchor_activity, skip=2)}", + "delete_hook hooks every MessagesController.deleteMessages overload", + len(plugin.delete_hook._unhooks) == 2, + f"installed hooks: {len(plugin.delete_hook._unhooks)} (expected 2)", ) - # Injected cards carry negative ids and must never be chosen as the - # anchor, or a second summary would anchor to the first one's card. - mixed_anchor = _AnchorActivity([-700007, 900, 800]) + # Storage settings: an approximate size readout and a destructive, + # confirmed "clear all" action. + storage_rows = {getattr(r, "text", None): r for r in settings_rows} + clear_row = storage_rows.get("Clear all summaries") + size_row = storage_rows.get("Saved summaries") check( - "an existing injected card is never picked as the anchor", - plugin._newest_visible_message_id(mixed_anchor) == 900, - f"got {plugin._newest_visible_message_id(mixed_anchor)}", + "settings show the saved summary count and approximate size", + size_row is not None and "saved" in (size_row.subtext or "") and "B" in size_row.subtext, + f"subtext={getattr(size_row, 'subtext', None)!r}", + ) + check( + "settings offer a red Clear all summaries action", + clear_row is not None and clear_row.red is True and callable(clear_row.on_click), + f"row={getattr(clear_row, '__dict__', None)}", ) # ------------------------------------------------------------------ @@ -1176,6 +1408,14 @@ def main(argv=None) -> int: [b.tl_name for b in answered.blocks] == ["pageBlockHeading1"], f"blocks={[b.tl_name for b in answered.blocks]}", ) + plain = rich_markdown.markdown_to_plain( + "# Title\n\n**Bold** and *it* with `code` by [Alice](tg://user?id=1)\n> quote\n```\nx = 1\n```" + ) + check( + "markdown_to_plain keeps the readable text and drops the markup", + plain == "Title\n\nBold and it with code by Alice\nquote\nx = 1", + f"plain={plain!r}", + ) empty_rm = rich_markdown.build_rich_message("", streaming=True) check( "an empty RichMessage still carries one block so the cell renders", @@ -1481,6 +1721,25 @@ def main(argv=None) -> int: f"topic0={len(db.get_for_scope(-100, 0))} topic7={len(db.get_for_scope(-100, 7))}", ) + reserved, created = db.reserve(-100, 0, 555, 9) + check( + "reserving an anchor that already has a summary keeps its text", + reserved == first and created is False and db.get(first).text == "text two", + f"reserved={reserved} created={created} text={db.get(first).text!r}", + ) + fresh, fresh_created = db.reserve(-100, 0, 777, 9) + check( + "reserving a new anchor creates an empty row", + fresh_created is True and db.get(fresh).text == "", + f"created={fresh_created}", + ) + check( + "get_for_dialog spans every topic of the chat", + sorted((r.topic_id, r.anchor_msg_id) for r in db.get_for_dialog(-100)) + == [(0, 555), (0, 777), (7, 555)], + f"rows={db.get_for_dialog(-100)}", + ) + check("summary_db.delete removes a row", db.delete(first) and db.get(first) is None) injector = bundled("services.rich_injector") @@ -1547,6 +1806,10 @@ def main(argv=None) -> int: # Insertion order: ChatActivity.messages is newest-first, so the # summary must land at the anchor's index to sit after it on screen. class _FakeAdapter: + # ChatActivityAdapter rows before the first message (header, + # loading row): the jump must offset the list index by this. + messagesStartRow = 2 + def __init__(self): self.notified = 0 @@ -1581,6 +1844,13 @@ def main(argv=None) -> int: class _FakeListView: def __init__(self): self.cells = [] + self.stopped = 0 + + def getMeasuredHeight(self): + return 900 + + def stopScroll(self): + self.stopped += 1 def getChildCount(self): return len(self.cells) @@ -1588,6 +1858,13 @@ def main(argv=None) -> int: def getChildAt(self, index): return self.cells[index] + class _FakeLayoutManager: + def __init__(self): + self.jumps = [] + + def scrollToPositionWithOffset(self, position, offset, bottom): + self.jumps.append((position, offset, bottom)) + class _FakeChatActivity: def __init__(self, ids): self.messages = _JavaList() @@ -1598,13 +1875,24 @@ def main(argv=None) -> int: self.chatAdapter = _FakeAdapter() self.chatMessageCellDelegate = _FakeDelegate() self.chatListView = _FakeListView() + self.chatLayoutManager = _FakeLayoutManager() self.scrolled_to = [] + # Ids scrollToMessageId will "load" into the list, mimicking + # the history request it issues for an unloaded id. + self.loadable = {} def getDialogId(self): return -4242 def scrollToMessageId(self, msg_id, from_id, select, load_index, force, unread): self.scrolled_to.append(msg_id) + for mid in self.loadable.pop(msg_id, ()): + owner_obj = _TLObj("TL_message") + owner_obj.id = mid + self.messages.add(_FakeMessageObject(0, owner_obj)) + + def ids(self): + return [m.messageOwner.id for m in self.messages] activity = _FakeChatActivity([600, 555, 500]) check( @@ -1684,7 +1972,7 @@ def main(argv=None) -> int: virtual_id = injector.virtual_id_for(9, 555) inj = injector.SummaryInjection(activity, 0, -4242, 0, 555, virtual_id) - inserted = inj.insert(status="Fetching messages…", streaming=True) + inserted = inj.insert(thinking="first chunk", streaming=True) check( "injection inserts one message at the anchor index and refreshes the adapter", inserted @@ -1695,12 +1983,18 @@ def main(argv=None) -> int: ) # The analyzed range can end far up the history, so the card may be - # injected off-screen; the chat has to move to it. The anchor is the - # scroll target because it is a real message the app can resolve, and - # the card sits directly above it. + # injected off-screen; the chat has to move to it. scrollToMessageId + # only smooth-scrolls from stale cell positions right after an insert, + # so placement hard-jumps the layout manager to the card's own row. check( - "inserting the card scrolls the chat to the analyzed range", - activity.scrolled_to == [555], + "placing the card hard-jumps the layout manager to the card's row", + activity.chatLayoutManager.jumps == [(2 + 1, 300, False)] + and activity.chatListView.stopped == 1, + f"jumps={activity.chatLayoutManager.jumps} stopped={activity.chatListView.stopped}", + ) + check( + "a loaded anchor needs no history load before placement", + activity.scrolled_to == [], f"scrolled_to={activity.scrolled_to}", ) @@ -1720,8 +2014,8 @@ def main(argv=None) -> int: ) check( "a streaming update does not scroll again and fight the user", - activity.scrolled_to == [555], - f"scrolled_to={activity.scrolled_to}", + activity.scrolled_to == [] and len(activity.chatLayoutManager.jumps) == 1, + f"scrolled_to={activity.scrolled_to} jumps={activity.chatLayoutManager.jumps}", ) # Streaming repaints should hit the one visible cell, not rebind every @@ -1750,46 +2044,6 @@ def main(argv=None) -> int: f"notified={activity.chatAdapter.notified} (was {notified_before})", ) - # The card is injected before the fetch, so its anchor starts - # provisional and must be corrected once the real range is known. - reanchor_activity = _FakeChatActivity([600, 555, 500]) - provisional_id = injector.virtual_id_for(0, 600) - reinj = injector.SummaryInjection(reanchor_activity, 0, -4242, 0, 600, provisional_id) - reinj.insert(status="Fetching messages…", streaming=True) - check( - "provisional card lands at the newest message before the fetch", - reanchor_activity.messages.get(0).messageOwner.id == provisional_id, - f"ids={[m.messageOwner.id for m in reanchor_activity.messages]}", - ) - - final_id = injector.virtual_id_for(11, 555) - reinj.reanchor(555, final_id) - ids = [m.messageOwner.id for m in reanchor_activity.messages] - check( - "reanchor moves the card to the real anchor and adopts its virtual id", - reanchor_activity.messages.size() == 4 - and ids.index(final_id) == 1 - and reanchor_activity.messages.get(1).messageOwner.reply_to.reply_to_msg_id == 555, - f"ids={ids}", - ) - check( - "reanchor follows the card to the real end of the analyzed range", - reanchor_activity.scrolled_to == [600, 555], - f"scrolled_to={reanchor_activity.scrolled_to}", - ) - - # Regression: a streaming update after a reanchor used to add a second - # card, because the lookup missed the old id and fell through to add(). - reinj.update(status="Summarizing 98 messages…", streaming=True) - ids = [m.messageOwner.id for m in reanchor_activity.messages] - check( - "updating after a reanchor leaves exactly one summary card", - reanchor_activity.messages.size() == 4 - and ids.count(final_id) == 1 - and provisional_id not in ids, - f"ids={ids}", - ) - # Re-summarizing a range whose saved card the restore hook already # re-injected must claim that card, not sit beside it. collide_activity = _FakeChatActivity([600, 555, 500]) @@ -1801,18 +2055,101 @@ def main(argv=None) -> int: 0, [summary_db_mod.StoredSummary(12, -4242, 0, 555, "previous run", "", 5, 0)], ) - collide_provisional = injector.virtual_id_for(0, 600) - collide = injector.SummaryInjection(collide_activity, 0, -4242, 0, 600, collide_provisional) - collide.insert(status="Fetching messages…", streaming=True) - collide.reanchor(555, saved_id) - collide.update(status="Summarizing…", streaming=True) - ids = [m.messageOwner.id for m in collide_activity.messages] + collide = injector.SummaryInjection(collide_activity, 0, -4242, 0, 555, saved_id) + collide.insert(thinking="new run", streaming=True) + collide.update(answer_markdown="new answer", streaming=True) + ids = collide_activity.ids() check( "re-summarizing a restored range reuses its card instead of duplicating", - ids.count(saved_id) == 1 and collide_provisional not in ids, + ids.count(saved_id) == 1 and len(ids) == 4, f"ids={ids}", ) + # The anchor is the newest ANALYZED message; with an offset it can be + # outside the loaded window. The card must never fall back to the chat + # bottom: the chat is asked to load the anchor, and the card is placed + # directly after it once it arrives. + far = _FakeChatActivity([900, 800]) + far.loadable[300] = [310, 300, 290] + far_card = injector.virtual_id_for(5, 300) + far_inj = injector.SummaryInjection(far, 0, -4242, 0, 300, far_card) + far_inj.insert(thinking="chunk", streaming=True) + far_ids = far.ids() + check( + "an unloaded anchor is loaded first, then the card lands right after it", + far.scrolled_to == [300] + and far_inj.placed + and far_ids.index(far_card) == far_ids.index(300) - 1, + f"scrolled_to={far.scrolled_to} ids={far_ids}", + ) + check( + "the card is never parked at the bottom while its anchor loads", + far_ids[0] == 900, + f"ids={far_ids}", + ) + + # An anchor that never loads: no card anywhere, rather than a card + # attached to messages it did not analyze. + never = _FakeChatActivity([900, 800]) + never_card = injector.virtual_id_for(6, 300) + never_inj = injector.SummaryInjection(never, 0, -4242, 0, 300, never_card) + never_inj.insert(thinking="chunk", streaming=True) + never_inj.update(answer_markdown="more", streaming=True) + check( + "an anchor that never loads places no card at a wrong position", + never_card not in never.ids() and not never_inj.placed and never.scrolled_to == [300], + f"ids={never.ids()} scrolled_to={never.scrolled_to}", + ) + # ...and the card shows up once the user scrolls the anchor in. + for mid in (310, 300): + owner_obj = _TLObj("TL_message") + owner_obj.id = mid + never.messages.add(_FakeMessageObject(0, owner_obj)) + never_inj.update(answer_markdown="final", streaming=False, tagged=True) + never_ids = never.ids() + check( + "a card whose anchor loads late is placed on the next render", + never_inj.placed and never_ids.index(never_card) == never_ids.index(300) - 1, + f"ids={never_ids}", + ) + + # Finished summaries carry the #aisummaries tag, in the card and in + # the plain-text body (with a hashtag entity), so copied text and any + # text matching see it. Streaming renders stay untagged. + tagged_mo = injector.build_summary_message( + 0, -4242, 0, 555, injector.virtual_id_for(1, 555), + "**Done** see [Alice](tg://user?id=1)", "", False, tagged=True, + ) + tagged_owner = tagged_mo.messageOwner + last_block = tagged_owner.rich_message.blocks[-1] + entity = tagged_owner.entities[0] if len(tagged_owner.entities) else None + tag_node = getattr(last_block, "text", None) + check( + "a finished summary ends with a native #aisummaries hashtag node", + last_block.tl_name == "pageBlockParagraph" + and getattr(tag_node, "tl_name", None) == "textHashtag" + and getattr(tag_node.text, "text", None) == "#aisummaries", + f"last={last_block.tl_name} node={getattr(tag_node, '__dict__', None)}", + ) + check( + "the plain-text body is searchable text plus a hashtag entity", + tagged_owner.message == "Done see Alice\n\n#aisummaries" + and entity is not None + and entity.tl_name == "TL_messageEntityHashtag" + and entity.offset == len("Done see Alice\n\n") + and entity.length == len("#aisummaries"), + f"message={tagged_owner.message!r} entity={getattr(entity, '__dict__', None)}", + ) + streaming_mo = injector.build_summary_message( + 0, -4242, 0, 555, injector.virtual_id_for(1, 555), "partial", "", True + ) + check( + "a streaming render is not tagged", + "#aisummaries" not in streaming_mo.messageOwner.message + and len(streaming_mo.messageOwner.entities) == 0, + f"message={streaming_mo.messageOwner.message!r}", + ) + # Re-inserting the same virtual id must replace, never duplicate. inj2 = injector.SummaryInjection(activity, 0, -4242, 0, 555, virtual_id) inj2.insert(answer_markdown="again", thinking="", streaming=False) @@ -2015,23 +2352,7 @@ def main(argv=None) -> int: f"dispatched={sep_n} ids={sep_ids}", ) - # A live status card is the only feedback that pressing Summarize did - # anything, so it must appear even when the provisional anchor cannot be - # placed: a large offset reaches past the loaded window, and an - # unresolved anchor arrives as 0. Requiring a loaded anchor here left the - # user staring at an unchanged chat. - for label, anchor in (("unresolved anchor 0", 0), ("anchor past the window", 4242)): - live = _FakeChatActivity([700, 690, 680]) - live_injection = injector.SummaryInjection( - live, 0, -4242, 0, anchor, injector.virtual_id_for(0, anchor or 1) - ) - shown = live_injection.insert(status="Fetching messages…", streaming=True) - live_ids = [m.messageOwner.id for m in live.messages] - check( - f"the status card still appears with an {label}", - shown and len([i for i in live_ids if i < 0]) == 1, - f"inserted={shown} ids={live_ids}", - ) + run_pipeline_checks(plugin if plugin_cls is not None else None, injector, summary_db_mod, _FakeChatActivity, _FakeMessage if plugin_cls is not None else None) check( "sys.modules gained no entry for bundled plugin module names",