From 5141f47878648822e4e37824bad40270bce4bef4 Mon Sep 17 00:00:00 2001 From: lyuksovannyy Date: Wed, 14 Oct 2026 17:31:22 +0200 Subject: [PATCH] Message Logger: removed forums load, stay smooth and cached - Topics load fully again: the bottom-bar cleanup no longer hides the message list; each page fetches a full window from the server and answers only what the app asked for. - Reopened chats and topics come from memory (pages, logged copies, downloaded media); pages are also kept on the device for restarts and server timeouts, with one retry before falling back. - Media downloads run in one background queue per chat instead of blocking page answers or spawning a thread per page. - The deleted badge is drawn in DialogCell.drawAvatarOverlays, under the avatar's own transform, so it follows the avatar through transitions; its paints are built once and the hot hooks skip reflection. - Senders keep their last known profile (name, username, photo reference) when the app no longer has them (people.py). - Removed chats are searchable, show "deleted chat" only in place of the loading subtitle, hide the join bar, order topics by activity, and have no unread counters. - Uninstall takes removed chats out on the UI thread. --- plugin.json | 2 +- src/api.py | 8 +- src/hooks/chat.py | 16 + src/hooks/removed_chats.py | 658 ++++++++++++++++++++++++++++++++++--- src/jutil.py | 98 +++++- src/main.py | 2 +- src/people.py | 182 ++++++++++ src/views/comments.py | 130 ++++++-- src/views/removed_chat.py | 406 ++++++++++++++++++++--- 9 files changed, 1372 insertions(+), 130 deletions(-) create mode 100644 src/people.py diff --git a/plugin.json b/plugin.json index 970bf59..c0cd611 100644 --- a/plugin.json +++ b/plugin.json @@ -3,7 +3,7 @@ "name": "Message Logger", "description": "Shows deleted and edited messages and their edit history from your telegram-panel message logger.", "author": "@lyuksovannyy", - "version": "0.1.6", + "version": "0.1.7", "icon": "exteraGramPluginIcons/4", "app_version": ">=12.5.1", "sdk_version": ">=1.4.4.3", diff --git a/src/api.py b/src/api.py index b8f57ba..a1a589b 100644 --- a/src/api.py +++ b/src/api.py @@ -209,10 +209,12 @@ class Api: chats.setdefault(int(chat["chat_id"]), chat) return list(chats.values()) if any_result else None - def history(self, account_id, chat_id, before=0, limit=100): - """Logged messages of a chat older than `before`, newest first: the - first master that has any.""" + def history(self, account_id, chat_id, before=0, limit=100, topic_id=None): + """Logged messages older than `before`, newest first. For a forum + topic, pass its source topic id; 0 selects General.""" params = {"account_id": account_id, "before": int(before), "limit": int(limit)} + if topic_id is not None: + params["topic_id"] = int(topic_id) for _token, result in self._each( "GET", f"/chats/{chat_id}/history", params=params ): diff --git a/src/hooks/chat.py b/src/hooks/chat.py index 55d506b..7400a6c 100644 --- a/src/hooks/chat.py +++ b/src/hooks/chat.py @@ -218,9 +218,25 @@ def build_from_copy(account, dialog_id, message_id, state, copy, strip): own = peers.own_user_id(account) msg.out = sender is not None and int(sender) == own msg.unread = False + if sender: + _remember_sender(account, _android_peer_id(sender)) return MessageObject(account, msg, True, True) +def _remember_sender(account, peer_id): + """Snapshot the sender's profile while the app can still read it, so it + can be shown later when the app no longer has it.""" + try: + import people + + if peer_id > 0: + people.user(account, peer_id) + elif peer_id < 0: + people.chat(account, -peer_id) + except Exception as exc: + diag.error("chat.remember_sender", exc) + + class ChatSync: def __init__(self, plugin, api): self.plugin = plugin diff --git a/src/hooks/removed_chats.py b/src/hooks/removed_chats.py index ec31aa6..f97db5b 100644 --- a/src/hooks/removed_chats.py +++ b/src/hooks/removed_chats.py @@ -33,7 +33,7 @@ import store from api import ApiError from config import DEFAULTS, SHOW_REMOVED_CHATS from hooks import avatars -from jutil import field, hook_methods, in_background, java_list, on_ui +from jutil import cached_field, field, hook_methods, in_background, java_list, on_ui from views import removed_chat from views.comments import make_read_only from views.history import _epoch @@ -43,13 +43,19 @@ CONNECTIONS = "org.telegram.tgnet.ConnectionsManager" CHAT = "org.telegram.ui.ChatActivity" IMAGES = "org.telegram.messenger.ImageReceiver" STORAGE = "org.telegram.messenger.MessagesStorage" -_NOTICE = "Removed chat · kept by the logger" +TOPICS = "org.telegram.messenger.TopicsController" +FORUM = "org.telegram.ui.TopicsFragment" +SEARCH = "org.telegram.ui.Adapters.DialogsSearchAdapter" +_NOTICE = "Viewing deleted chat/topic" _REFRESH_SECONDS = 900 _REPORT_AGAIN_SECONDS = 30 _ANSWERED = ( - "TLRPC$TL_messages_getHistory", - "TLRPC$TL_channels_getMessages", - "TLRPC$TL_messages_search", + "TL_messages_getHistory", + "TL_channels_getMessages", + "TL_messages_search", + "TL_messages_getForumTopics", + "TL_messages_getForumTopicsByID", + "TL_messages_getReplies", ) @@ -69,8 +75,44 @@ def _dialog_id(chat_id): return _android_peer_id(int(chat_id)) +def _topic_start(controller, chat_id, topic): + """The topic's "topic created" message, as the app builds one when it + has none (TopicsController.processTopics).""" + from org.telegram.messenger import MessagesController + from org.telegram.tgnet import TLRPC + + start = TLRPC.TL_message() + start.message = "" + start.id = int(topic.id) + start.peer_id = MessagesController.getInstance( + int(field(controller, "currentAccount") or 0) + ).getPeer(-int(chat_id)) + start.action = TLRPC.TL_messageActionTopicCreate() + start.action.title = topic.title + return start + + +def _topics_loaded(account, chat_id): + from java import jarray, jlong + from java.lang import Boolean, Long, Object + from org.telegram.messenger import NotificationCenter + + NotificationCenter.getInstance(account).postNotificationName( + NotificationCenter.topicsDidLoaded, + jarray(Object)([Long.valueOf(jlong(int(chat_id))), Boolean.TRUE]), + ) + + +# Request classes with no peer field: cached so the hook on every outgoing +# request skips them without any reflection. +_NO_TARGET = set() + + def _target(request): """The dialog a request is about (its peer/channel/user), else None.""" + kind_name = type(request).__name__ + if kind_name in _NO_TARGET: + return None for name in ("peer", "channel", "user_id", "id"): value = getattr(request, name, None) if value is None or isinstance(value, (bool, int, float, str)): @@ -82,9 +124,114 @@ def _target(request): return -int(value.chat_id) if "nputPeerUser" in kind or kind.endswith("TL_inputUser"): return int(value.user_id) + if not any(hasattr(request, n) for n in ("peer", "channel", "user_id")): + _NO_TARGET.add(kind_name) return None +_SUBTITLE = "deleted chat" + + +def _clear_loading(chat): + """Replace the header's "connecting/loading" subtitle on a removed chat. + + The app keeps that subtitle until a channel-info reply arrives, and for + a removed chat no reply ever comes (its requests are answered here or + dropped), so it would say "loading" forever.""" + try: + container = field(chat, "avatarContainer") + if container is None: + return + view = field(container, "subtitleTextView") + current = "" + if view is not None: + try: + current = str(view.getText() or "") + except Exception: + current = "" + # Only replace the placeholder: a topic header legitimately shows + # "In ", and overwriting it made the two flicker. + if current and not _is_loading(current): + return + container.setSubtitle(_SUBTITLE) + except Exception as exc: + diag.error("removed_chats.subtitle", exc) + + +_LOADING_KEYS = ("Loading", "Connecting", "Updating", "WaitingForNetwork", "ConnectingToProxy") +_loading_texts = None + + +def _is_loading(text): + """Whether a header subtitle is one of the app's placeholder strings. + + Read from the app's own localisation, so this works in any language + (the device shows Dutch: "Bezig met laden").""" + global _loading_texts + if _loading_texts is None: + found = set() + try: + from org.telegram.messenger import LocaleController + + for key in _LOADING_KEYS: + try: + value = LocaleController.getString(key) + if value: + found.add(str(value).strip().lower()) + except Exception: + pass + except Exception as exc: + diag.error("removed_chats.loading_strings", exc) + _loading_texts = found + cleaned = str(text).strip().lower().rstrip(".…") + return any(cleaned == known.rstrip(".…") for known in _loading_texts) + + +def _hide_forum_join(fragment): + """Hide a removed forum's "join group" bar. + + TopicsFragment shows the bar for a chat the account is not in, but a + removed chat cannot be joined: the button would send a request that is + answered here (or dropped) and never completes.""" + try: + from android.view import View + + for name in ("bottomOverlayContainer", "bottomOverlayChatText", + "bottomOverlayProgress"): + view = cached_field(fragment, name) + if view is None or not hasattr(view, "setVisibility"): + continue + try: + # Only when visible: setVisibility on an already-gone view + # still requests a layout pass during the open animation. + if view.getVisibility() != View.GONE: + view.setVisibility(View.GONE) + except Exception: + pass + except Exception as exc: + diag.error("removed_chats.forum_join", exc) + + +def _badge_paints(): + """(icon, red paint, ring paint, dp table) for the deleted badge, built + once: the badge is drawn every frame for every visible removed chat.""" + from android.graphics import Paint, PorterDuff, PorterDuffColorFilter + from java import jint + from org.telegram.messenger import AndroidUtilities, R + from org.telegram.ui.ActionBar import Theme + + from hooks.render import _trash_drawable + + icon = _trash_drawable(R.drawable.msg_delete) + icon.setColorFilter(PorterDuffColorFilter(jint(-1), PorterDuff.Mode.SRC_IN)) + red = Paint(Paint.ANTI_ALIAS_FLAG) + red.setColor(jint(Theme.getColor(Theme.key_text_RedBold))) + ring = Paint(Paint.ANTI_ALIAS_FLAG) + ring.setColor(jint(Theme.getColor(Theme.key_windowBackgroundWhite))) + dp = {v: float(AndroidUtilities.dp(v)) for v in (1.5, 8, 11, 54)} + return icon, red, ring, dp + + class RemovedChats: def __init__(self, plugin, api): self.plugin = plugin @@ -95,7 +242,14 @@ class RemovedChats: # account -> {dialog_id: (Dialog, [MessageObject], its top message id)} self._injected = {} self._histories = {} # (account, dialog_id) -> views.removed_chat.History + # (account, dialog_id) -> java ArrayList of TL_forumTopic (removed forums) + self._topics = {} + self._topic_index = {} # (account, dialog_id) -> {topic id: TL_forumTopic} + self._topics_loading = set() + self._badge = None # (trash icon, red paint, ring paint) + self._search_logged = False self._avatar_paths = {} # (account, dialog_id) -> local photo path + self._positions = {} # (account, dialog_id) -> saved dialog placement self._reported = {} # (account, dialog_id) -> monotonic time self._lock = threading.Lock() self._timer = None @@ -131,6 +285,13 @@ class RemovedChats: hook(CHAT, "updateBottomOverlay", after=self.after_overlay) hook(STORAGE, "putMessages", before=self.before_store) hook(IMAGES, "setForUserOrChat", before=self.before_avatar, arity=6) + hook("org.telegram.ui.Cells.DialogCell", "drawAvatarOverlays", after=self.after_cell_draw, arity=1) + hook(TOPICS, "getTopics", before=self.before_topics) + hook(TOPICS, "findTopic", before=self.before_find_topic, arity=2) + hook(TOPICS, "loadTopics", before=self.before_load_topics, arity=3) + hook(FORUM, "updateSubtitle", after=self.after_forum_subtitle) + hook(FORUM, "onResume", after=self.after_forum_resume) + hook(SEARCH, "searchDialogs", after=self.after_search_dialogs, arity=3) in_background(self._start) def _start(self): @@ -139,6 +300,15 @@ class RemovedChats: cached = store.get_meta(f"removed_chats:{account}") if cached: self._set_list(account, json.loads(cached)) + saved = store.get_meta(f"removed_positions:{account}") + if saved: + try: + self._positions.update( + {(account, int(dialog_id)): value + for dialog_id, value in json.loads(saved).items()} + ) + except Exception as exc: + diag.error("removed_chats.positions", exc) on_ui(self.inject_all) self._schedule(5) @@ -151,12 +321,20 @@ class RemovedChats: except Exception: pass self.handles = [] - for account in list(self._injected): + # inject() re-sorts the app's chat list and redraws it, which must + # happen on the UI thread; uninstall runs on the plugins thread. + accounts_listed = list(self._injected) + for account in accounts_listed: self._lists[account] = {} - try: - self.inject(account) - except Exception as exc: - diag.error("removed_chats.uninstall", exc) + + def take_out(): + for account in accounts_listed: + try: + self.inject(account) + except Exception as exc: + diag.error("removed_chats.uninstall", exc) + + on_ui(take_out) # the server's list (background) @@ -186,11 +364,26 @@ class RemovedChats: return if chats is None: return + previous = self._lists.get(account) or {} store.put_meta(f"removed_chats:{account}", json.dumps(chats)) self._set_list(account, chats) + # Only the chats whose server snapshot really changed lose their + # cached history/topics. Wiping every chat each cycle made an open + # forum reload its 17 topics on every refresh (topicsDidLoaded -> + # DiffUtil -> full relayout), which is what made it crawl. with self._lock: - for key in [k for k in self._histories if k[0] == account]: - del self._histories[key] # read again on next open + for dialog_id, item in (self._lists.get(account) or {}).items(): + before = previous.get(dialog_id) + if before is not None and before == item: + continue + self._histories.pop((account, dialog_id), None) + self._topics.pop((account, dialog_id), None) + self._topic_index.pop((account, dialog_id), None) + for dialog_id in previous: + if dialog_id not in (self._lists.get(account) or {}): + self._histories.pop((account, dialog_id), None) + self._topics.pop((account, dialog_id), None) + self._topic_index.pop((account, dialog_id), None) on_ui(lambda: self.inject(account)) for dialog_id, item in self._lists.get(account, {}).items(): avatar = item.get("avatar") or {} @@ -207,24 +400,13 @@ class RemovedChats: self._lists[account] = lists def _peer(self, account, dialog_id, item): - """The app's user/chat (cache or storage), else a stand-in.""" - from org.telegram.messenger import MessagesController, MessagesStorage + """The removed chat's user/chat: the app's, else the last snapshot + saved while it was readable, else a stand-in named from the logger.""" + import people - import tg - - controller = MessagesController.getInstance(account) if dialog_id > 0: - user = controller.getUser(dialog_id) - if user is None: - try: - user = MessagesStorage.getInstance(account).getUserSync(dialog_id) - except Exception: - user = None - return user or removed_chat.stand_in_user(controller, dialog_id, item.get("title")) - chat = tg.channel_chat(account, -dialog_id) - return chat or removed_chat.stand_in_chat( - controller, -dialog_id, item.get("title"), item.get("kind") - ) + return people.user(account, dialog_id, item.get("title")) + return people.chat(account, -dialog_id, item.get("title"), item.get("kind")) def _load_avatar(self, account, dialog_id, avatar): found = avatars.logged_photo(account, avatar, self.api.masters()) @@ -244,16 +426,14 @@ class RemovedChats: self.inject(account) def inject(self, account): - """Puts the account's removed chats into the app's chat list (and - takes out those no longer removed).""" + """Puts the account's removed chats into the app's chat list.""" from org.telegram.messenger import MessagesController controller = MessagesController.getInstance(account) wanted = {} if self._enabled(): wanted = { - d: item - for d, item in self._lists.get(account, {}).items() + d: item for d, item in self._lists.get(account, {}).items() if not item.get("hidden_at") and item.get("top_message") } current = self._injected.setdefault(account, {}) @@ -262,8 +442,23 @@ class RemovedChats: self._take_out(controller, dialog_id, current.pop(dialog_id)[0]) store.set_removed_dialog(account, dialog_id, False) for dialog_id, item in wanted.items(): - if dialog_id in current or controller.dialogs_dict.get(dialog_id) is not None: - continue # listed, or the app has the chat (it is back) + if dialog_id in current: + continue + existing = controller.dialogs_dict.get(dialog_id) + if existing is not None: + # Reloads can leave a previously injected dialog in the + # controller while the in-memory membership cache is reset. + # Match its snapshot top message and re-adopt it. + if int(getattr(existing, "top_message", 0)) == int(item["top_message"]["message_id"]): + current[dialog_id] = ( + existing, + controller.dialogMessage.get(dialog_id), + int(existing.top_message), + ) + # Without this the normal chat sync treats the removed + # chat as live after a reload and re-syncs its history. + store.set_removed_dialog(account, dialog_id, True) + continue current[dialog_id] = self._dialog(controller, account, dialog_id, item) store.set_removed_dialog(account, dialog_id, True) self._put(controller, account) @@ -284,14 +479,22 @@ class RemovedChats: dialog.id = dialog_id dialog.peer = controller.getPeer(dialog_id) dialog.top_message = int(top["message_id"]) + # Deleted chats are historical snapshots: never expose stale unread + # state, otherwise Telegram draws a counter over our deleted badge. + for name, value in (("unread_count", 0), ("unread_mentions_count", 0), ("unread_marked", False)): + if hasattr(dialog, name): + setattr(dialog, name, value) dialog.read_inbox_max_id = dialog.top_message dialog.read_outbox_max_id = dialog.top_message dialog.last_message_date = _epoch(top.get("source_date")) + saved = self._positions.get((account, dialog_id)) or {} + for name in ("pinned", "folder_id", "pinnedNum"): + if name in saved and hasattr(dialog, name): + setattr(dialog, name, saved[name]) from java.util import ArrayList message = removed_chat.from_text(account, dialog_id, top) store.put_server_states(account, dialog_id, [top]) - # dialogMessage holds the dialog's preview messages (an album's parts). preview = ArrayList() preview.add(MessageObject(account, message, False, False)) return dialog, preview, int(dialog.top_message) @@ -314,6 +517,27 @@ class RemovedChats: dialogs.put(dialog_id, dialog) if not everything.contains(dialog): everything.add(dialog) + # Chat search reads its own dialog lists, not allDialogs. Mirror + # the dialog into each one the controller keeps, so a removed + # chat can be found by name like any other. + if dialog_id > 0: + kind_lists = ("dialogsUsersOnly",) + else: + peer = self._peers.get((account, dialog_id)) + broadcast = bool(getattr(peer, "broadcast", False)) + kind_lists = ( + ("dialogsChannelsOnly", "dialogsMyChannels") if broadcast + else ("dialogsGroupsOnly", "dialogsMyGroups", "dialogsCanAddUsers") + ) + for name in ("dialogsServerOnly", "dialogsForward") + kind_lists: + extra = field(controller, name) + if extra is None or not hasattr(extra, "contains"): + continue + try: + if not extra.contains(dialog): + extra.add(dialog) + except Exception: + pass if messages.get(dialog_id) is None: messages.put(dialog_id, message) @@ -324,6 +548,15 @@ class RemovedChats: controller.dialogs_dict.remove(dialog_id) controller.dialogMessage.remove(dialog_id) controller.allDialogs.remove(dialog) + for name in ("dialogsServerOnly", "dialogsForward", "dialogsUsersOnly", + "dialogsGroupsOnly", "dialogsChannelsOnly", "dialogsMyGroups", + "dialogsMyChannels", "dialogsCanAddUsers"): + extra = field(controller, name) + if extra is not None and hasattr(extra, "remove"): + try: + extra.remove(dialog) + except Exception: + pass def _redraw(self, account): from java import jarray @@ -335,7 +568,9 @@ class RemovedChats: ) def before_sort(self, param): - account = int(field(param.thisObject, "currentAccount") or 0) + if not self._injected: + return + account = int(cached_field(param.thisObject, "currentAccount") or 0) if self._injected.get(account): self._put(param.thisObject, account) @@ -368,6 +603,9 @@ class RemovedChats: return if only_history == 1 or not self.api.configured: return + existing = param.thisObject.dialogs_dict.get(dialog_id) + if existing is not None: + self._save_position(account, dialog_id, existing) key = (account, dialog_id) now = time.monotonic() if now - self._reported.get(key, -_REPORT_AGAIN_SECONDS) < _REPORT_AGAIN_SECONDS: @@ -378,6 +616,88 @@ class RemovedChats: return own = _from_screens() in_background(lambda: self._report(account, chat_id, own)) + def after_search_dialogs(self, param): + """searchDialogs(query, ...) ran: append matching removed chats. + + The app searches its own storage, which removed chats are kept out + of on purpose (they must not become cached app chats), so they never + appear on their own. The adapter's result lists are filled + asynchronously, so the chats are added once those lists exist and + the adapter is told to redraw.""" + try: + query = "" + for arg in param.args or []: + if isinstance(arg, str): + query = arg + break + query = (query or "").strip().lower() + if not query or not any(self._injected.values()): + return + adapter = param.thisObject + # The lists are filled asynchronously, so append a moment later + # (twice: the adapter may refill them). Each pass re-reads the + # adapter's current query, so a stale keystroke never adds rows + # for a query the user has already moved past. + on_ui(lambda: self._append_removed(adapter, query), 150) + on_ui(lambda: self._append_removed(adapter, query), 500) + except Exception as exc: + diag.error("removed_chats.search_dialogs", exc) + + def _append_removed(self, adapter, query): + """Put the removed chats matching `query` into the adapter's results.""" + try: + # Prefer the adapter's own current query: this runs delayed, and + # the user may have typed more since. + current = field(adapter, "lastSearchText") + if current is not None: + current = str(current).strip().lower() + if current and current != query: + query = current + if not query: + return + result = field(adapter, "searchResult") + names = field(adapter, "searchResultNames") + if result is None or names is None: + return + listed = set() + for index in range(result.size()): + value = getattr(result.get(index), "id", None) + if value is not None: + listed.add(int(value)) + from android.text import SpannableStringBuilder + + added = 0 + for account in list(self._injected): + for dialog_id in list(self._injected.get(account) or {}): + if not self.is_injected(account, dialog_id): + continue + peer = self._peers.get((account, dialog_id)) + if peer is None: + continue + title = str(getattr(peer, "title", "") or "").strip() + if not title: + first = str(getattr(peer, "first_name", "") or "") + last = str(getattr(peer, "last_name", "") or "") + title = (first + " " + last).strip() + if not title or query not in title.lower(): + continue + key = int(getattr(peer, "id", 0) or 0) + if key in listed: + continue + result.add(peer) + names.add(SpannableStringBuilder(title)) + listed.add(key) + added += 1 + if added: + try: + adapter.notifyDataSetChanged() + except Exception: + pass + if not self._search_logged: + self._search_logged = True + diag.log(f"search: added {added} removed chat(s) for {query!r}") + except Exception as exc: + diag.error("removed_chats.append_removed", exc) def _report(self, account, chat_id, own): try: @@ -390,6 +710,41 @@ class RemovedChats: time.sleep(2) self.refresh(account) + def after_forum_subtitle(self, param): + """A removed forum's header: the app leaves it on "loading". + + TopicsFragment (not ChatActivity) owns the topic list's header, and + it keeps the loading subtitle until channel info arrives, which + never does for a chat whose requests are answered here.""" + fragment = param.thisObject + chat_id = int(cached_field(fragment, "chatId") or 0) + if not chat_id: + return + account = int(cached_field(fragment, "currentAccount") or 0) + if not self.is_injected(account, -chat_id): + return + _hide_forum_join(fragment) + container = cached_field(fragment, "avatarContainer") + if container is None: + return + try: + # Replace ONLY the "connecting/updating/loading" placeholder. + # Overwriting unconditionally fought the app's own subtitle and + # made the header flicker between it and ours every tick. + view = cached_field(container, "subtitleTextView") + current = "" + if view is not None: + try: + current = str(view.getText() or "") + except Exception: + current = "" + if current == _SUBTITLE: + return + if not current or _is_loading(current): + container.setSubtitle(_SUBTITLE) + except Exception as exc: + diag.error("removed_chats.forum_subtitle", exc) + def _hide(self, account, dialog_id): item = self._lists.get(account, {}).get(dialog_id) if item is None: @@ -408,17 +763,43 @@ class RemovedChats: # an opened removed chat + def after_forum_resume(self, param): + """A removed forum became visible: hide its join bar. + + updateSubtitle is not called on every open, so the bar is hidden + here as well (and again once the screen settles).""" + fragment = param.thisObject + chat_id = int(cached_field(fragment, "chatId") or 0) + if not chat_id: + return + account = int(cached_field(fragment, "currentAccount") or 0) + if not self.is_injected(account, -chat_id): + return + _hide_forum_join(fragment) + on_ui(lambda: _hide_forum_join(fragment), 300) + on_ui(lambda: _hide_forum_join(fragment), 900) + def after_overlay(self, param): chat = param.thisObject if self.is_injected(chat.getCurrentAccount(), chat.getDialogId()): make_read_only(chat, _NOTICE) - # Again once the screen settled (a channel's bar comes back). - on_ui(lambda: make_read_only(chat, _NOTICE), 600) + _clear_loading(chat) + on_ui(lambda: (make_read_only(chat, _NOTICE), _clear_loading(chat)), 600) + + def _save_position(self, account, dialog_id, dialog): + saved = {} + for name in ("pinned", "folder_id", "pinnedNum"): + value = field(dialog, name) + if value is not None: + saved[name] = int(value) if isinstance(value, (int, float)) else value + self._positions[(account, dialog_id)] = saved + store.put_meta( + f"removed_positions:{account}", + json.dumps({str(d): value for (a, d), value in self._positions.items() if a == account}), + ) def before_store(self, param): - """putMessages(messages_Messages, dialogId, ...): a removed chat's - logged messages are not stored as the app's own (they would make it - a chat of the app's, read from its cache from then on).""" + """Prevent logged messages from becoming Telegram's cached history.""" args = param.args if len(args) < 2 or not hasattr(args[0], "messages"): return @@ -429,7 +810,6 @@ class RemovedChats: done = args[-1] if done is not None and hasattr(done, "run"): done.run() - def _history(self, account, dialog_id): with self._lock: history = self._histories.get((account, dialog_id)) @@ -442,15 +822,106 @@ class RemovedChats: history.load() return history + def _removed_forum(self, param): + """(account, chat id) when a TopicsController call is about a + removed chat listed here, else None. Called during the topic list's + layout, so it does no reflection unless a removed chat is listed.""" + if not param.args or not any(self._injected.values()): + return None + chat_id = int(param.args[0]) + account = int(cached_field(param.thisObject, "currentAccount") or 0) + if not self.is_injected(account, -chat_id): + return None + return account, chat_id + + def before_topics(self, param): + """TopicsController.getTopics(chatId): a removed forum's topics are + the logger's (from its history), not those the app cached while the + account was in it. Loaded on first use; empty until then.""" + found = self._removed_forum(param) + if found is None: + return + account, chat_id = found + topics = self._topics.get((account, -chat_id)) + if topics is None: + from java.util import ArrayList + + self._load_topics(param.thisObject, account, chat_id) + topics = ArrayList() + param.setResult(topics) + + def before_find_topic(self, param): + """findTopic(chatId, topicId): from the logger's topics, by id (this + is called for every visible row on every layout pass).""" + found = self._removed_forum(param) + if found is None: + return + account, chat_id = found + by_id = self._topic_index.get((account, -chat_id)) or {} + param.setResult(by_id.get(int(param.args[1]))) + + def before_load_topics(self, param): + """loadTopics(chatId, fromCache, loadType): nothing to ask Telegram; + the logger's topics are loaded once (getTopics).""" + found = self._removed_forum(param) + if found is None: + return + param.setResult(None) + account, chat_id = found + self._load_topics(param.thisObject, account, chat_id) + + def _load_topics(self, controller, account, chat_id): + key = (account, -chat_id) + with self._lock: + if key in self._topics or key in self._topics_loading: + return + self._topics_loading.add(key) + + def load(): + history = self._history(account, -chat_id) + response = history.forum_topics() + + def apply(): + from java.util import ArrayList + + topics = ArrayList() + messages = {int(m.id): m for m in java_list(response.messages)} + for topic in java_list(response.topics): + topic.topMessage = messages.get(int(topic.top_message)) + topic.topicStartMessage = _topic_start(controller, chat_id, topic) + topics.add(topic) + self._topics[key] = topics + self._topic_index[key] = {int(t.id): t for t in java_list(topics)} + self._topics_loading.discard(key) + diag.log(f"removed forum {chat_id}: {topics.size()} topics shown") + _topics_loaded(account, chat_id) + + on_ui(apply) + + in_background(load) + def before_request(self, param): request = param.args[0] if request is None or not any(self._injected.values()): return - account = int(field(param.thisObject, "currentAccount") or 0) + account = int(cached_field(param.thisObject, "currentAccount") or 0) dialog_id = _target(request) if dialog_id is None or not self.is_injected(account, dialog_id): return - name = type(request).__name__ + name = type(request).__name__.rsplit("$", 1)[-1] + answered = name in _ANSWERED + if answered: + # Only the requests actually answered here are logged: the app + # retries getOnlines/getSearchCounters/search on a loop for a + # chat it cannot reach, and logging each one wrote to disk on + # every retry. + detail = "" + if name in ("TL_messages_search", "TL_messages_getReplies"): + detail = " " + " ".join( + f"{k}={getattr(request, k, None)}" + for k in ("msg_id", "offset_id", "add_offset", "limit") + ) + diag.log(f"removed chat request {name} {dialog_id}{detail}") delegate, timestamped = param.args[1], param.args[2] if param.method.getReturnType().getName() == "void": param.setResult(None) @@ -458,20 +929,52 @@ class RemovedChats: from java import jint param.setResult(jint(0)) - if name not in _ANSWERED: + if not answered: return # Telegram has nothing for this chat def deliver(): history = self._history(account, dialog_id) - if name == "TLRPC$TL_messages_getHistory": - picked = history.page( - int(request.offset_id), int(request.add_offset), int(request.limit) + if name == "TL_messages_getForumTopics": + response = history.forum_topics( + int(getattr(request, "offset_topic", 0) or 0), + int(getattr(request, "limit", 100) or 100), + ) + elif name == "TL_messages_getForumTopicsByID": + response = history.forum_topics( + only=[int(t) for t in java_list(request.topics)] ) - elif name == "TLRPC$TL_channels_getMessages": - picked = history.by_ids(int(m.id) for m in java_list(request.id)) else: - picked = [] - response = history.response(picked) + total = None + if name == "TL_messages_getHistory": + picked = history.page( + int(request.offset_id), int(request.add_offset), int(request.limit) + ) + total = len(history.messages) + elif name == "TL_messages_getReplies": + picked, total = history.topic_page( + int(request.msg_id), int(request.offset_id), int(request.limit) + ) + # The app probes a thread with limit=1 first. The reply + # must look like a window into a larger thread (a count + # beyond this page, and where this page sits), or the + # app treats the thread as fully loaded and the topic + # never leaves its placeholders. + asked = int(request.limit or 0) + total = int(total or 0) + if not total: + total = len(picked) + (1 if asked and len(picked) >= asked else 0) + elif name == "TL_channels_getMessages": + picked = history.by_ids( + int(getattr(m, "id", m)) for m in java_list(request.id) + ) + else: + picked = [] + response = history.response(picked, total, 0) + if name == "TL_messages_getReplies": + diag.log( + f"removed chat topic {int(request.msg_id)}: " + f"{len(picked)} messages (total {total})" + ) if delegate is not None: delegate.run(response, None) elif timestamped is not None: @@ -504,3 +1007,54 @@ class RemovedChats: ImageLocation.getForPath(path), "50_50", param.args[1], None, target, jint(0) ) param.setResult(None) + + def after_cell_draw(self, param): + """Draw the deleted badge on a removed chat's avatar. + + Hooked on DialogCell.drawAvatarOverlays, which the cell calls under + the same canvas transform it draws the avatar with: drawing there + keeps the badge glued to the avatar through the forum-pane slide + and the chat open/close transitions (drawing after onDraw used the + un-transformed canvas, so the badge flew away from the avatar). + + Runs per drawn cell per frame, so the common path returns before any + reflection and the paints are built once.""" + injected = self._injected + if not injected: + return + cell = param.thisObject + dialog_id = int(cached_field(cell, "currentDialogId") or 0) + if not dialog_id: + return + account = int(cached_field(cell, "currentAccount") or 0) + entry = (injected.get(account) or {}).get(dialog_id) + if entry is None or int(entry[0].top_message) != entry[2]: + return + # TopicsFragment draws its topic rows with DialogCell too, and they + # all carry the forum's dialog id: only real chat-list cells are + # marked. + if cached_field(cell, "forumTopic") is not None: + return + avatar = cached_field(cell, "avatarImage") + if avatar is None: + return + if self._badge is None: + self._badge = _badge_paints() + icon, red, ring, dp = self._badge + canvas = param.args[0] + left = float(avatar.getImageX()) + top = float(avatar.getImageY()) + size = float(avatar.getImageWidth()) + if size <= 0: + return + # Scaled with the avatar, so it shrinks with it in the side pane. + scale = max(0.45, min(1.0, size / dp[54])) + radius = dp[8] * scale + inset = size * 0.14 + cx = left + size - inset + cy = top + float(avatar.getImageHeight()) - inset + canvas.drawCircle(cx, cy, radius + dp[1.5] * scale, ring) + canvas.drawCircle(cx, cy, radius, red) + inner = int(dp[11] * scale) // 2 + icon.setBounds(int(cx) - inner, int(cy) - inner, int(cx) + inner, int(cy) + inner) + icon.draw(canvas) diff --git a/src/jutil.py b/src/jutil.py index 2722392..ee445f0 100644 --- a/src/jutil.py +++ b/src/jutil.py @@ -36,15 +36,76 @@ def methods_named(class_name, name): return [m for m in cls.getDeclaredMethods() if m.getName() == name] +# Hook profiling: per hook name, [calls, total ns, max ns]. Off in releases. +PROFILE = {} +_PROFILE_ON = False + + +def _profiled(hook, label): + """Wrap a MethodHook so each call's time is counted under `label`.""" + if not _PROFILE_ON: + return hook + import time as _time + + from base_plugin import MethodHook + + inner = hook + + class Timed(MethodHook): + def before_hooked_method(self, param): + start = _time.perf_counter_ns() + try: + if hasattr(inner, "before_hooked_method"): + inner.before_hooked_method(param) + finally: + _account(label + ":before", _time.perf_counter_ns() - start) + + def after_hooked_method(self, param): + start = _time.perf_counter_ns() + try: + if hasattr(inner, "after_hooked_method"): + inner.after_hooked_method(param) + finally: + _account(label + ":after", _time.perf_counter_ns() - start) + + return Timed() + + +def _account(label, spent): + entry = PROFILE.get(label) + if entry is None: + entry = PROFILE[label] = [0, 0, 0] + entry[0] += 1 + entry[1] += spent + if spent > entry[2]: + entry[2] = spent + if spent > 16_000_000: + diag.log(f"SLOW {label}: {spent / 1e6:.1f}ms") + + +def profile_report(reset=True): + rows = sorted(PROFILE.items(), key=lambda kv: -kv[1][1]) + lines = [ + f"{label}: n={n} total={total / 1e6:.1f}ms max={peak / 1e6:.1f}ms" + for label, (n, total, peak) in rows + if n + ] + if reset: + PROFILE.clear() + return lines + + def hook_methods(plugin, class_name, name, hook, arity=None): """Hooks every overload of `class_name.name` (or those with `arity` parameters); returns the unhook handles.""" handles = [] + short = class_name.rsplit(".", 1)[-1] for method in methods_named(class_name, name): if arity is not None and len(method.getParameterTypes()) != arity: continue try: - handle = plugin.hook_method(method, hook) + label = f"{short}.{name}/{len(method.getParameterTypes())}" + handle = plugin.hook_method(method, _profiled(hook, label)) if handle is not None: handles.append(handle) except Exception as exc: @@ -67,6 +128,41 @@ def field(obj, name): return None +_FIELDS = {} +_MISSING = object() + + +def cached_field(obj, name): + """`field()` for hot paths: the reflected Field is resolved once per + (class, name) instead of on every call (per-frame draw hooks).""" + try: + cls = obj.getClass() + key = (cls.getName(), name) + except Exception: + return field(obj, name) + found = _FIELDS.get(key, _MISSING) + if found is _MISSING: + found, current = None, cls + while current is not None: + try: + found = current.getDeclaredField(name) + found.setAccessible(True) + break + except Exception: + found = None + try: + current = current.getSuperclass() + except Exception: + current = None + _FIELDS[key] = found + if found is None: + return field(obj, name) + try: + return found.get(obj) + except Exception: + return field(obj, name) + + def on_ui(fn, delay_ms=0): from android_utils import run_on_ui_thread diff --git a/src/main.py b/src/main.py index d793fab..455f24a 100644 --- a/src/main.py +++ b/src/main.py @@ -92,7 +92,7 @@ class MessageLoggerPlugin(BasePlugin): ) in_background(accounts.refresh) in_background(self.purge_cache) - diag.log("loaded") + diag.log(f"loaded (server {self.get_setting(SERVER_URL, '') or 'unset'})") def _reset_chat(self, context): """Chat menu: forget this chat's cached marks and check it again.""" diff --git a/src/people.py b/src/people.py new file mode 100644 index 0000000..e39dc8c --- /dev/null +++ b/src/people.py @@ -0,0 +1,182 @@ +"""The last known profile of every sender shown from the logger. + +The logger stores only a sender's id and name with each message, so the +app's own user/chat is what carries the avatar, username and the rest. That +object can be gone (the account never saw the sender again, the app's cache +was cleared) or useless (a deleted account: no name, no photo). So each time +a sender is readable here its full TLRPC object is saved, serialized, and it +is used whenever the app has nothing better. + +Resolution, best first: the app's cache, the app's database, the snapshot +saved here, a stand-in named from the logger, "Deleted". +""" + +import base64 +import threading + +import diag +import store + +_lock = threading.Lock() +_memory = {} # (account, peer_id) -> TLRPC.User/Chat +_saved = {} # (account, peer_id) -> fingerprint of the stored snapshot + + +def _key(account, peer_id): + return f"person:{int(account)}:{int(peer_id)}" + + +def _usable_user(user): + """A user worth showing: has a name, and isn't an emptied deleted + account (which the app keeps with no name and no photo).""" + if user is None: + return False + if type(user).__name__.endswith("userEmpty"): + return False + name = (getattr(user, "first_name", None) or "") + (getattr(user, "last_name", None) or "") + return bool(name.strip()) + + +def _usable_chat(chat): + if chat is None: + return False + if type(chat).__name__.endswith(("chatEmpty", "channelForbidden", "chatForbidden")): + return bool((getattr(chat, "title", None) or "").strip()) + return bool((getattr(chat, "title", None) or "").strip()) + + +def _fingerprint(obj): + photo = getattr(obj, "photo", None) + return ( + getattr(obj, "first_name", None), + getattr(obj, "last_name", None), + getattr(obj, "title", None), + getattr(obj, "username", None), + int(getattr(photo, "photo_id", 0) or 0) if photo is not None else 0, + ) + + +def _serialize(obj): + from org.telegram.tgnet import SerializedData + + data = SerializedData(obj.getObjectSize()) + obj.serializeToStream(data) + raw = bytes(data.toByteArray()) + data.cleanup() + return base64.b64encode(raw).decode("ascii") + + +def _deserialize(text, is_user): + from org.telegram.tgnet import SerializedData, TLRPC + + raw = base64.b64decode(text) + data = SerializedData(raw) + try: + constructor = data.readInt32(False) + cls = TLRPC.User if is_user else TLRPC.Chat + return cls.TLdeserialize(data, constructor, False) + finally: + data.cleanup() + + +def remember(account, peer_id, obj): + """Save `obj` (a User for peer_id > 0, a Chat for < 0) when it is worth + keeping and differs from the last snapshot.""" + usable = _usable_user(obj) if peer_id > 0 else _usable_chat(obj) + if not usable: + return + key = (int(account), int(peer_id)) + mark = _fingerprint(obj) + with _lock: + _memory[key] = obj + if _saved.get(key) == mark: + return + _saved[key] = mark + try: + text = _serialize(obj) + # After a restart nothing is marked as saved yet: skip the write + # when the stored snapshot is already this one. + if store.get_meta(_key(account, peer_id)) != text: + store.put_meta(_key(account, peer_id), text) + except Exception as exc: + diag.error("people.remember", exc) + + +def recall(account, peer_id): + """The last snapshot saved for the sender, else None.""" + key = (int(account), int(peer_id)) + with _lock: + if key in _memory: + return _memory[key] + text = store.get_meta(_key(account, peer_id)) + if not text: + return None + try: + obj = _deserialize(text, int(peer_id) > 0) + except Exception as exc: + diag.error("people.recall", exc) + return None + if obj is None: + return None + with _lock: + _memory[key] = obj + _saved[key] = _fingerprint(obj) + return obj + + +def _from_app(account, peer_id): + """The app's object from its in-memory cache. + + Not its database: getUserSync/getChatSync block on the storage thread, + and this runs while a topic's history is being built, so a busy storage + queue stalled the whole topic. What the database would add is what the + saved snapshot already holds.""" + from org.telegram.messenger import MessagesController + + controller = MessagesController.getInstance(int(account)) + if peer_id > 0: + return controller.getUser(int(peer_id)) + return controller.getChat(-int(peer_id)) + + +def user(account, peer_id, logged_name=None): + """The best User for a sender (peer_id > 0).""" + from org.telegram.tgnet import TLRPC + + found = _from_app(account, peer_id) + if _usable_user(found): + remember(account, peer_id, found) + return found + saved = recall(account, peer_id) + if _usable_user(saved): + return saved + stand_in = TLRPC.TL_user() + stand_in.id = int(peer_id) + stand_in.first_name = logged_name or "Deleted" + stand_in.flags |= 2 + return stand_in + + +def chat(account, chat_id, logged_title=None, kind="channel"): + """The best Chat for a sender or a removed chat (chat_id > 0).""" + from org.telegram.tgnet import TLRPC + + peer_id = -int(chat_id) + found = _from_app(account, peer_id) + if _usable_chat(found): + remember(account, peer_id, found) + return found + saved = recall(account, peer_id) + if _usable_chat(saved): + return saved + if found is not None: + return found + if kind == "group": + stand_in = TLRPC.TL_chat() + else: + stand_in = TLRPC.TL_channel() + stand_in.broadcast = kind == "channel" + stand_in.megagroup = kind != "channel" + stand_in.id = int(chat_id) + stand_in.title = logged_title or "Chat" + return stand_in diff --git a/src/views/comments.py b/src/views/comments.py index 1bd5695..e5e081d 100644 --- a/src/views/comments.py +++ b/src/views/comments.py @@ -47,15 +47,35 @@ def make_read_only(chat, notice="Viewing deleted comments"): text = field(chat, "bottomOverlayText") if overlay is None or text is None: return + # Hide every other control the app may put in the bottom bar. The names + # differ per build and per chat kind (a forum/discussion screen shows a + # start/join button of its own), so the ones we know are hidden by name + # and anything else inside the overlay is hidden by walking it: leaving + # one visible is what kept drawing a button over the notice. for name in ( "bottomOverlayChat", # a channel's bar (search, mute/join) "bottomOverlayChatText", "bottomOverlayStartButton", "bottomOverlayProgress", + "bottomOverlayImage", + "bottomOverlayLinks", + "bottomOverlayChatText2", + "bottomOverlayChatWaitsReply", + "replyButton", + "unblockButton", + "joinButton", + "addToContactsButton", + "reportSpamButton", + "topViewSeparator", ): other = field(chat, name) - if other is not None: - other.setVisibility(View.GONE) + if other is not None and other is not text: + try: + other.setVisibility(View.GONE) + except Exception: + pass + _hide_siblings(overlay, text, View) + _hide_overlay_peers(overlay, View) # The app animates the overlay out when the thread has an input bar: # stop that, or it ends invisible (alpha 0) after this. animation = field(chat, "bottomOverlayAnimation") @@ -63,10 +83,89 @@ def make_read_only(chat, notice="Viewing deleted comments"): animation.cancel() text.setText(notice) text.setVisibility(View.VISIBLE) + text.setAlpha(1.0) overlay.setAlpha(1.0) overlay.setVisibility(View.VISIBLE) +def _hide_siblings(overlay, keep, View): + """Hide every child of the bottom overlay except the notice label. + + The read-only notice is the app's own `bottomOverlayText`; any other + child (a join/start button, a progress spinner) would otherwise be + drawn over it. Chaquopy may hand out distinct Python wrappers for the + same Java view, so the one to keep is matched with Java equality + (`equals`) rather than Python identity.""" + try: + from android.view import ViewGroup + + if not isinstance(overlay, ViewGroup): + return + for index in range(overlay.getChildCount()): + child = overlay.getChildAt(index) + if child is None: + continue + try: + if child.equals(keep): + continue + except Exception: + if child is keep: + continue + try: + child.setVisibility(View.GONE) + except Exception: + pass + try: + keep.bringToFront() + except Exception: + pass + except Exception as exc: + diag.error("make_read_only.siblings", exc) + + +def _hide_overlay_peers(overlay, View): + """Hide sibling views that share the bottom strip with the overlay. + + Some screens (a forum/discussion ChatActivity) put their action button + next to `bottomOverlay` instead of inside it, so hiding the overlay's + own children is not enough: the button keeps covering the notice.""" + try: + from android.view import ViewGroup + + parent = overlay.getParent() + if parent is None or not isinstance(parent, ViewGroup): + return + top, bottom = overlay.getTop(), overlay.getBottom() + for index in range(parent.getChildCount()): + child = parent.getChildAt(index) + if child is None: + continue + try: + if child.equals(overlay): + continue + except Exception: + if child is overlay: + continue + try: + if child.getVisibility() != View.VISIBLE: + continue + # Only the views overlapping the overlay's band: the + # message list and the action bar must stay untouched. + if child.getBottom() <= top or child.getTop() >= bottom: + continue + # The message list spans the whole screen and so overlaps + # the band too; hiding it left topics on their loading + # placeholders. Only bar-sized views are buttons. + band = max(1, bottom - top) + if child.getHeight() > band * 2: + continue + child.setVisibility(View.GONE) + except Exception: + pass + except Exception as exc: + diag.error("make_read_only.peers", exc) + + def answer(account, request): """The response for a getReplies of a thread opened here, else None.""" with _lock: @@ -102,28 +201,17 @@ def _peer(controller, marked): return controller.getPeer(_android_peer_id(int(marked))) -def _sender_object(controller, marked, name): - """The sender's User/Chat from the account's cache, else a stand-in with - the logged name (avatar drawn from its initials).""" +def _sender_object(account, marked, name): + """The sender's User/Chat: the app's, else the last snapshot saved while + it was readable (name, username, photo), else a stand-in with the + logged name (avatar drawn from its initials).""" + import people from hooks.chat import _android_peer_id - from org.telegram.tgnet import TLRPC peer_id = _android_peer_id(int(marked)) if peer_id > 0: - user = controller.getUser(peer_id) - if user is None: - user = TLRPC.TL_user() - user.id = peer_id - user.first_name = name or "Deleted" - user.flags |= 2 - return user, None - chat = controller.getChat(-peer_id) - if chat is None: - chat = TLRPC.TL_channel() - chat.id = -peer_id - chat.title = name or "Channel" - chat.broadcast = True - return None, chat + return people.user(account, peer_id, name), None + return None, people.chat(account, -peer_id, name, "channel") class Comments(History): @@ -221,7 +309,7 @@ class Comments(History): self._into_thread(msg, item, discussion, top, sources) if item.get("sender_id"): user, chat = _sender_object( - controller, item["sender_id"], item.get("sender_name") + account, item["sender_id"], item.get("sender_name") ) if user is not None: users[user.id] = user diff --git a/src/views/removed_chat.py b/src/views/removed_chat.py index cf4f997..db80427 100644 --- a/src/views/removed_chat.py +++ b/src/views/removed_chat.py @@ -9,6 +9,7 @@ message. getHistory/getMessages for it (hooks/removed_chats.py). """ +import json import threading import diag @@ -23,34 +24,19 @@ _PAGES = 3 # history pages (of 100) loaded when the chat opens _COPIES_PER_REQUEST = 100 -def stand_in_user(controller, peer_id, name): - from org.telegram.tgnet import TLRPC +def stand_in_user(account, peer_id, name): + """The sender's User: the app's, else its last saved snapshot (name, + username, photo), else a stand-in with the logged name.""" + import people - user = controller.getUser(int(peer_id)) - if user is None: - user = TLRPC.TL_user() - user.id = int(peer_id) - user.first_name = name or "Deleted" - user.flags |= 2 - return user + return people.user(account, peer_id, name) -def stand_in_chat(controller, chat_id, title, kind): - """The app's chat (cached or stored), else a stand-in of the right kind.""" - from org.telegram.tgnet import TLRPC +def stand_in_chat(account, chat_id, title, kind): + """The app's chat, else its last saved snapshot, else a stand-in.""" + import people - chat = controller.getChat(int(chat_id)) - if chat is not None: - return chat - if kind == "group": - chat = TLRPC.TL_chat() - else: - chat = TLRPC.TL_channel() - chat.broadcast = kind == "channel" - chat.megagroup = kind != "channel" - chat.id = int(chat_id) - chat.title = title or "Chat" - return chat + return people.chat(account, chat_id, title, kind) class History: @@ -62,13 +48,23 @@ class History: self.messages = [] # TL messages, newest first self.users = {} self.chats = {} + # Forum topics from the server: [{"id", "title", "icon_emoji_id"}], + # newest activity first; empty when the chat is no forum. + self.topics = [] + self._topic_of = {} # message id -> forum topic id (1: General) + self._latest_by_topic = {} self.loaded = False self._lock = threading.Lock() + self._lock_copies = threading.Lock() + self._copy_cache = {} # (log chat, message id) -> copy (TL message) + self._media_done = set() # (log chat, message id) whose media is local + self._pages = {} # (topic id, before) -> (messages, total) + self._downloading = False + self._queue = [] # [(items, master)] waiting for their media # loading (background) - def load(self): - """Fetches and builds the chat's history once; later calls wait.""" + """Fetch and build the topic-preview messages once.""" with self._lock: if self.loaded: return @@ -80,25 +76,76 @@ class History: def _load(self): me = peers.own_user_id(self.account) - items, before, master = [], 0, None - for _page in range(_PAGES): + disk_key = f"chat_head:{self.account}:{self.chat_id}" + result = None + for attempt in range(2): try: - result = self.api.history(me, self.chat_id, before) + result = self.api.history(me, self.chat_id, 0, limit=1) + break except ApiError as exc: - diag.log(f"removed chat {self.chat_id}: {exc}") - break - page = (result or {}).get("messages") or [] - if not page: - break - master = result.get("master_tg_id") - items += page - before = min(int(m["message_id"]) for m in page) - if len(page) < 100: - break + diag.log(f"removed chat {self.chat_id} try {attempt + 1}: {exc}") + if result: + try: + store.put_meta(disk_key, json.dumps(result)) + except Exception as exc: + diag.error("removed_chat.head_cache", exc) + else: + # Unreachable: the last answer saved on this device, so the + # chat still opens with its topics and their latest messages. + saved = store.get_meta(disk_key) + if not saved: + return + result = json.loads(saved) + diag.log(f"removed chat {self.chat_id}: from device cache") + result = result or {} + self.topics = list(result.get("topics") or []) + items = list(result.get("messages") or []) + for topic in self.topics: + latest = topic.get("latest") or {} + if latest and int(latest.get("message_id") or 0) not in { + int(i.get("message_id") or 0) for i in items + }: + items.append(latest) + master = result.get("master_tg_id") revealed = store.is_revealed(self.account, self.dialog_id) items = [i for i in items if revealed or not i.get("hidden_at")] store.put_server_states(self.account, self.dialog_id, items) - copies = self._copies(items, master) + # The topic previews need the logged copies (so media shows instead + # of " unavailable"), but downloading every file here would + # block the topic list for as long as the transfers take. Build with + # the copies now and fetch their files in the background. + self._build(items, master, fetch_copies=True, download=False) + self._download_later(items, master) + + def _download_later(self, items, master): + """Queue the copies' media for download off the UI path. + + copy_media blocks per file (up to 20s each), so it never runs while a + page is being answered. One worker per chat drains the queue: a + thread per page made several pages download at once and compete + with the UI while the chat was opening.""" + with self._lock_copies: + self._queue.append((list(items), master)) + if self._downloading: + return + self._downloading = True + + def run(): + while True: + with self._lock_copies: + if not self._queue: + self._downloading = False + return + batch, batch_master = self._queue.pop(0) + try: + self._copies(batch, batch_master, download=True) + except Exception as exc: + diag.error("removed_chat.download", exc) + + threading.Thread(target=run, daemon=True).start() + + def _build(self, items, master=None, fetch_copies=True, download=True): + copies = self._copies(items, master, download=download) if fetch_copies else {} built = [] for item in items: try: @@ -106,7 +153,97 @@ class History: except Exception as exc: diag.error("removed_chat.message", exc) self.messages = sorted(built, key=lambda m: -int(m.id)) - diag.log(f"removed chat {self.chat_id}: {len(self.messages)} messages") + self._latest_by_topic = {} + if self.topics: + ids = {int(t["id"]) for t in self.topics} + by_id = {int(m.id): m for m in self.messages} + for item in items: + message = by_id.get(int(item["message_id"])) + if message is None: + continue + topic = int(item.get("source_topic_id") or 0) + topic = topic if topic in ids else 1 + self._topic_of[int(message.id)] = topic + self._latest_by_topic.setdefault(topic, message) + diag.log( + f"removed chat {self.chat_id}: {len(self.messages)} latest messages, " + f"{len(self.topics)} topics" + ) + + def topic_page(self, topic_id, before=0, limit=100): + """A topic's logged messages, newest first, with its total count. + + Returns (messages, total): the app pages with limit=1 first, so the + total must describe the whole topic, not this slice.""" + # A page already built is answered from memory: reopening a topic + # used to refetch it from the server and rebuild every message. + key = (int(topic_id), int(before or 0)) + with self._lock_copies: + page = self._pages.get(key) + if page is not None: + messages, total = page + asked = int(limit or 0) + return (messages[:asked] if asked > 0 else messages), total + me = peers.own_user_id(self.account) + # The app probes a thread with limit=1 before asking for a page. + # Passing that straight through fetched a single message and the + # topic stayed on its loading placeholders, so a full page is + # always fetched and the slice is taken below. + want = max(int(limit or 0), 100) + disk_key = f"topic_page:{self.account}:{self.chat_id}:{int(topic_id)}:{int(before or 0)}" + result = None + # Up to two tries: a single slow answer from the server used to leave + # the topic empty for good (it looked frozen). + for attempt in range(2): + try: + result = self.api.history( + me, self.chat_id, before, min(100, want), topic_id + ) or {} + break + except ApiError as exc: + diag.log(f"removed topic {self.chat_id}/{topic_id} try {attempt + 1}: {exc}") + if result is not None and result.get("messages"): + try: + store.put_meta(disk_key, json.dumps(result)) + except Exception as exc: + diag.error("removed_chat.page_cache", exc) + else: + # Unreachable or empty: the last answer saved on this device. + saved = store.get_meta(disk_key) + result = json.loads(saved) if saved else None + if result is None: + return [], 0 + diag.log(f"removed topic {self.chat_id}/{topic_id}: from device cache") + items = result.get("messages") or [] + revealed = store.is_revealed(self.account, self.dialog_id) + items = [i for i in items if revealed or not i.get("hidden_at")] + # Without the logged copies every message falls back to its server + # text and its media is marked unavailable: no previews in a topic. + # The copies are read now (their media is already in them, by file + # id), but the files are downloaded in the background: downloading + # them inline held the page for up to 20s per file, so the next + # page request waited and the topic sat on its placeholders. + master = result.get("master_tg_id") + copies = self._copies(items, master, download=False) + self._download_later(items, master) + built = [] + for item in items: + try: + built.append(self._message(item, copies.get(self._pointer(item)))) + except Exception as exc: + diag.error("removed_chat.topic_message", exc) + ordered = sorted(built, key=lambda m: -int(m.id)) + total = len(ordered) + if ordered: + with self._lock_copies: + self._pages[key] = (ordered, total) + # Answer with at most what was asked for: the app drops a reply that + # holds more messages than its limit, which left the topic on its + # placeholders even though the history had arrived. + asked = int(limit or 0) + if asked > 0: + ordered = ordered[:asked] + return ordered, total @staticmethod def _pointer(item): @@ -115,7 +252,7 @@ class History: return None return int(logged["chat_id"]), int(logged["message_id"]) - def _copies(self, items, master): + def _copies(self, items, master, download=True): """Logged copies by (log chat, message), read through an account here that can, per log group.""" by_group = {} @@ -126,19 +263,67 @@ class History: found = {} masters = [master, *self.api.masters()] for log_chat, ids in by_group.items(): + # Copies already read are reused: a reopened chat or topic would + # otherwise ask Telegram for every logged copy again. + with self._lock_copies: + missing = [i for i in ids if (log_chat, i) not in self._copy_cache] + for i in ids: + cached = self._copy_cache.get((log_chat, i)) + if cached is not None: + found[(log_chat, i)] = cached + if not missing: + if download: + self._download_copies(log_chat, ids, master, found) + continue channel_id = peers.channel_id_from_marked(log_chat) reader = media_share.reader_for(self.account, log_chat, masters) if reader is None: + # No signed-in account is a member of this log group, so its + # copies (and their media) cannot be read at all: every + # message falls back to " unavailable". + diag.log(f"copies: no reader for log group {log_chat} ({len(missing)} messages)") continue - ordered = sorted(ids) + ordered = sorted(missing) + fetched = 0 for start in range(0, len(ordered), _COPIES_PER_REQUEST): chunk = ordered[start : start + _COPIES_PER_REQUEST] for message_id, copy in _fetch(reader, -channel_id, chunk).items(): - if reader != self.account: - media_share.copy_media(reader, copy) + fetched += 1 found[(log_chat, message_id)] = copy + with self._lock_copies: + self._copy_cache[(log_chat, message_id)] = copy + diag.log( + f"copies: log group {log_chat} reader={reader} " + f"asked={len(ordered)} got={fetched}" + ) + if download: + self._download_copies(log_chat, ids, master, found) return found + def _download_copies(self, log_chat, ids, master, found): + """Download each copy's media once (blocking; background only). + + The media is fetched through a log-group member by the copy's own + file reference. Files already downloaded are skipped, so reopening a + chat costs nothing; a file that failed is retried on a later open.""" + reader = media_share.reader_for(self.account, log_chat, [master, *self.api.masters()]) + if reader is None: + return + ready = failed = 0 + for i in ids: + key = (log_chat, i) + copy = found.get(key) + if copy is None or key in self._media_done: + continue + got, total = media_share.copy_media(reader, copy) + if got >= total: + self._media_done.add(key) + ready += got + else: + failed += total - got + if ready or failed: + diag.log(f"media: log group {log_chat} ready={ready} failed={failed}") + # building def _message(self, item, copy): @@ -159,12 +344,27 @@ class History: if sender: peer_id = _android_peer_id(int(sender)) if peer_id > 0: - user = stand_in_user(controller, peer_id, item.get("sender_name")) + user = stand_in_user(self.account, peer_id, item.get("sender_name")) self.users[int(user.id)] = user else: - chat = stand_in_chat(controller, -peer_id, item.get("sender_name"), "channel") + chat = stand_in_chat(self.account, -peer_id, item.get("sender_name"), "channel") self.chats[int(chat.id)] = chat - if item.get("reply_to"): + topic = int(item.get("source_topic_id") or 0) + if topic and self.topics: + # In a forum topic: the reply header names it (its start message + # id), with the replied message when there is one. + reply = TLRPC.TL_messageReplyHeader() + reply.forum_topic = True + reply.flags |= 8 + target = int(item.get("reply_to") or 0) + reply.reply_to_msg_id = target or topic + reply.flags |= 16 + if target and target != topic: + reply.reply_to_top_id = topic + reply.flags |= 2 + msg.reply_to = reply + msg.flags |= 8 + elif item.get("reply_to"): reply = TLRPC.TL_messageReplyHeader() reply.reply_to_msg_id = int(item["reply_to"]) reply.flags |= 16 @@ -172,9 +372,113 @@ class History: msg.flags |= 8 return msg + # forum topics + + def topic_of(self, message): + return self._topic_of.get(int(message.id), 1) + + def topic_messages(self, topic_id): + return [self._latest_by_topic[topic_id]] if int(topic_id) in self._latest_by_topic else [] + + def forum_topics(self, offset_topic=0, limit=100, only=None): + """messages.getForumTopics' answer: General first, then the topics by + newest message; each with its newest message. Every topic asked for + (`only`) is answered, empty ones too, or the app asks again forever.""" + from java.util import ArrayList + from org.telegram.tgnet import TLRPC + entries = list(self.topics) + if not any(int(e["id"]) == 1 for e in entries): + entries.insert(0, {"id": 1, "title": "General"}) + + def _recency(entry): + """Newest logged activity in the topic (0 when nothing logged).""" + latest = entry.get("latest") or {} + return _epoch(latest.get("source_date")) if latest else 0 + + # Telegram orders a forum by the most recent topic, General first. + # The logger records no pinned flag or order (the server sends only + # id/title/icon/latest), so recency is the ordering that can be + # rebuilt faithfully; without this the topics came out in the + # server's arbitrary order. + if only is None: + general = [e for e in entries if int(e["id"]) == 1] + rest = [e for e in entries if int(e["id"]) != 1] + rest.sort(key=_recency, reverse=True) + entries = general + rest + if only is not None: + known = {int(e["id"]) for e in entries} + entries = [e for e in entries if int(e["id"]) in only] + [ + {"id": i, "title": f"Topic {i}"} for i in only if i not in known + ] + elif offset_topic: + ids = [int(e["id"]) for e in entries] + if int(offset_topic) in ids: + entries = entries[ids.index(int(offset_topic)) + 1 :] + entries = entries[: max(1, int(limit or 100))] + response = TLRPC.TL_messages_forumTopics() + response.topics = ArrayList() + response.messages = ArrayList() + tops = set() + for entry in entries: + topic_id = int(entry["id"]) + messages = self.topic_messages(topic_id) + if not messages and only is None: + continue + topic = TLRPC.TL_forumTopic() + topic.id = topic_id + topic.title = str(entry.get("title") or f"Topic {topic_id}") + icon = entry.get("icon_emoji_id") + if icon: + topic.icon_emoji_id = int(icon) + topic.flags |= 1 + topic.icon_color = 0x6FB9F0 + if not messages: + # Nothing logged in it: an empty stand-in, so the app has a + # top message and does not ask for the topic again. + empty = TLRPC.TL_message() + empty.id = topic_id + empty.peer_id = self.messages[0].peer_id if self.messages else None + empty.message = "" + messages = [empty] + # The newest message dates the topic: messages are newest-first, + # so messages[-1] is the OLDEST and dating the topic by it put + # active topics at the bottom of the forum. + topic.date = int(messages[0].date) + topic.top_message = int(messages[0].id) + # Deleted topics are read-only snapshots; don't let stale topic + # unread state create a badge in the topic list. + topic.read_inbox_max_id = topic.top_message + topic.read_outbox_max_id = topic.top_message + if hasattr(topic, "unread_count"): + topic.unread_count = 0 + if hasattr(topic, "unread_mentions_count"): + topic.unread_mentions_count = 0 + # The topic's creator shown in the list is the newest message's + # sender, matching the preview drawn from messages[0]. + if messages[0].from_id is not None: + topic.from_id = messages[0].from_id + response.topics.add(topic) + if topic.top_message not in tops: + tops.add(topic.top_message) + response.messages.add(messages[0]) + response.count = response.topics.size() + response.pts = 0 + response.users = ArrayList() + for user in self.users.values(): + response.users.add(user) + response.chats = ArrayList() + for chat in self.chats.values(): + response.chats.add(chat) + return response + # answering the app's requests - def response(self, picked): + def response(self, picked, total=None, offset=None): + """A getHistory/getReplies answer. + + TL_messages_messages is what the app expects here: switching to + TL_messages_channelMessages (to carry a count) made every topic + hang on its loading placeholders, so the plain form is kept.""" from java.util import ArrayList from org.telegram.tgnet import TLRPC @@ -190,10 +494,10 @@ class History: response.chats.add(chat) return response - def page(self, offset_id, add_offset, limit): + def page(self, offset_id, add_offset, limit, messages=None): """getHistory's slice: newest first, from `offset_id` (exclusive; 0: the newest) moved by `add_offset` (negative: newer ones too).""" - messages = self.messages + messages = self.messages if messages is None else messages if offset_id: start = next( (i for i, m in enumerate(messages) if int(m.id) < offset_id),