From 63e52169a3e176c46480358bcd0edff584f8ac5c Mon Sep 17 00:00:00 2001 From: Kumi Date: Thu, 14 Dec 2023 18:10:12 +0100 Subject: [PATCH] Fix file handling in encrypted rooms and update dependencies Resolved an issue that prevented the bot from responding when files were uploaded to encrypted rooms by implementing a workaround. The bot now tries to generate text from uploaded files and logs errors without interrupting the message flow. Upgraded the Pantalaimon dependency to ensure compatibility. Also, refined the message processing logic to handle different message types correctly and made the download_file method asynchronous to match the matrix client's expected behavior. Additionally, updated the changelog and bumped the project version to reflect these fixes and improvements. Known issues have been documented, including a limitation when using Pantalaimon where the bot cannot download/use files uploaded to encrypted rooms. --- CHANGELOG.md | 7 ++++- pyproject.toml | 4 +-- src/gptbot/classes/bot.py | 66 ++++++++++++++++++++++++++++----------- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcf54da..6e17bea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,14 @@ # Changelog -### 0.3.2 (in the works) +### 0.3.2 (2023-12-14) * Removed key upload from room event handler * Fixed output of `python -m gptbot -v` to display currently installed version +* Workaround for bug preventing bot from responding when files are uploaded to an encrypted room + +#### Known Issues + +* When using Pantalaimon: Bot is unable to download/use files uploaded to encrypted rooms ### 0.3.1 (2023-12-07) diff --git a/pyproject.toml b/pyproject.toml index b2e727d..d5133fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ allow-direct-references = true [project] name = "matrix-gptbot" -version = "0.3.2-dev0" +version = "0.3.2" authors = [ { name="Kumi Mitterer", email="gptbot@kumi.email" }, @@ -47,7 +47,7 @@ wolframalpha = [ ] e2ee = [ - "pantalaimon", + "pantalaimon>=0.10.5", ] all = [ diff --git a/src/gptbot/classes/bot.py b/src/gptbot/classes/bot.py index a1bd219..8a45d2b 100644 --- a/src/gptbot/classes/bot.py +++ b/src/gptbot/classes/bot.py @@ -1054,7 +1054,7 @@ class GPTBot: message_body = message.body if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message.body}] chat_messages.append({"role": role, "content": message_body}) - if isinstance(message, RoomMessageAudio) or (isinstance(message, RoomMessageFile) and message.body.endswith(".mp3")): + elif isinstance(message, RoomMessageAudio) or (isinstance(message, RoomMessageFile) and message.body.endswith(".mp3")): role = ( "assistant" if message.sender == self.matrix_client.user_id else "user" ) @@ -1072,24 +1072,54 @@ class GPTBot: message_body = message_text if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message_text}] chat_messages.append({"role": role, "content": message_body}) - if self.chat_api.supports_chat_images() and isinstance(message, RoomMessageImage): - image_url = message.url - download = await self.download_file(image_url) + elif isinstance(message, RoomMessageFile): + try: + download = await self.download_file(message.url) + if download: + try: + text = download.body.decode("utf-8") + except UnicodeDecodeError: + text = None - if download: - encoded_url = f"data:{download.content_type};base64,{base64.b64encode(download.body).decode('utf-8')}" - parent = chat_messages[-1] if chat_messages and chat_messages[-1]["role"] == ("assistant" if message.sender == self.matrix_client.user_id else "user") else None + if text: + role = ( + "assistant" + if message.sender == self.matrix_client.user_id + else "user" + ) + if message == event or (not message.event_id == event.event_id): + message_body = text if not self.chat_api.supports_chat_images() else [{"type": "text", "text": text}] + chat_messages.append({"role": role, "content": message_body}) - if not parent: - chat_messages.append({"role": ("assistant" if message.sender == self.matrix_client.user_id else "user"), "content": []}) - parent = chat_messages[-1] + except Exception as e: + self.logger.log(f"Error generating text from file: {e}", "error") + message_body = message.body if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message.body}] + chat_messages.append({"role": "system", "content": message_body}) - parent["content"].append({ - "type": "image_url", - "image_url": { - "url": encoded_url - } - }) + elif self.chat_api.supports_chat_images() and isinstance(message, RoomMessageImage): + try: + image_url = message.url + download = await self.download_file(image_url) + + if download: + encoded_url = f"data:{download.content_type};base64,{base64.b64encode(download.body).decode('utf-8')}" + parent = chat_messages[-1] if chat_messages and chat_messages[-1]["role"] == ("assistant" if message.sender == self.matrix_client.user_id else "user") else None + + if not parent: + chat_messages.append({"role": ("assistant" if message.sender == self.matrix_client.user_id else "user"), "content": []}) + parent = chat_messages[-1] + + parent["content"].append({ + "type": "image_url", + "image_url": { + "url": encoded_url + } + }) + + except Exception as e: + self.logger.log(f"Error generating image from file: {e}", "error") + message_body = message.body if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message.body}] + chat_messages.append({"role": "system", "content": message_body}) # Truncate messages to fit within the token limit truncated_messages = self._truncate( @@ -1136,7 +1166,7 @@ class GPTBot: await self.matrix_client.room_typing(room.room_id, False) - def download_file(self, mxc) -> Optional[bytes]: + async def download_file(self, mxc) -> Optional[bytes]: """Download a file from the homeserver. Args: @@ -1146,7 +1176,7 @@ class GPTBot: Optional[bytes]: The downloaded file, or None if there was an error. """ - download = self.matrix_client.download(mxc) + download = await self.matrix_client.download(mxc) if isinstance(download, DownloadError): self.logger.log(f"Error downloading file: {download.message}", "error")