Fix file handling in encrypted rooms and update dependencies

Resolved an issue that prevented the bot from responding when files were uploaded to encrypted rooms by implementing a workaround. The bot now tries to generate text from uploaded files and logs errors without interrupting the message flow. Upgraded the Pantalaimon dependency to ensure compatibility. Also, refined the message processing logic to handle different message types correctly and made the download_file method asynchronous to match the matrix client's expected behavior. Additionally, updated the changelog and bumped the project version to reflect these fixes and improvements.

Known issues have been documented, including a limitation when using Pantalaimon where the bot cannot download/use files uploaded to encrypted rooms.

CHANGELOG.md

@@ -1,9 +1,14 @@
 # Changelog
 
-### 0.3.2 (in the works)
+### 0.3.2 (2023-12-14)
 
 * Removed key upload from room event handler
 * Fixed output of `python -m gptbot -v` to display currently installed version
+* Workaround for bug preventing bot from responding when files are uploaded to an encrypted room
+
+#### Known Issues
+
+* When using Pantalaimon: Bot is unable to download/use files uploaded to encrypted rooms
 
 ### 0.3.1 (2023-12-07)
 

pyproject.toml

@@ -7,7 +7,7 @@ allow-direct-references = true
 
 [project]
 name = "matrix-gptbot"
-version = "0.3.2-dev0"
+version = "0.3.2"
 
 authors = [
   { name="Kumi Mitterer", email="gptbot@kumi.email" },
@@ -47,7 +47,7 @@ wolframalpha = [
 ]
 
 e2ee = [
-    "pantalaimon",
+    "pantalaimon>=0.10.5",
 ]
 
 all = [

src/gptbot/classes/bot.py

@@ -1054,7 +1054,7 @@ class GPTBot:
                     message_body = message.body if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message.body}]
                     chat_messages.append({"role": role, "content": message_body})
 
-            if isinstance(message, RoomMessageAudio) or (isinstance(message, RoomMessageFile) and message.body.endswith(".mp3")):
+            elif isinstance(message, RoomMessageAudio) or (isinstance(message, RoomMessageFile) and message.body.endswith(".mp3")):
                 role = (
                     "assistant" if message.sender == self.matrix_client.user_id else "user"
                 )
@@ -1072,24 +1072,54 @@ class GPTBot:
                     message_body = message_text if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message_text}]
                     chat_messages.append({"role": role, "content": message_body})
 
-            if self.chat_api.supports_chat_images() and isinstance(message, RoomMessageImage):
+            elif isinstance(message, RoomMessageFile):
-                image_url = message.url
+                try:
-                download = await self.download_file(image_url)
+                    download = await self.download_file(message.url)
+                    if download:
+                        try:
+                            text = download.body.decode("utf-8")
+                        except UnicodeDecodeError:
+                            text = None
 
-                if download:
+                        if text:
-                    encoded_url = f"data:{download.content_type};base64,{base64.b64encode(download.body).decode('utf-8')}"
+                            role = (
-                    parent = chat_messages[-1] if chat_messages and chat_messages[-1]["role"] == ("assistant" if message.sender == self.matrix_client.user_id else "user") else None
+                                "assistant"
+                                if message.sender == self.matrix_client.user_id
+                                else "user"
+                            )
+                            if message == event or (not message.event_id == event.event_id):
+                                message_body = text if not self.chat_api.supports_chat_images() else [{"type": "text", "text": text}]
+                                chat_messages.append({"role": role, "content": message_body})
 
-                    if not parent:
+                except Exception as e:
-                        chat_messages.append({"role": ("assistant" if message.sender == self.matrix_client.user_id else "user"), "content": []})
+                    self.logger.log(f"Error generating text from file: {e}", "error")
-                        parent = chat_messages[-1]
+                    message_body = message.body if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message.body}]
+                    chat_messages.append({"role": "system", "content": message_body})
 
-                    parent["content"].append({
+            elif self.chat_api.supports_chat_images() and isinstance(message, RoomMessageImage):
-                        "type": "image_url",
+                try:
-                        "image_url": {
+                    image_url = message.url
-                            "url": encoded_url
+                    download = await self.download_file(image_url)
-                        }
+
-                    })
+                    if download:
+                        encoded_url = f"data:{download.content_type};base64,{base64.b64encode(download.body).decode('utf-8')}"
+                        parent = chat_messages[-1] if chat_messages and chat_messages[-1]["role"] == ("assistant" if message.sender == self.matrix_client.user_id else "user") else None
+
+                        if not parent:
+                            chat_messages.append({"role": ("assistant" if message.sender == self.matrix_client.user_id else "user"), "content": []})
+                            parent = chat_messages[-1]
+
+                        parent["content"].append({
+                            "type": "image_url",
+                            "image_url": {
+                                "url": encoded_url
+                            }
+                        })
+
+                except Exception as e:
+                    self.logger.log(f"Error generating image from file: {e}", "error")
+                    message_body = message.body if not self.chat_api.supports_chat_images() else [{"type": "text", "text": message.body}]
+                    chat_messages.append({"role": "system", "content": message_body})
 
         # Truncate messages to fit within the token limit
         truncated_messages = self._truncate(
@@ -1136,7 +1166,7 @@ class GPTBot:
 
         await self.matrix_client.room_typing(room.room_id, False)
 
-    def download_file(self, mxc) -> Optional[bytes]:
+    async def download_file(self, mxc) -> Optional[bytes]:
         """Download a file from the homeserver.
 
         Args:
@@ -1146,7 +1176,7 @@ class GPTBot:
             Optional[bytes]: The downloaded file, or None if there was an error.
         """
 
-        download = self.matrix_client.download(mxc)
+        download = await self.matrix_client.download(mxc)
 
         if isinstance(download, DownloadError):
             self.logger.log(f"Error downloading file: {download.message}", "error")