chore(release): bump version to 0.3.21

Incremented the project version from 0.3.20 to 0.3.21 to signify the release of new updates or bug fixes. This minor version bump indicates backward-compatible changes or improvements.

.forgejo/workflows/docker-tag.yml

@@ -0,0 +1,33 @@
+name: Docker CI/CD
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  docker:
+    name: Docker Build and Push to Docker Hub
+    container:
+      image: node:20-bookworm
+    steps:
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y docker.io
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push to Docker Hub
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: |
+            kumitterer/matrix-gptbot:latest
+            kumitterer/matrix-gptbot:${{ env.GITHUB_REF_NAME }}

.forgejo/workflows/release.yml

@@ -0,0 +1,54 @@
+name: Python Package CI/CD
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  setup:
+    name: Setup and Test
+    container:
+      image: node:20-bookworm
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y python3 python3-venv
+
+      - name: Set up Python environment
+        run: |
+          python3 -V
+          python3 -m venv venv
+          . ./venv/bin/activate
+          pip install -U pip
+          pip install .[all]
+
+  publish:
+    name: Publish to PyPI
+    container:
+      image: node:20-bookworm
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y python3 python3-venv
+
+      - name: Publish to PyPI
+        run: |
+          python3 -m venv venv
+          . ./venv/bin/activate
+          pip install -U hatchling twine build
+          python -m build .
+          python -m twine upload --username __token__ --password ${PYPI_TOKEN} dist/*
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"

.gitignore

@@ -5,4 +5,7 @@ config.ini
 venv/
 *.pyc
 __pycache__/
-*.bak
+*.bak
+dist/
+pantalaimon.conf
+.ruff_cache/

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Module",
+            "type": "python",
+            "request": "launch",
+            "module": "gptbot",
+            "justMyCode": true
+        }
+    ]
+}

CHANGELOG.md

@@ -0,0 +1,79 @@
+# Changelog
+
+### 0.3.14 (2024-05-21)
+
+- Fixed issue in handling of login credentials, added error handling for login failures
+
+### 0.3.13 (2024-05-20)
+
+- **Breaking Change**: The `ForceTools` configuration option behavior has changed. Instead of using a separate model for tools, the bot will now try to use the default chat model for tool requests, even if that model is not known to support tools.
+- Added `ToolModel` to OpenAI configuration to allow specifying a separate model for tool requests
+- Automatically resize context images to a default maximum of 2000x768 pixels before sending them to the AI model
+
+### 0.3.12 (2024-05-17)
+
+- Added `ForceVision` to OpenAI configuration to allow third-party models to be used for image recognition
+- Added some missing properties to `OpenAI` class
+
+### 0.3.11 (2024-05-17)
+
+- Refactoring of AI provider handling in preparation for multiple AI providers: Introduced a `BaseAI` class that all AI providers must inherit from
+- Added support for temperature, top_p, frequency_penalty, and presence_penalty in `AllowedUsers`
+- Introduced ruff as a development dependency for linting and applied some linting fixes
+- Fixed `gptbot` command line tool
+- Changed default chat model to `gpt-4o`
+- Changed default image generation model to `dall-e-3`
+- Removed currently unused sections from `config.dist.ini`
+- Changed provided Pantalaimon config file to not use a key ring by default
+- Prevent bot from crashing when an unneeded dependency is missing
+
+### 0.3.10 (2024-05-16)
+
+- Add support for specifying room IDs in `AllowedUsers`
+- Minor fixes
+
+### 0.3.9 (2024-04-23)
+
+- Add Docker support for running the bot in a container
+- Add TrackingMore dependency to pyproject.toml
+- Replace deprecated `pkg_resources` with `importlib.metadata`
+- Allow password-based login on first login
+
+### 0.3.7 / 0.3.8 (2024-04-15)
+
+- Changes to URLs in pyproject.toml
+- Migrated build pipeline to Forgejo Actions
+
+### 0.3.6 (2024-04-11)
+
+- Fix issue where message type detection would fail for some messages (cece8cfb24e6f2e98d80d233b688c3e2c0ff05ae)
+
+### 0.3.5
+
+- Only set room avatar if it is not already set (a9c23ee9c42d0a741a7eb485315e3e2d0a526725)
+
+### 0.3.4 (2024-02-18)
+
+- Optimize chat model and message handling (10b74187eb43bca516e2a469b69be1dbc9496408)
+- Fix parameter passing in chat response calls (2d564afd979e7bc9eee8204450254c9f86b663b5)
+- Refine message filtering in bot event processing (c47f947f80f79a443bbd622833662e3122b121ef)
+
+### 0.3.3 (2024-01-26)
+
+- Implement recursion check in response generation (e6bc23e564e51aa149432fc67ce381a9260ee5f5)
+- Implement tool emulation for models without tool support (0acc1456f9e4efa09e799f6ce2ec9a31f439fe4a)
+- Allow selection of chat model by room (87173ae284957f66594e66166508e4e3bd60c26b)
+
+### 0.3.2 (2023-12-14)
+
+- Removed key upload from room event handler
+- Fixed output of `python -m gptbot -v` to display currently installed version
+- Workaround for bug preventing bot from responding when files are uploaded to an encrypted room
+
+#### Known Issues
+
+- When using Pantalaimon: Bot is unable to download/use files uploaded to unencrypted rooms
+
+### 0.3.1 (2023-12-07)
+
+- Fixed issue in newroom task causing it to be called over and over again

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+COPY src/ /app/src
+COPY pyproject.toml /app
+COPY README.md /app
+COPY LICENSE /app
+
+RUN apt update
+RUN apt install -y build-essential libpython3-dev ffmpeg
+RUN pip install .[all]
+RUN pip install 'future==1.0.0'
+
+CMD ["python", "-m", "gptbot"]

LICENSE

@@ -1,4 +1,5 @@
-Copyright (c) 2023 Kumi Mitterer <gptbot@kumi.email>
+Copyright (c) 2023-2024 Kumi Mitterer <gptbot@kumi.email>, Private.coffee Team
+<support@private.coffee>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,91 +1,151 @@
 # GPTbot
 
-GPTbot is a simple bot that uses different APIs to generate responses to 
-messages in a Matrix room.
+[![Support Private.coffee!](https://shields.private.coffee/badge/private.coffee-support%20us!-pink?logo=coffeescript)](https://private.coffee)
+[![Matrix](https://shields.private.coffee/badge/Matrix-join%20us!-blue?logo=matrix)](https://matrix.to/#/#matrix-gptbot:private.coffee)
+[![PyPI](https://shields.private.coffee/pypi/v/matrix-gptbot)](https://pypi.org/project/matrix-gptbot/)
+[![PyPI - Python Version](https://shields.private.coffee/pypi/pyversions/matrix-gptbot)](https://pypi.org/project/matrix-gptbot/)
+[![PyPI - License](https://shields.private.coffee/pypi/l/matrix-gptbot)](https://pypi.org/project/matrix-gptbot/)
+[![Latest Git Commit](https://shields.private.coffee/gitea/last-commit/privatecoffee/matrix-gptbot?gitea_url=https://git.private.coffee)](https://git.private.coffee/privatecoffee/matrix-gptbot)
 
-It is called GPTbot because it was originally intended to only use GPT-3 to
-generate responses. However, it supports other services/APIs, and I will 
-probably add more in the future, so the name is a bit misleading.
+GPTbot is a simple bot that uses different APIs to generate responses to
+messages in a Matrix room.
 
 ## Features
 
-- AI-generated responses to messages in a Matrix room (chatbot)
-  - Currently supports OpenAI (tested with `gpt-3.5-turbo` and `gpt-4`)
-- AI-generated pictures via the `!gptbot imagine` command
-  - Currently supports OpenAI (DALL-E)
+- AI-generated responses to text, image and voice messages in a Matrix room
+  (chatbot)
+  - Currently supports OpenAI (`gpt-3.5-turbo` and `gpt-4`, `gpt-4o`, `whisper`
+    and `tts`) and compatible APIs (e.g. `ollama`)
+  - Able to generate pictures using OpenAI `dall-e-2`/`dall-e-3` models
+  - Able to browse the web to find information
+  - Able to use OpenWeatherMap to get weather information (requires separate
+    API key)
+  - Even able to roll dice!
 - Mathematical calculations via the `!gptbot calculate` command
-  - Currently supports WolframAlpha
-- Automatic classification of messages (for `imagine`, `calculate`, etc.)
-  - Beta feature, see Usage section for details
+  - Currently supports WolframAlpha (requires separate API key)
 - Really useful commands like `!gptbot help` and `!gptbot coin`
 - sqlite3 database to store room settings
 
-## Planned features
-
-- End-to-end encryption support (partly implemented, but not yet working)
-
 ## Installation
 
-To run the bot, you will need Python 3.10 or newer. 
+To run the bot, you will need Python 3.10 or newer.
 
-The bot has been tested with Python 3.11 on Arch, but should work with any 
+The bot has been tested with Python 3.12 on Arch, but should work with any
 current version, and should not require any special dependencies or operating
 system features.
 
 ### Production
 
-The easiest way to install the bot is to use pip to install it directly from
-[its Git repository](https://kumig.it/kumitterer/matrix-gptbot/):
+#### PyPI
+
+The recommended way to install the bot is to use pip to install it from PyPI.
 
 ```shell
-# If desired, activate a venv first
+# Recommended: activate a venv first
 
 python -m venv venv
 . venv/bin/activate
 
 # Install the bot
 
-pip install git+https://kumig.it/kumitterer/matrix-gptbot.git
+pip install matrix-gptbot[all]
 ```
 
-This will install the bot from the main branch and all required dependencies.
-A release to PyPI is planned, but not yet available.
+This will install the latest release of the bot and all required dependencies
+for all available features.
 
-### Development
+You can also use `pip install git+https://git.private.coffee/privatecoffee/matrix-gptbot.git`
+to install the latest version from the Git repository.
 
-Clone the repository and install the requirements to a virtual environment. 
+#### Docker
+
+A `docker-compose.yml` file is provided that you can use to run the bot with
+Docker Compose. You will need to create a `config.ini` file as described in the
+`Running` section.
 
 ```shell
 # Clone the repository
+git clone https://git.private.coffee/privatecoffee/matrix-gptbot.git
+cd matrix-gptbot
 
-git clone https://kumig.it/kumitterer/matrix-gptbot.git
+# Create a config file
+cp config.dist.ini config.ini
+# Edit the config file to your needs
+
+# Initialize the database file
+sqlite3 database.db "SELECT 1"
+
+# Optionally, create Pantalaimon config
+cp contrib/pantalaimon.example.conf pantalaimon.conf
+# Edit the Pantalaimon config file to your needs
+# Update your homeserver URL in the bot's config.ini to point to Pantalaimon (probably http://pantalaimon:8009 if you used the provided example config)
+# You can use `fetch_access_token.py` to get an access token for the bot
+
+# Start the bot
+docker-compose up -d
+```
+
+#### End-to-end encryption
+
+WARNING: Using end-to-end encryption seems to sometimes cause problems with
+file attachments, especially in rooms that are not encrypted, if the same
+user also uses the bot in encrypted rooms.
+
+The bot itself does not implement end-to-end encryption. However, it can be
+used in conjunction with [pantalaimon](https://github.com/matrix-org/pantalaimon).
+
+You first have to log in to your homeserver using `python fetch_access_token.py`,
+and can then use the returned access token in your bot's `config.ini` file.
+
+Make sure to also point the bot to your pantalaimon instance by setting
+`homeserver` to your pantalaimon instance instead of directly to your
+homeserver in your `config.ini`.
+
+Note: If you don't use pantalaimon, the bot will still work, but it will not
+be able to decrypt or encrypt messages. This means that you cannot use it in
+rooms with end-to-end encryption enabled.
+
+### Development
+
+Clone the repository and install the requirements to a virtual environment.
+
+```shell
+# Clone the repository
+git clone https://git.private.coffee/privatecoffee/matrix-gptbot.git
 cd matrix-gptbot
 
 # If desired, activate a venv first
-
 python -m venv venv
 . venv/bin/activate
 
 # Install the bot in editable mode
-
 pip install -e .[dev]
 
 # Go to the bot directory and start working
-
 cd src/gptbot
 ```
 
 Of course, you can also fork the repository on [GitHub](https://github.com/kumitterer/matrix-gptbot/)
 and work on your own copy.
 
-### Configuration
+#### Repository policy
 
-The bot requires a configuration file to be present in the working directory.
-Copy the provided `config.dist.ini` to `config.ini` and edit it to your needs.
+Generally, the `main` branch is considered unstable and should not be used in
+production. Instead, use the latest release tag. The `main` branch is used for
+development and may contain breaking changes at any time.
+
+For development, a feature branch should be created from `main` and merged back
+into `main` with a pull request. The pull request will be reviewed and tested
+before merging.
 
 ## Running
 
-The bot can be run with `python -m gptbot`. If required, activate a venv first.
+The bot requires a configuration file to be present in the working directory.
+
+Copy the provided `config.dist.ini` to `config.ini` and edit it to your needs.
+
+The bot can then be run with `python -m gptbot`. If required, activate a venv
+first.
 
 You may want to run the bot in a screen or tmux session, or use a process
 manager like systemd. The repository contains a sample systemd service file
@@ -95,6 +155,9 @@ adjust the paths in the file to match your setup, then copy it to
 `systemctl start gptbot` and enable it to start automatically on boot with
 `systemctl enable gptbot`.
 
+Analogously, you can use the provided `gptbot-pantalaimon.service` file to run
+pantalaimon as a systemd service.
+
 ## Usage
 
 Once it is running, just invite the bot to a room and it will start responding
@@ -115,35 +178,42 @@ With this setting, the bot will only be triggered if a message begins with
 bot to generate a response to the message `Hello, how are you?`. The bot will
 still get previous messages in the room as context for generating the response.
 
+### Tools
+
+The bot has a selection of tools at its disposal that it will automatically use
+to generate responses. For example, if you send a message like "Draw me a
+picture of a cat", the bot will automatically use DALL-E to generate an image
+of a cat.
+
+Note that this only works if the bot is configured to use a model that supports
+tools. This currently is only the case for OpenAI's `gpt-3.5-turbo` model. If
+you wish to use `gpt-4` instead, you can set the `ForceTools` option in the
+`[OpenAI]` section of the config file to `1`. This will cause the bot to use
+`gpt-3.5-turbo` for tool generation and `gpt-4` for generating the final text
+response.
+
+Similarly, it will attempt to use the `gpt-4-vision-preview` model to "read"
+the contents of images if a non-vision model is used.
+
 ### Commands
 
-There are a few commands that you can use to interact with the bot. For example,
-if you want to generate an image from a text prompt, you can use the
-`!gptbot imagine` command. For example, `!gptbot imagine a cat` will cause the
-bot to generate an image of a cat.
+There are a few commands that you can use to explicitly call a certain feature
+of the bot. For example, if you want to generate an image from a text prompt,
+you can use the `!gptbot imagine` command. For example, `!gptbot imagine a cat`
+will cause the bot to generate an image of a cat.
 
 To learn more about the available commands, `!gptbot help` will print a list of
 available commands.
 
-### Automatic classification
+### Voice input and output
 
-As a beta feature, the bot can automatically classify messages and use the
-appropriate API to generate a response. For example, if you send a message
-like "Draw me a picture of a cat", the bot will automatically use the
-`imagine` command to generate an image of a cat.
+The bot supports voice input and output, but it is disabled by default. To
+enable it, use the `!gptbot roomsettings` command to change the settings for
+the current room. `!gptbot roomsettings stt true` will enable voice input using
+OpenAI's `whisper` model, and `!gptbot roomsettings tts true` will enable voice
+output using the `tts` model.
 
-This feature is disabled by default. To enable it, use the `!gptbot roomsettings`
-command to change the settings for the current room. `!gptbot roomsettings classification true`
-will enable automatic classification, and `!gptbot roomsettings classification false`
-will disable it again.
-
-Note that this feature is still in beta and may not work as expected. You can
-always use the commands manually if the automatic classification doesn't work
-for you (including `!gptbot chat` for a regular chat message).
-
-Also note that this feature conflicts with the `always_reply false` setting -
-or rather, it doesn't make sense then because you already have to explicitly
-specify the command to use.
+Note that this currently only works for audio messages and .mp3 file uploads.
 
 ## Troubleshooting
 
@@ -152,10 +222,12 @@ specify the command to use.
 First of all, make sure that the bot is actually running. (Okay, that's not
 really troubleshooting, but it's a good start.)
 
-If the bot is running, check the logs. The first few lines should contain
-"Starting bot...", "Syncing..." and "Bot started". If you don't see these
-lines, something went wrong during startup. Fortunately, the logs should
-contain more information about what went wrong.
+If the bot is running, check the logs, these should tell you what is going on.
+For example, if the bot is showing an error message like "Timed out, retrying",
+it is unable to reach your homeserver. In this case, check your homeserver URL
+and make sure that the bot can reach it. If you are using Pantalaimon, make
+sure that the bot is pointed to Pantalaimon and not directly to your 
+homeserver, and that Pantalaimon is running and reachable.
 
 If you need help figuring out what went wrong, feel free to open an issue.
 
@@ -181,4 +253,5 @@ please check the logs and open an issue if you can't figure out what's going on.
 
 ## License
 
-This project is licensed under the terms of the MIT license. See the [LICENSE](LICENSE) file for details.
+This project is licensed under the terms of the MIT license. See the [LICENSE](LICENSE)
+file for details.

config.dist.ini

@@ -45,10 +45,11 @@ Operator = Contact details not set
 # DisplayName = GPTBot
 
 # A list of allowed users
-# If not defined, everyone is allowed to use the bot
+# If not defined, everyone is allowed to use the bot (so you should really define this)
 # Use the "*:homeserver.matrix" syntax to allow everyone on a given homeserver
+# Alternatively, you can also specify a room ID to allow everyone in the room to use the bot within that room
 #
-# AllowedUsers = ["*:matrix.local"]
+# AllowedUsers = ["*:matrix.local", "!roomid:matrix.local"]
 
 # Minimum level of log messages that should be printed
 # Available log levels in ascending order: trace, debug, info, warning, error, critical
@@ -62,16 +63,20 @@ LogLevel = info
 
 # The Chat Completion model you want to use.
 #
-# Unless you are in the GPT-4 beta (if you don't know - you aren't),
-# leave this as the default value (gpt-3.5-turbo)
+# Model = gpt-4o
+
+# The Image Generation model you want to use.
 #
-# Model = gpt-3.5-turbo
+# ImageModel = dall-e-3
 
 # Your OpenAI API key
 #
 # Find this in your OpenAI account:
 # https://platform.openai.com/account/api-keys 
 #
+# This may not be required for self-hosted models – in that case, just leave it
+# as it is.
+#
 APIKey = sk-yoursecretkey
 
 # The maximum amount of input sent to the API
@@ -96,9 +101,78 @@ APIKey = sk-yoursecretkey
 # The base URL of the OpenAI API
 #
 # Setting this allows you to use a self-hosted AI model for chat completions
-# using something like https://github.com/abetlen/llama-cpp-python
+# using something like llama-cpp-python or ollama
 #
-# BaseURL = https://openai.local/v1
+# BaseURL = https://api.openai.com/v1/
+
+# Whether to force the use of tools in the chat completion model
+#
+# This will make the bot allow the use of tools in the chat completion model,
+# even if the model you are using isn't known to support tools. This is useful
+# if you are using a self-hosted model that supports tools, but the bot doesn't
+# know about it.
+#
+# ForceTools = 1
+
+# Whether a dedicated model should be used for tools
+#
+# This will make the bot use a dedicated model for tools. This is useful if you
+# want to use a model that doesn't support tools, but still want to be able to
+# use tools.
+#
+# ToolModel = gpt-4o
+
+# Whether to emulate tools in the chat completion model
+#
+# This will make the bot use the default model to *emulate* tools. This is
+# useful if you want to use a model that doesn't support tools, but still want
+# to be able to use tools. However, this may cause all kinds of weird results.
+#
+# EmulateTools = 0
+
+# Force vision in the chat completion model
+#
+# By default, the bot only supports image recognition in known vision models.
+# If you set this to 1, the bot will assume that the model you're using supports
+# vision, and will send images to the model as well. This may be required for
+# some self-hosted models.
+#
+# ForceVision = 0
+
+# Maximum width and height of images sent to the API if vision is enabled
+#
+# The OpenAI API has a limit of 2000 pixels for the long side of an image, and 
+# 768 pixels for the short side. You may have to adjust these values if you're
+# using a self-hosted model that has different limits. You can also set these
+# to 0 to disable image resizing.
+#
+# MaxImageLongSide = 2000
+# MaxImageShortSide = 768
+
+# Whether the used model supports video files as input
+#
+# If you are using a model that supports video files as input, set this to 1.
+# This will make the bot send video files to the model as well as images.
+# This may be possible with some self-hosted models, but is not supported by
+# the OpenAI API at this time.
+#
+# ForceVideoInput = 0
+
+# Advanced settings for the OpenAI API
+#
+# These settings are not required for normal operation, but can be used to
+# tweak the behavior of the bot.
+#
+# Note: These settings are not validated by the bot, so make sure they are
+# correct before setting them, or the bot may not work as expected.
+#
+# For more information, see the OpenAI documentation:
+# https://platform.openai.com/docs/api-reference/chat/create
+#
+# Temperature = 1
+# TopP = 1
+# FrequencyPenalty = 0
+# PresencePenalty = 0
 
 ###############################################################################
 
@@ -117,20 +191,29 @@ APIKey = sk-yoursecretkey
 
 # The URL to your Matrix homeserver
 #
+# If you are using Pantalaimon, this should be the URL of your Pantalaimon
+# instance, not the Matrix homeserver itself.
+#
 Homeserver = https://matrix.local
 
 # An Access Token for the user your bot runs as
-# Can be obtained using a request like this:
 #
 # See https://www.matrix.org/docs/guides/client-server-api#login
 # for information on how to obtain this value
 #
 AccessToken = syt_yoursynapsetoken
 
-# The Matrix user ID of the bot (@local:domain.tld)
-# Only specify this if the bot fails to figure it out by itself
+# Instead of an Access Token, you can also use a User ID and password
+# to log in. Upon first run, the bot will automatically turn this into
+# an Access Token and store it in the config file, and remove the
+# password from the config file.
+#
+# This is particularly useful if you are using Pantalaimon, where this
+# is the only (easy) way to generate an Access Token.
 #
 # UserID = @gptbot:matrix.local
+# Password = yourpassword
+
 
 ###############################################################################
 
@@ -141,11 +224,6 @@ AccessToken = syt_yoursynapsetoken
 #
 Path = database.db
 
-# Path of the Crypto Store - required to support encrypted rooms
-# (not tested/supported yet)
-#
-CryptoStore = store.db
-
 ###############################################################################
 
 [TrackingMore]
@@ -157,21 +235,10 @@ CryptoStore = store.db
 
 ###############################################################################
 
-[Replicate]
+[OpenWeatherMap]
 
-# API key for replicate.com
-# Can be used to run lots of different AI models
-# If not defined, the features that depend on it are not available
-#
-# APIKey = r8_alotoflettersandnumbershere
-
-###############################################################################
-
-[HuggingFace]
-
-# API key for Hugging Face
-# Can be used to run lots of different AI models
-# If not defined, the features that depend on it are not available
+# API key for OpenWeatherMap
+# If not defined, the bot will be unable to provide weather information
 #
 # APIKey = __________________________
 

contrib/pantalaimon.example.conf

@@ -0,0 +1,7 @@
+[Homeserver]
+Homeserver = https://example.com
+ListenAddress = localhost
+ListenPort = 8009
+IgnoreVerification = True
+LogLevel = debug
+UseKeyring = no

docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  gptbot:
+    image: kumitterer/matrix-gptbot
+    volumes:
+      - ./config.ini:/app/config.ini
+      - ./database.db:/app/database.db
+
+  pantalaimon:
+    image: matrixdotorg/pantalaimon
+    volumes:
+      - ./pantalaimon.conf:/etc/pantalaimon/pantalaimon.conf
+    ports:
+      - "8009:8009"

fetch_access_token.py

@@ -0,0 +1,22 @@
+from nio import AsyncClient
+
+from configparser import ConfigParser
+
+async def main():
+    config = ConfigParser()
+    config.read("config.ini")
+
+    user_id = input("User ID: ")
+    password = input("Password: ")
+
+    client = AsyncClient(config["Matrix"]["Homeserver"])
+    client.user = user_id
+    await client.login(password)
+
+    print("Access token: " + client.access_token)
+
+    await client.close()
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.get_event_loop().run_until_complete(main())

gptbot-pantalaimon.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Pantalaimon for GPTbot
+Requires=network.target
+
+[Service]
+Type=simple
+User=gptbot
+Group=gptbot
+WorkingDirectory=/opt/gptbot
+ExecStart=/opt/gptbot/venv/bin/python3 -um pantalaimon.main -c pantalaimon.conf
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

pyproject.toml

@@ -7,63 +7,59 @@ allow-direct-references = true
 
 [project]
 name = "matrix-gptbot"
-version = "0.1.0"
+version = "0.3.21"
 
 authors = [
-  { name="Kumi Mitterer", email="gptbot@kumi.email" },
+  { name = "Kumi", email = "gptbot@kumi.email" },
+  { name = "Private.coffee Team", email = "support@private.coffee" },
 ]
 
 description = "Multifunctional Chatbot for Matrix"
 readme = "README.md"
-license = { file="LICENSE" }
+license = { file = "LICENSE" }
 requires-python = ">=3.10"
 
-packages = [
-    "src/gptbot"
-]
+packages = ["src/gptbot"]
 
 classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
 ]
 
 dependencies = [
-    "matrix-nio[e2e]",
-    "markdown2[all]",
-    "tiktoken",
-    "python-magic",
-    "pillow",
-    ]
+  "matrix-nio[e2e]>=0.24.0",
+  "markdown2[all]",
+  "tiktoken",
+  "python-magic",
+  "pillow",
+  "future>=1.0.0",
+]
 
 [project.optional-dependencies]
-openai = [
-    "openai",
-]
+openai = ["openai>=1.2", "pydub"]
 
-wolframalpha = [
-    "wolframalpha",
-]
+google = ["google-generativeai"]
 
-trackingmore = [
-    "trackingmore @ git+https://kumig.it/kumitterer/trackingmore-api-tool.git",
-]
+wolframalpha = ["wolframalpha"]
+
+trackingmore = ["trackingmore-api-tool"]
 
 all = [
-  "matrix-gptbot[openai,wolframalpha,trackingmore]",
+  "matrix-gptbot[openai,wolframalpha,trackingmore,google]",
+  "geopy",
+  "beautifulsoup4",
 ]
 
-dev = [
-  "matrix-gptbot[all]",
-  "black",
-]
+dev = ["matrix-gptbot[all]", "black", "hatchling", "twine", "build", "ruff"]
 
 [project.urls]
-"Homepage" = "https://kumig.it/kumitterer/matrix-gptbot"
-"Bug Tracker" = "https://kumig.it/kumitterer/matrix-gptbot/issues"
+"Homepage" = "https://git.private.coffee/privatecoffee/matrix-gptbot"
+"Bug Tracker" = "https://git.private.coffee/privatecoffee/matrix-gptbot/issues"
+"Source Code" = "https://git.private.coffee/privatecoffee/matrix-gptbot"
 
 [project.scripts]
-gptbot = "gptbot:main"
+gptbot = "gptbot.__main__:main_sync"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/gptbot"]
+packages = ["src/gptbot"]

requirements.txt

@@ -1,10 +0,0 @@
-openai
-matrix-nio[e2e]
-markdown2[all]
-tiktoken
-duckdb
-python-magic
-pillow
-wolframalpha
-
-git+https://kumig.it/kumitterer/trackingmore-api-tool.git

src/gptbot/__main__.py

@@ -5,13 +5,22 @@ from configparser import ConfigParser
 
 import signal
 import asyncio
+import importlib.metadata
 
 
 def sigterm_handler(_signo, _stack_frame):
     exit()
 
 
-if __name__ == "__main__":
+def get_version():
+    try:
+        package_version = importlib.metadata.version("matrix_gptbot")
+    except Exception:
+        return None
+    return package_version
+
+
+async def main():
     # Parse command line arguments
     parser = ArgumentParser()
     parser.add_argument(
@@ -25,7 +34,7 @@ if __name__ == "__main__":
         "-v",
         help="Print version and exit",
         action="version",
-        version="GPTBot v0.1.0",
+        version=f"GPTBot {get_version() or '- version unknown'}",
     )
     args = parser.parse_args()
 
@@ -34,15 +43,28 @@ if __name__ == "__main__":
     config.read(args.config)
 
     # Create bot
-    bot = GPTBot.from_config(config)
+    bot, new_config = await GPTBot.from_config(config)
+
+    # Update config with new values
+    if new_config:
+        with open(args.config, "w") as configfile:
+            new_config.write(configfile)
 
     # Listen for SIGTERM
     signal.signal(signal.SIGTERM, sigterm_handler)
 
     # Start bot
     try:
-        asyncio.run(bot.run())
+        await bot.run()
     except KeyboardInterrupt:
         print("Received KeyboardInterrupt - exiting...")
     except SystemExit:
         print("Received SIGTERM - exiting...")
+
+
+def main_sync():
+    asyncio.run(main())
+
+
+if __name__ == "__main__":
+    main_sync()

src/gptbot/callbacks/__init__.py

@@ -1,35 +1,24 @@
 from nio import (
     RoomMessageText,
-    MegolmEvent,
     InviteEvent,
-    Event,
     SyncResponse,
     JoinResponse,
-    InviteEvent,
-    OlmEvent,
-    MegolmEvent,
     RoomMemberEvent,
-    Response,
 )
 
-from .test import test_callback
 from .sync import sync_callback
 from .invite import room_invite_callback
 from .join import join_callback
 from .message import message_callback
 from .roommember import roommember_callback
-from .test_response import test_response_callback
 
 RESPONSE_CALLBACKS = {
-    Response: test_response_callback,
     SyncResponse: sync_callback,
     JoinResponse: join_callback,
 }
 
 EVENT_CALLBACKS = {
-    Event: test_callback,
     InviteEvent: room_invite_callback,
     RoomMessageText: message_callback,
-    MegolmEvent: message_callback,
     RoomMemberEvent: roommember_callback,
-}
+}

src/gptbot/callbacks/invite.py

@@ -2,9 +2,9 @@ from nio import InviteEvent, MatrixRoom
 
 async def room_invite_callback(room: MatrixRoom, event: InviteEvent, bot):
     if room.room_id in bot.matrix_client.rooms:
-        logging(f"Already in room {room.room_id} - ignoring invite")
+        bot.logger.log(f"Already in room {room.room_id} - ignoring invite")
         return
 
     bot.logger.log(f"Received invite to room {room.room_id} - joining...")
 
-    response = await bot.matrix_client.join(room.room_id)
+    await bot.matrix_client.join(room.room_id)

src/gptbot/callbacks/join.py

@@ -8,11 +8,13 @@ async def join_callback(response, bot):
 
     with closing(bot.database.cursor()) as cursor:
         cursor.execute(
-            "SELECT space_id FROM user_spaces WHERE user_id = ? AND active = TRUE", (event.sender,))
+            "SELECT space_id FROM user_spaces WHERE user_id = ? AND active = TRUE", (response.sender,))
         space = cursor.fetchone()
 
     if space:
         bot.logger.log(f"Adding new room to space {space[0]}...")
-        await bot.add_rooms_to_space(space[0], [new_room.room_id])
+        await bot.add_rooms_to_space(space[0], [response.room_id])
+
+    bot.matrix_client.keys_upload()
 
     await bot.send_message(bot.matrix_client.rooms[response.room_id], "Hello! Thanks for inviting me! How can I help you today?")

src/gptbot/callbacks/message.py

@@ -1,39 +1,18 @@
-from nio import MatrixRoom, RoomMessageText, MegolmEvent, RoomKeyRequestError, RoomKeyRequestResponse
+from nio import MatrixRoom, RoomMessageText
 
 from datetime import datetime
 
-async def message_callback(room: MatrixRoom | str, event: RoomMessageText | MegolmEvent, bot):
+async def message_callback(room: MatrixRoom | str, event: RoomMessageText, bot):
     bot.logger.log(f"Received message from {event.sender} in room {room.room_id}")
 
     sent = datetime.fromtimestamp(event.server_timestamp / 1000)
     received = datetime.now()
     latency = received - sent
 
-    if isinstance(event, MegolmEvent):
-        try:
-            event = await bot.matrix_client.decrypt_event(event)
-        except Exception as e:
-            try:
-                bot.logger.log("Requesting new encryption keys...")
-                response = await bot.matrix_client.request_room_key(event)
-
-                if isinstance(response, RoomKeyRequestError):
-                    bot.logger.log(f"Error requesting encryption keys: {response}", "error")
-                elif isinstance(response, RoomKeyRequestResponse):
-                    bot.logger.log(f"Encryption keys received: {response}", "debug")
-                    bot.matrix_bot.olm.handle_response(response)
-                    event = await bot.matrix_client.decrypt_event(event)
-            except:
-                pass
-
-            bot.logger.log(f"Error decrypting message: {e}", "error")
-            await bot.send_message(room, "Sorry, I couldn't decrypt that message. Please try again later or switch to a room without encryption.", True)
-            return
-
     if event.sender == bot.matrix_client.user_id:
         bot.logger.log("Message is from bot itself - ignoring")
 
-    elif event.body.startswith("!gptbot"):
+    elif event.body.startswith("!gptbot") or event.body.startswith("* !gptbot"):
         await bot.process_command(room, event)
 
     elif event.body.startswith("!"):

src/gptbot/callbacks/test.py

@@ -1,11 +0,0 @@
-from nio import MatrixRoom, Event
-
-async def test_callback(room: MatrixRoom, event: Event, bot):
-    """Test callback for debugging purposes.
-
-    Args:
-        room (MatrixRoom): The room the event was sent in.
-        event (Event): The event that was sent.
-    """
-
-    bot.logger.log(f"Test callback called: {room.room_id} {event.event_id} {event.sender} {event.__class__}")

src/gptbot/callbacks/test_response.py

@@ -1,11 +0,0 @@
-from nio import ErrorResponse
-
-
-async def test_response_callback(response, bot):
-    if isinstance(response, ErrorResponse):
-        bot.logger.log(
-            f"Error response received ({response.__class__.__name__}): {response.message}",
-            "warning",
-        )
-    else:
-        bot.logger.log(f"{response.__class__} response received", "debug")

src/gptbot/classes/ai/base.py

@@ -0,0 +1,76 @@
+from ...classes.logging import Logger
+
+import asyncio
+from functools import partial
+from typing import Any, AsyncGenerator, Dict, Optional, Mapping
+
+from nio import Event
+
+
+class AttributeDictionary(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttributeDictionary, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+class BaseAI:
+    bot: Any
+    logger: Logger
+
+    def __init__(self, bot, config: Mapping, logger: Optional[Logger] = None):
+        self.bot = bot
+        self.logger = logger or bot.logger or Logger()
+        self._config = config
+
+    @property
+    def chat_api(self) -> str:
+        return self.chat_model
+
+    async def prepare_messages(
+        self, event: Event, messages: list[Any], system_message: Optional[str] = None
+    ) -> list[Any]:
+        """A helper method to prepare messages for the AI.
+
+        This converts a list of Matrix messages into whatever format the AI requires.
+
+        Args:
+            event (Event): The event that triggered the message generation. Generally a text message from a user.
+            messages (list[Dict[str, str]]): The messages to prepare. Generally of type RoomMessage*.
+            system_message (Optional[str], optional): A system message to include. Defaults to None.
+
+        Returns:
+            list[Any]: The prepared messages in the format the AI requires.
+
+        Raises:
+            NotImplementedError: If the method is not implemented in the subclass.
+        """
+
+        raise NotImplementedError(
+            "Implementations of BaseAI must implement prepare_messages."
+        )
+
+    async def _request_with_retries(
+        self, request: partial, attempts: int = 5, retry_interval: int = 2
+    ) -> AsyncGenerator[Any | list | Dict, None]:
+        """Retry a request a set number of times if it fails.
+
+        Args:
+            request (partial): The request to make with retries.
+            attempts (int, optional): The number of attempts to make. Defaults to 5.
+            retry_interval (int, optional): The interval in seconds between attempts. Defaults to 2 seconds.
+
+        Returns:
+            AsyncGenerator[Any | list | Dict, None]: The response for the request.
+        """
+        current_attempt = 1
+        while current_attempt <= attempts:
+            try:
+                response = await request()
+                return response
+            except Exception as e:
+                self.logger.log(f"Request failed: {e}", "error")
+                self.logger.log(f"Retrying in {retry_interval} seconds...")
+                await asyncio.sleep(retry_interval)
+                current_attempt += 1
+
+        raise Exception("Request failed after all attempts.")

src/gptbot/classes/ai/google.py

@@ -0,0 +1,73 @@
+from .base import BaseAI
+from ..logging import Logger
+
+from typing import Optional, Mapping, List, Dict, Tuple
+
+import google.generativeai as genai
+
+
+class GeminiAI(BaseAI):
+    api_code: str = "google"
+
+    @property
+    def chat_api(self) -> str:
+        return self.chat_model
+
+    google_api: genai.GenerativeModel
+
+    operator: str = "Google (https://ai.google)"
+
+    def __init__(
+        self,
+        bot,
+        config: Mapping,
+        logger: Optional[Logger] = None,
+    ):
+        super().__init__(bot, config, logger)
+        genai.configure(api_key=self.api_key)
+        self.gemini_api = genai.GenerativeModel(self.chat_model)
+
+    @property
+    def api_key(self):
+        return self._config["APIKey"]
+
+    @property
+    def chat_model(self):
+        return self._config.get("Model", fallback="gemini-pro")
+
+    def prepare_messages(event, messages: List[Dict[str, str]], ) -> List[str]:
+        return [message["content"] for message in messages]
+
+    async def generate_chat_response(
+        self,
+        messages: List[Dict[str, str]],
+        user: Optional[str] = None,
+        room: Optional[str] = None,
+        use_tools: bool = True,
+        model: Optional[str] = None,
+    ) -> Tuple[str, int]:
+        """Generate a response to a chat message.
+
+        Args:
+            messages (List[Dict[str, str]]): A list of messages to use as context.
+            user (Optional[str], optional): The user to use the assistant for. Defaults to None.
+            room (Optional[str], optional): The room to use the assistant for. Defaults to None.
+            use_tools (bool, optional): Whether to use tools. Defaults to True.
+            model (Optional[str], optional): The model to use. Defaults to None, which uses the default chat model.
+
+        Returns:
+            Tuple[str, int]: The response text and the number of tokens used.
+        """
+        self.logger.log(
+            f"Generating response to {len(messages)} messages for user {user} in room {room}..."
+        )
+
+        messages = self.prepare_messages(messages)
+
+        return self.gemini_api.generate_content(
+            messages=messages,
+            user=user,
+            room=room,
+            use_tools=use_tools,
+            model=model,
+        )

src/gptbot/classes/bot.py

@@ -1,7 +1,6 @@
 import markdown2
 import tiktoken
 import asyncio
-import functools
 
 from PIL import Image
 
@@ -15,9 +14,6 @@ from nio import (
     MatrixRoom,
     Api,
     RoomMessagesError,
-    MegolmEvent,
-    GroupEncryptionError,
-    EncryptionError,
     RoomMessageText,
     RoomSendResponse,
     SyncResponse,
@@ -27,11 +23,17 @@ from nio import (
     RoomSendError,
     RoomVisibility,
     RoomCreateError,
+    RoomMessageMedia,
+    DownloadError,
+    RoomGetStateError,
+    DiskDownloadResponse,
+    MemoryDownloadResponse,
+    LoginError,
 )
-from nio.crypto import Olm
 from nio.store import SqliteStore
 
-from typing import Optional, List
+
+from typing import Optional, List, Any, Union
 from configparser import ConfigParser
 from datetime import datetime
 from io import BytesIO
@@ -41,46 +43,132 @@ from contextlib import closing
 import uuid
 import traceback
 import json
-import importlib.util
-import sys
 import sqlite3
 
 from .logging import Logger
 from ..migrations import migrate
 from ..callbacks import RESPONSE_CALLBACKS, EVENT_CALLBACKS
 from ..commands import COMMANDS
-from .openai import OpenAI
-from .wolframalpha import WolframAlpha
-from .trackingmore import TrackingMore
+from ..tools import TOOLS, Handover, StopProcessing
+from .ai.base import BaseAI
+from .exceptions import DownloadException
 
 
 class GPTBot:
     # Default values
     database: Optional[sqlite3.Connection] = None
-    crypto_store_path: Optional[str | Path] = None
-    # Default name of rooms created by the bot
-    display_name = default_room_name = "GPTBot"
-    default_system_message: str = "You are a helpful assistant."
-    # Force default system message to be included even if a custom room message is set
-    force_system_message: bool = False
-    max_tokens: int = 3000  # Maximum number of input tokens
-    max_messages: int = 30  # Maximum number of messages to consider as input
+    database_path: Optional[str | Path] = None
     matrix_client: Optional[AsyncClient] = None
     sync_token: Optional[str] = None
     logger: Optional[Logger] = Logger()
-    chat_api: Optional[OpenAI] = None
-    image_api: Optional[OpenAI] = None
-    classification_api: Optional[OpenAI] = None
-    parcel_api: Optional[TrackingMore] = None
-    operator: Optional[str] = None
+    chat_api: Optional[BaseAI] = None
+    image_api: Optional[BaseAI] = None
+    classification_api: Optional[BaseAI] = None
+    tts_api: Optional[BaseAI] = None
+    stt_api: Optional[BaseAI] = None
+    parcel_api: Optional[Any] = None
+    calculation_api: Optional[Any] = None
     room_ignore_list: List[str] = []  # List of rooms to ignore invites from
-    debug: bool = False
     logo: Optional[Image.Image] = None
     logo_uri: Optional[str] = None
-    allowed_users: List[str] = []
+    config: ConfigParser = ConfigParser()
+
+    # Properties
+
+    @property
+    def allowed_users(self) -> List[str]:
+        """List of users allowed to use the bot.
+
+        Returns:
+            List[str]: List of user IDs. Defaults to [], which means all users are allowed.
+        """
+        try:
+            return json.loads(self.config["GPTBot"]["AllowedUsers"])
+        except Exception:
+            return []
+
+    @property
+    def display_name(self) -> str:
+        """Display name of the bot user.
+
+        Returns:
+            str: The display name of the bot user. Defaults to "GPTBot".
+        """
+        return self.config["GPTBot"].get("DisplayName", "GPTBot")
+
+    @property
+    def default_room_name(self) -> str:
+        """Default name of rooms created by the bot.
+
+        Returns:
+            str: The default name of rooms created by the bot. Defaults to the display name of the bot.
+        """
+        return self.config["GPTBot"].get("DefaultRoomName", self.display_name)
+
+    @property
+    def default_system_message(self) -> str:
+        """Default system message to include in rooms created by the bot.
+
+        Returns:
+            str: The default system message to include in rooms created by the bot. Defaults to "You are a helpful assistant.".
+        """
+        return self.config["GPTBot"].get(
+            "SystemMessage",
+            "You are a helpful assistant.",
+        )
+
+    @property
+    def force_system_message(self) -> bool:
+        """Whether to force the default system message to be included even if a custom room message is set.
+
+        Returns:
+            bool: Whether to force the default system message to be included even if a custom room message is set. Defaults to False.
+        """
+        return self.config["GPTBot"].getboolean("ForceSystemMessage", False)
+
+    @property
+    def operator(self) -> Optional[str]:
+        """Operator of the bot.
+
+        Returns:
+            Optional[str]: The matrix user ID of the operator of the bot. Defaults to None.
+        """
+        return self.config["GPTBot"].get("Operator")
+
+    @property
+    def debug(self) -> bool:
+        """Whether to enable debug logging.
+
+        Returns:
+            bool: Whether to enable debug logging. Defaults to False.
+        """
+        return self.config["GPTBot"].getboolean("Debug", False)
+
+    @property
+    def logo_path(self) -> str:
+        """Path to the logo of the bot.
+
+        Returns:
+            str: The path to the logo of the bot. Defaults to "assets/logo.png" in the bot's directory.
+        """
+        return self.config["GPTBot"].get(
+            "Logo", str(Path(__file__).parent.parent / "assets/logo.png")
+        )
+
+    @property
+    def allow_model_override(self) -> bool:
+        """Whether to allow per-room model overrides.
+
+        Returns:
+            bool: Whether to allow per-room model overrides. Defaults to False.
+        """
+        return self.config["GPTBot"].getboolean("AllowModelOverride", False)
+
+    # User agent to use for HTTP requests
+    USER_AGENT = "matrix-gptbot/dev (+https://kumig.it/kumitterer/matrix-gptbot)"
 
     @classmethod
-    def from_config(cls, config: ConfigParser):
+    async def from_config(cls, config: ConfigParser):
         """Create a new GPTBot instance from a config file.
 
         Args:
@@ -92,83 +180,90 @@ class GPTBot:
 
         # Create a new GPTBot instance
         bot = cls()
+        bot.config = config
 
         # Set the database connection
-        bot.database = (
-            sqlite3.connect(config["Database"]["Path"])
+        bot.database_path = (
+            config["Database"]["Path"]
             if "Database" in config and "Path" in config["Database"]
             else None
         )
-
-        bot.crypto_store_path = (
-            config["Database"]["CryptoStore"]
-            if "Database" in config and "CryptoStore" in config["Database"]
-            else None
-        )
+        bot.database = sqlite3.connect(bot.database_path) if bot.database_path else None
 
         # Override default values
         if "GPTBot" in config:
-            bot.operator = config["GPTBot"].get("Operator", bot.operator)
-            bot.default_room_name = config["GPTBot"].get(
-                "DefaultRoomName", bot.default_room_name
-            )
-            bot.default_system_message = config["GPTBot"].get(
-                "SystemMessage", bot.default_system_message
-            )
-            bot.force_system_message = config["GPTBot"].getboolean(
-                "ForceSystemMessage", bot.force_system_message
-            )
-            bot.debug = config["GPTBot"].getboolean("Debug", bot.debug)
-
             if "LogLevel" in config["GPTBot"]:
                 bot.logger = Logger(config["GPTBot"]["LogLevel"])
 
-            logo_path = config["GPTBot"].get(
-                "Logo", str(Path(__file__).parent.parent / "assets/logo.png")
-            )
+            bot.logger.log(f"Loading logo from {bot.logo_path}", "debug")
 
-            bot.logger.log(f"Loading logo from {logo_path}", "debug")
+            if Path(bot.logo_path).exists() and Path(bot.logo_path).is_file():
+                bot.logo = Image.open(bot.logo_path)
 
-            if Path(logo_path).exists() and Path(logo_path).is_file():
-                bot.logo = Image.open(logo_path)
+        # Set up OpenAI
+        assert (
+            "OpenAI" in config
+        ), "OpenAI config not found"  # TODO: Update this to support other providers
 
-            bot.display_name = config["GPTBot"].get("DisplayName", bot.display_name)
+        from .ai.openai import OpenAI
 
-            if "AllowedUsers" in config["GPTBot"]:
-                bot.allowed_users = json.loads(config["GPTBot"]["AllowedUsers"])
+        openai_api = OpenAI(bot=bot, config=config["OpenAI"])
 
-        bot.chat_api = bot.image_api = bot.classification_api = OpenAI(
-            config["OpenAI"]["APIKey"], config["OpenAI"].get("Model"), bot.logger
-        )
-        bot.max_tokens = config["OpenAI"].getint("MaxTokens", bot.max_tokens)
-        bot.max_messages = config["OpenAI"].getint("MaxMessages", bot.max_messages)
+        if "Model" in config["OpenAI"]:
+            bot.chat_api = openai_api
+            bot.classification_api = openai_api
 
-        if "BaseURL" in config["OpenAI"]:
-            bot.chat_api.base_url = config["OpenAI"]["BaseURL"]
-            bot.image_api = None
+        if "ImageModel" in config["OpenAI"]:
+            bot.image_api = openai_api
+
+        if "TTSModel" in config["OpenAI"]:
+            bot.tts_api = openai_api
+
+        if "STTModel" in config["OpenAI"]:
+            bot.stt_api = openai_api
 
         # Set up WolframAlpha
         if "WolframAlpha" in config:
+            from .wolframalpha import WolframAlpha
+
             bot.calculation_api = WolframAlpha(
                 config["WolframAlpha"]["APIKey"], bot.logger
             )
 
         # Set up TrackingMore
         if "TrackingMore" in config:
+            from .trackingmore import TrackingMore
+
             bot.parcel_api = TrackingMore(config["TrackingMore"]["APIKey"], bot.logger)
 
         # Set up the Matrix client
-
         assert "Matrix" in config, "Matrix config not found"
 
         homeserver = config["Matrix"]["Homeserver"]
-        bot.matrix_client = AsyncClient(homeserver)
-        bot.matrix_client.access_token = config["Matrix"]["AccessToken"]
-        bot.matrix_client.user_id = config["Matrix"].get("UserID")
-        bot.matrix_client.device_id = config["Matrix"].get("DeviceID")
 
-        # Return the new GPTBot instance
-        return bot
+        if config.get("Matrix", "Password", fallback=""):
+            if not config.get("Matrix", "UserID", fallback=""):
+                raise Exception("Cannot log in: UserID not set in config")
+
+            bot.matrix_client = AsyncClient(homeserver, user=config["Matrix"]["UserID"])
+            login = await bot.matrix_client.login(password=config["Matrix"]["Password"])
+
+            if isinstance(login, LoginError):
+                raise Exception(f"Could not log in: {login.message}")
+
+            config["Matrix"]["AccessToken"] = bot.matrix_client.access_token
+            config["Matrix"]["DeviceID"] = bot.matrix_client.device_id
+            config["Matrix"]["Password"] = ""
+
+        else:
+            bot.matrix_client = AsyncClient(homeserver)
+
+            bot.matrix_client.access_token = config["Matrix"]["AccessToken"]
+            bot.matrix_client.user_id = config["Matrix"].get("UserID")
+            bot.matrix_client.device_id = config["Matrix"].get("DeviceID")
+
+        # Return the new GPTBot instance and the (potentially modified) config
+        return bot, config
 
     async def _get_user_id(self) -> str:
         """Get the user ID of the bot from the whoami endpoint.
@@ -194,9 +289,14 @@ class GPTBot:
 
         return user_id
 
-    async def _last_n_messages(self, room: str | MatrixRoom, n: Optional[int]):
+    async def _last_n_messages(
+        self,
+        room: str | MatrixRoom,
+        n: Optional[int],
+        ignore_notices: bool = True,
+    ):
         messages = []
-        n = n or self.max_messages
+        n = n or self.chat_api.max_messages
         room_id = room.room_id if isinstance(room, MatrixRoom) else room
 
         self.logger.log(
@@ -217,72 +317,47 @@ class GPTBot:
             )
 
         for event in response.chunk:
-            if len(messages) >= n:
-                break
-            if isinstance(event, MegolmEvent):
+            try:
+                event_type = event.type
+            except AttributeError:
                 try:
-                    event = await self.matrix_client.decrypt_event(event)
-                except (GroupEncryptionError, EncryptionError):
-                    self.logger.log(
-                        f"Could not decrypt message {event.event_id} in room {room_id}",
-                        "error",
-                    )
-                    continue
-            if isinstance(event, (RoomMessageText, RoomMessageNotice)):
-                if event.body.startswith("!gptbot ignoreolder"):
+                    event_type = event.source["content"]["msgtype"]
+                except KeyError:
+                    if event.__class__.__name__ in ("RoomMemberEvent",):
+                        self.logger.log(
+                            f"Ignoring event of type {event.__class__.__name__}",
+                            "debug",
+                        )
+                        continue
+                    self.logger.log(f"Could not process event: {event}", "warning")
+                    continue  # This is most likely not a message event
+
+            if event_type.startswith("gptbot"):
+                messages.append(event)
+
+            elif isinstance(event, RoomMessageText):
+                if event.body.split() == ["!gptbot", "ignoreolder"]:
                     break
                 if (not event.body.startswith("!")) or (
-                    event.body.startswith("!gptbot")
+                    event.body.split()[1] == "custom"
                 ):
                     messages.append(event)
 
+            elif isinstance(event, RoomMessageNotice):
+                if not ignore_notices:
+                    messages.append(event)
+
+            elif isinstance(event, RoomMessageMedia):
+                messages.append(event)
+
+            if len(messages) >= n:
+                break
+
         self.logger.log(f"Found {len(messages)} messages (limit: {n})", "debug")
 
         # Reverse the list so that messages are in chronological order
         return messages[::-1]
 
-    def _truncate(
-        self,
-        messages: list,
-        max_tokens: Optional[int] = None,
-        model: Optional[str] = None,
-        system_message: Optional[str] = None,
-    ):
-        max_tokens = max_tokens or self.max_tokens
-        model = model or self.chat_api.chat_model
-        system_message = (
-            self.default_system_message if system_message is None else system_message
-        )
-
-        encoding = tiktoken.encoding_for_model(model)
-        total_tokens = 0
-
-        system_message_tokens = (
-            0 if not system_message else (len(encoding.encode(system_message)) + 1)
-        )
-
-        if system_message_tokens > max_tokens:
-            self.logger.log(
-                f"System message is too long to fit within token limit ({system_message_tokens} tokens) - cannot proceed",
-                "error",
-            )
-            return []
-
-        total_tokens += system_message_tokens
-
-        total_tokens = len(system_message) + 1
-        truncated_messages = []
-
-        for message in [messages[0]] + list(reversed(messages[1:])):
-            content = message["content"]
-            tokens = len(encoding.encode(content)) + 1
-            if total_tokens + tokens > max_tokens:
-                break
-            total_tokens += tokens
-            truncated_messages.append(message)
-
-        return [truncated_messages[0]] + list(reversed(truncated_messages[1:]))
-
     async def _get_device_id(self) -> str:
         """Guess the device ID of the bot.
         Requires an access token to be set up.
@@ -305,6 +380,44 @@ class GPTBot:
 
         return device_id
 
+    async def call_tool(self, tool_call: dict, room: str, user: str, **kwargs):
+        """Call a tool.
+
+        Args:
+            tool_call (dict): The tool call to make.
+            room (str): The room to call the tool in.
+            user (str): The user to call the tool as.
+        """
+
+        tool = tool_call.function.name
+        args = json.loads(tool_call.function.arguments)
+
+        self.logger.log(
+            f"Calling tool {tool} with args {args} for user {user} in room {room}",
+            "debug",
+        )
+
+        await self.send_message(
+            room, f"Calling tool {tool} with arguments {args}.", True
+        )
+
+        try:
+            tool_class = TOOLS[tool]
+            result = await tool_class(**args, room=room, bot=self, user=user).run()
+            await self.send_message(room, result, msgtype="gptbot.tool_result")
+            return result
+
+        except (Handover, StopProcessing):
+            raise
+
+        except KeyError:
+            self.logger.log(f"Tool {tool} not found", "error")
+            return "Error: Tool not found"
+
+        except Exception as e:
+            self.logger.log(f"Error calling tool {tool}: {e}", "error")
+            return f"Error: Something went wrong calling tool {tool}"
+
     async def process_command(self, room: MatrixRoom, event: RoomMessageText):
         """Process a command. Called from the event_callback() method.
         Delegates to the appropriate command handler.
@@ -318,6 +431,10 @@ class GPTBot:
             f"Received command {event.body} from {event.sender} in room {room.room_id}",
             "debug",
         )
+
+        if event.body.startswith("* "):
+            event.body = event.body[2:]
+
         command = event.body.split()[1] if event.body.split()[1:] else None
 
         await COMMANDS.get(command, COMMANDS[None])(room, event, self)
@@ -371,13 +488,31 @@ class GPTBot:
         return (
             (
                 user_id in self.allowed_users
-                or f"*:{user_id.split(':')[1]}" in self.allowed_users
-                or f"@*:{user_id.split(':')[1]}" in self.allowed_users
+                or (
+                    (
+                        f"*:{user_id.split(':')[1]}" in self.allowed_users
+                        or f"@*:{user_id.split(':')[1]}" in self.allowed_users
+                    )
+                    if not user_id.startswith("!") or user_id.startswith("#")
+                    else False
+                )
             )
             if self.allowed_users
             else True
         )
 
+    def room_is_allowed(self, room_id: str) -> bool:
+        """Check if everyone in a room is allowed to use the bot.
+
+        Args:
+            room_id (str): The room ID to check.
+
+        Returns:
+            bool: Whether everyone in the room is allowed to use the bot.
+        """
+        # TODO: Handle published aliases
+        return self.user_is_allowed(room_id)
+
     async def event_callback(self, room: MatrixRoom, event: Event):
         """Callback for events.
 
@@ -389,7 +524,9 @@ class GPTBot:
         if event.sender == self.matrix_client.user_id:
             return
 
-        if not self.user_is_allowed(event.sender):
+        if not (
+            self.user_is_allowed(event.sender) or self.room_is_allowed(room.room_id)
+        ):
             if len(room.users) == 2:
                 await self.matrix_client.room_send(
                     room.room_id,
@@ -401,7 +538,7 @@ class GPTBot:
                 )
             return
 
-        task = asyncio.create_task(self._event_callback(room, event))
+        asyncio.create_task(self._event_callback(room, event))
 
     def room_uses_timing(self, room: MatrixRoom):
         """Check if a room uses timing.
@@ -429,7 +566,7 @@ class GPTBot:
                 await callback(response, self)
 
     async def response_callback(self, response: Response):
-        task = asyncio.create_task(self._response_callback(response))
+        asyncio.create_task(self._response_callback(response))
 
     async def accept_pending_invites(self):
         """Accept all pending invites."""
@@ -438,7 +575,7 @@ class GPTBot:
 
         invites = self.matrix_client.invited_rooms
 
-        for invite in invites.keys():
+        for invite in [k for k in invites.keys()]:
             if invite in self.room_ignore_list:
                 self.logger.log(
                     f"Ignoring invite to room {invite} (room is in ignore list)",
@@ -497,13 +634,16 @@ class GPTBot:
         """Send an image to a room.
 
         Args:
-            room (MatrixRoom): The room to send the image to.
+            room (MatrixRoom|str): The room to send the image to.
             image (bytes): The image to send.
             message (str, optional): The message to send with the image. Defaults to None.
         """
 
+        if isinstance(room, MatrixRoom):
+            room = room.room_id
+
         self.logger.log(
-            f"Sending image of size {len(image)} bytes to room {room.room_id}", "debug"
+            f"Sending image of size {len(image)} bytes to room {room}", "debug"
         )
 
         bio = BytesIO(image)
@@ -533,14 +673,50 @@ class GPTBot:
             "url": content_uri,
         }
 
-        status = await self.matrix_client.room_send(
-            room.room_id, "m.room.message", content
-        )
+        await self.matrix_client.room_send(room, "m.room.message", content)
 
         self.logger.log("Sent image", "debug")
 
+    async def send_file(
+        self, room: MatrixRoom, file: bytes, filename: str, mime: str, msgtype: str
+    ):
+        """Send a file to a room.
+
+        Args:
+            room (MatrixRoom|str): The room to send the file to.
+            file (bytes): The file to send.
+            filename (str): The name of the file.
+            mime (str): The MIME type of the file.
+        """
+
+        if isinstance(room, MatrixRoom):
+            room = room.room_id
+
+        self.logger.log(
+            f"Sending file of size {len(file)} bytes to room {room}", "debug"
+        )
+
+        content_uri = await self.upload_file(file, filename, mime)
+
+        self.logger.log("Uploaded file - sending message...", "debug")
+
+        content = {
+            "body": filename,
+            "info": {"mimetype": mime, "size": len(file)},
+            "msgtype": msgtype,
+            "url": content_uri,
+        }
+
+        await self.matrix_client.room_send(room, "m.room.message", content)
+
+        self.logger.log("Sent file", "debug")
+
     async def send_message(
-        self, room: MatrixRoom | str, message: str, notice: bool = False
+        self,
+        room: MatrixRoom | str,
+        message: str,
+        notice: bool = False,
+        msgtype: Optional[str] = None,
     ):
         """Send a message to a room.
 
@@ -556,47 +732,24 @@ class GPTBot:
         markdowner = markdown2.Markdown(extras=["fenced-code-blocks"])
         formatted_body = markdowner.convert(message)
 
-        msgtype = "m.notice" if notice else "m.text"
+        msgtype = msgtype if msgtype else "m.notice" if notice else "m.text"
 
-        msgcontent = {
-            "msgtype": msgtype,
-            "body": message,
-            "format": "org.matrix.custom.html",
-            "formatted_body": formatted_body,
-        }
+        if not msgtype.startswith("gptbot."):
+            msgcontent = {
+                "msgtype": msgtype,
+                "body": message,
+                "format": "org.matrix.custom.html",
+                "formatted_body": formatted_body,
+            }
+
+        else:
+            msgcontent = {
+                "msgtype": msgtype,
+                "content": message,
+            }
 
         content = None
 
-        if self.matrix_client.olm and room.encrypted:
-            try:
-                if not room.members_synced:
-                    responses = []
-                    responses.append(
-                        await self.matrix_client.joined_members(room.room_id)
-                    )
-
-                if self.matrix_client.olm.should_share_group_session(room.room_id):
-                    try:
-                        event = self.matrix_client.sharing_session[room.room_id]
-                        await event.wait()
-                    except KeyError:
-                        await self.matrix_client.share_group_session(
-                            room.room_id,
-                            ignore_unverified_devices=True,
-                        )
-
-                if msgtype != "m.reaction":
-                    response = self.matrix_client.encrypt(
-                        room.room_id, "m.room.message", msgcontent
-                    )
-                    msgtype, content = response
-
-            except Exception as e:
-                self.logger.log(
-                    f"Error encrypting message: {e} - sending unencrypted", "warning"
-                )
-                raise
-
         if not content:
             msgtype = "m.room.message"
             content = msgcontent
@@ -643,6 +796,22 @@ class GPTBot:
             (message, room, tokens, api, datetime.now()),
         )
 
+    async def get_state_event(
+        self, room: MatrixRoom | str, event_type: str, state_key: Optional[str] = None
+    ):
+        if isinstance(room, MatrixRoom):
+            room = room.room_id
+
+        state = await self.matrix_client.room_get_state(room)
+
+        if isinstance(state, RoomGetStateError):
+            self.logger.log(f"Could not get state for room {room}")
+
+        for event in state.events:
+            if event["type"] == event_type:
+                if state_key is None or event["state_key"] == state_key:
+                    return event
+
     async def run(self):
         """Start the bot."""
 
@@ -657,16 +826,10 @@ class GPTBot:
         if not self.matrix_client.device_id:
             self.matrix_client.device_id = await self._get_device_id()
 
-        # Set up database
-
-        IN_MEMORY = False
         if not self.database:
-            self.logger.log(
-                "No database connection set up, using in-memory database. Data will be lost on bot shutdown.",
-                "warning",
+            self.database = sqlite3.connect(
+                Path(__file__).parent.parent / "database.db"
             )
-            IN_MEMORY = True
-            self.database = sqlite3.connect(":memory:")
 
         self.logger.log("Running migrations...")
 
@@ -686,35 +849,17 @@ class GPTBot:
         else:
             self.logger.log(f"Already at latest version {after}.")
 
-        if IN_MEMORY:
-            client_config = AsyncClientConfig(
-                store_sync_tokens=True, encryption_enabled=False
-            )
-        else:
-            matrix_store = SqliteStore
-            client_config = AsyncClientConfig(
-                store_sync_tokens=True, encryption_enabled=True, store=matrix_store
-            )
-            self.matrix_client.config = client_config
-            self.matrix_client.store = matrix_store(
-                self.matrix_client.user_id,
-                self.matrix_client.device_id,
-                '.', #store path
-                database_name=self.crypto_store_path or "",
-            )
+        matrix_store = SqliteStore
+        client_config = AsyncClientConfig(
+            store_sync_tokens=True, encryption_enabled=False, store=matrix_store
+        )
+        self.matrix_client.config = client_config
 
-            self.matrix_client.olm = Olm(
-                self.matrix_client.user_id,
-                self.matrix_client.device_id,
-                self.matrix_client.store,
-            )
+        # Run initial sync (includes joining rooms)
 
-            self.matrix_client.encrypted_rooms = (
-                self.matrix_client.store.load_encrypted_rooms()
-            )
+        self.logger.log("Running initial sync...", "debug")
 
-        # Run initial sync (now includes joining rooms)
-        sync = await self.matrix_client.sync(timeout=30000)
+        sync = await self.matrix_client.sync(timeout=30000, full_state=True)
         if isinstance(sync, SyncResponse):
             await self.response_callback(sync)
         else:
@@ -723,6 +868,8 @@ class GPTBot:
 
         # Set up callbacks
 
+        self.logger.log("Setting up callbacks...", "debug")
+
         self.matrix_client.add_event_callback(self.event_callback, Event)
         self.matrix_client.add_response_callback(self.response_callback, Response)
 
@@ -743,20 +890,22 @@ class GPTBot:
             asyncio.create_task(self.matrix_client.set_avatar(uri))
 
             for room in self.matrix_client.rooms.keys():
-                self.logger.log(f"Setting avatar for {room}...", "debug")
-                asyncio.create_task(
-                    self.matrix_client.room_put_state(
-                        room, "m.room.avatar", {"url": uri}, ""
+                room_avatar = await self.get_state_event(room, "m.room.avatar")
+                if not room_avatar:
+                    self.logger.log(f"Setting avatar for {room}...", "debug")
+                    asyncio.create_task(
+                        self.matrix_client.room_put_state(
+                            room, "m.room.avatar", {"url": uri}, ""
+                        )
                     )
-                )
 
         # Start syncing events
         self.logger.log("Starting sync loop...", "warning")
         try:
-            await self.matrix_client.sync_forever(timeout=30000)
+            await self.matrix_client.sync_forever(timeout=30000, full_state=True)
         finally:
             self.logger.log("Syncing one last time...", "warning")
-            await self.matrix_client.sync(timeout=30000)
+            await self.matrix_client.sync(timeout=30000, full_state=True)
 
     async def create_space(self, name, visibility=RoomVisibility.private) -> str:
         """Create a space.
@@ -818,6 +967,46 @@ class GPTBot:
                 space,
             )
 
+    def room_uses_stt(self, room: MatrixRoom | str) -> bool:
+        """Check if a room uses STT.
+
+        Args:
+            room (MatrixRoom | str): The room to check.
+
+        Returns:
+            bool: Whether the room uses STT.
+        """
+        room_id = room.room_id if isinstance(room, MatrixRoom) else room
+
+        with closing(self.database.cursor()) as cursor:
+            cursor.execute(
+                "SELECT value FROM room_settings WHERE room_id = ? AND setting = ?",
+                (room_id, "stt"),
+            )
+            result = cursor.fetchone()
+
+        return False if not result else bool(int(result[0]))
+
+    def room_uses_tts(self, room: MatrixRoom | str) -> bool:
+        """Check if a room uses TTS.
+
+        Args:
+            room (MatrixRoom | str): The room to check.
+
+        Returns:
+            bool: Whether the room uses TTS.
+        """
+        room_id = room.room_id if isinstance(room, MatrixRoom) else room
+
+        with closing(self.database.cursor()) as cursor:
+            cursor.execute(
+                "SELECT value FROM room_settings WHERE room_id = ? AND setting = ?",
+                (room_id, "tts"),
+            )
+            result = cursor.fetchone()
+
+        return False if not result else bool(int(result[0]))
+
     def respond_to_room_messages(self, room: MatrixRoom | str) -> bool:
         """Check whether the bot should respond to all messages sent in a room.
 
@@ -840,6 +1029,28 @@ class GPTBot:
 
         return True if not result else bool(int(result[0]))
 
+    async def get_room_model(self, room: MatrixRoom | str) -> str:
+        """Get the model used for a room.
+
+        Args:
+            room (MatrixRoom | str): The room to check.
+
+        Returns:
+            str: The model used for the room.
+        """
+
+        if isinstance(room, MatrixRoom):
+            room = room.room_id
+
+        with closing(self.database.cursor()) as cursor:
+            cursor.execute(
+                "SELECT value FROM room_settings WHERE room_id = ? AND setting = ?",
+                (room, "model"),
+            )
+            result = cursor.fetchone()
+
+        return result[0] if result else self.chat_api.chat_model
+
     async def process_query(
         self, room: MatrixRoom, event: RoomMessageText, from_chat_command: bool = False
     ):
@@ -889,7 +1100,10 @@ class GPTBot:
                 return
 
         try:
-            last_messages = await self._last_n_messages(room.room_id, 20)
+            last_messages = await self._last_n_messages(
+                room.room_id, self.chat_api.max_messages
+            )
+            self.logger.log(f"Last messages: {last_messages}", "debug")
         except Exception as e:
             self.logger.log(f"Error getting last messages: {e}", "error")
             await self.send_message(
@@ -899,28 +1113,24 @@ class GPTBot:
 
         system_message = self.get_system_message(room)
 
-        chat_messages = [{"role": "system", "content": system_message}]
-
-        for message in last_messages:
-            role = (
-                "assistant" if message.sender == self.matrix_client.user_id else "user"
-            )
-            if not message.event_id == event.event_id:
-                chat_messages.append({"role": role, "content": message.body})
-
-        chat_messages.append({"role": "user", "content": event.body})
-
-        # Truncate messages to fit within the token limit
-        truncated_messages = self._truncate(
-            chat_messages, self.max_tokens - 1, system_message=system_message
+        chat_messages = await self.chat_api.prepare_messages(
+            event, last_messages, system_message
         )
 
+        # Check for a model override
+        if self.allow_model_override:
+            model = await self.get_room_model(room)
+        else:
+            model = self.chat_api.chat_model
+
         try:
             response, tokens_used = await self.chat_api.generate_chat_response(
-                chat_messages, user=room.room_id
+                chat_messages, user=event.sender, room=room.room_id, model=model
             )
         except Exception as e:
+            print(traceback.format_exc())
             self.logger.log(f"Error generating response: {e}", "error")
+
             await self.send_message(
                 room, "Something went wrong. Please try again.", True
             )
@@ -936,19 +1146,51 @@ class GPTBot:
 
             self.logger.log(f"Sending response to room {room.room_id}...")
 
-            # Convert markdown to HTML
+            if self.room_uses_tts(room):
+                self.logger.log("TTS enabled for room", "debug")
 
-            message = await self.send_message(room, response)
+                try:
+                    audio = await self.tts_api.text_to_speech(response)
+                    await self.send_file(room, audio, response, "audio/mpeg", "m.audio")
+                    return
 
-        else:
-            # Send a notice to the room if there was an error
-            self.logger.log("Didn't get a response from GPT API", "error")
-            await self.send_message(
-                room, "Something went wrong. Please try again.", True
-            )
+                except Exception as e:
+                    self.logger.log(f"Error generating audio: {e}", "error")
+                    await self.send_message(
+                        room, "Something went wrong generating audio file.", True
+                    )
+                    
+                    if self.debug:
+                        await self.send_message(
+                            room, f"Error: {e}\n\n```\n{traceback.format_exc()}\n```", True
+                        )
+
+            await self.send_message(room, response)
 
         await self.matrix_client.room_typing(room.room_id, False)
 
+    async def download_file(
+        self, mxc: str, raise_error: bool = False
+    ) -> Union[DiskDownloadResponse, MemoryDownloadResponse]:
+        """Download a file from the homeserver.
+
+        Args:
+            mxc (str): The MXC URI of the file to download.
+
+        Returns:
+            Optional[bytes]: The downloaded file, or None if there was an error.
+        """
+
+        download = await self.matrix_client.download(mxc)
+
+        if isinstance(download, DownloadError):
+            self.logger.log(f"Error downloading file: {download.message}", "error")
+            if raise_error:
+                raise DownloadException(download.message)
+            return
+
+        return download
+
     def get_system_message(self, room: MatrixRoom | str) -> str:
         """Get the system message for a room.
 

src/gptbot/classes/exceptions.py

@@ -0,0 +1,2 @@
+class DownloadException(Exception):
+    pass

src/gptbot/classes/openai.py

@@ -1,164 +0,0 @@
-import openai
-import requests
-
-import asyncio
-import json
-from functools import partial
-
-from .logging import Logger
-
-from typing import Dict, List, Tuple, Generator, AsyncGenerator, Optional, Any
-
-class OpenAI:
-    api_key: str
-    chat_model: str = "gpt-3.5-turbo"
-    logger: Logger
-
-    api_code: str = "openai"
-
-    @property
-    def chat_api(self) -> str:
-        return self.chat_model
-
-    classification_api = chat_api
-    image_api: str = "dalle"
-
-    operator: str = "OpenAI ([https://openai.com](https://openai.com))"
-
-    def __init__(self, api_key, chat_model=None, logger=None):
-        self.api_key = api_key
-        self.chat_model = chat_model or self.chat_model
-        self.logger = logger or Logger()
-        self.base_url = openai.api_base
-
-    async def _request_with_retries(self, request: partial, attempts: int = 5, retry_interval: int = 2) -> AsyncGenerator[Any | list | Dict, None]:
-        """Retry a request a set number of times if it fails.
-
-        Args:
-            request (partial): The request to make with retries.
-            attempts (int, optional): The number of attempts to make. Defaults to 5.
-            retry_interval (int, optional): The interval in seconds between attempts. Defaults to 2 seconds.
-
-        Returns:
-            AsyncGenerator[Any | list | Dict, None]: The OpenAI response for the request.
-        """
-        # call the request function and return the response if it succeeds, else retry
-        current_attempt = 1
-        while current_attempt <= attempts:
-            try:
-                response = await request()
-                return response
-            except Exception as e:
-                self.logger.log(f"Request failed: {e}", "error")
-                self.logger.log(f"Retrying in {retry_interval} seconds...")
-                await asyncio.sleep(retry_interval)
-                current_attempt += 1
-
-        # if all attempts failed, raise an exception
-        raise Exception("Request failed after all attempts.")
-
-    async def generate_chat_response(self, messages: List[Dict[str, str]], user: Optional[str] = None) -> Tuple[str, int]:
-        """Generate a response to a chat message.
-
-        Args:
-            messages (List[Dict[str, str]]): A list of messages to use as context.
-
-        Returns:
-            Tuple[str, int]: The response text and the number of tokens used.
-        """
-        self.logger.log(f"Generating response to {len(messages)} messages using {self.chat_model}...")
-
-
-        chat_partial = partial(
-            openai.ChatCompletion.acreate,
-                model=self.chat_model,
-                messages=messages,
-                api_key=self.api_key,
-                user=user,
-                api_base=self.base_url,
-        )
-        response = await self._request_with_retries(chat_partial)
-
-
-        result_text = response.choices[0].message['content']
-        tokens_used = response.usage["total_tokens"]
-        self.logger.log(f"Generated response with {tokens_used} tokens.")
-        return result_text, tokens_used
-
-    async def classify_message(self, query: str, user: Optional[str] = None) -> Tuple[Dict[str, str], int]:
-        system_message = """You are a classifier for different types of messages. You decide whether an incoming message is meant to be a prompt for an AI chat model, or meant for a different API. You respond with a JSON object like this:
-
-{ "type": event_type, "prompt": prompt }
-
-- If the message you received is meant for the AI chat model, the event_type is "chat", and the prompt is the literal content of the message you received. This is also the default if none of the other options apply.
-- If it is a prompt for a calculation that can be answered better by WolframAlpha than an AI chat bot, the event_type is "calculate". Optimize the message you received for input to WolframAlpha, and return it as the prompt attribute.
-- If it is a prompt for an AI image generation, the event_type is "imagine". Optimize the message you received for use with DALL-E, and return it as the prompt attribute.
-- If the user is asking you to create a new room, the event_type is "newroom", and the prompt is the name of the room, if one is given, else an empty string.
-- If the user is asking you to throw a coin, the event_type is "coin". The prompt is an empty string.
-- If the user is asking you to roll a dice, the event_type is "dice". The prompt is an string containing an optional number of sides, if one is given, else an empty string.
-- If for any reason you are unable to classify the message (for example, if it infringes on your terms of service), the event_type is "error", and the prompt is a message explaining why you are unable to process the message.
-
-Only the event_types mentioned above are allowed, you must not respond in any other way."""
-        messages = [
-            {
-                "role": "system",
-                "content": system_message
-            },
-            {
-                "role": "user",
-                "content": query
-            }
-        ]
-
-        self.logger.log(f"Classifying message '{query}'...")
-
-        chat_partial = partial(
-            openai.ChatCompletion.acreate,
-                model=self.chat_model,
-                messages=messages,
-                api_key=self.api_key,
-                user=user,
-                api_base=self.base_url,
-        )
-        response = await self._request_with_retries(chat_partial)
-
-        try:
-            result = json.loads(response.choices[0].message['content'])
-        except:
-            result = {"type": "chat", "prompt": query}
-
-        tokens_used = response.usage["total_tokens"]
-
-        self.logger.log(f"Classified message as {result['type']} with {tokens_used} tokens.")
-
-        return result, tokens_used
-
-    async def generate_image(self, prompt: str, user: Optional[str] = None) -> Generator[bytes, None, None]:
-        """Generate an image from a prompt.
-
-        Args:
-            prompt (str): The prompt to use.
-
-        Yields:
-            bytes: The image data.
-        """
-        self.logger.log(f"Generating image from prompt '{prompt}'...")
-
-        image_partial = partial(
-            openai.Image.acreate,
-                prompt=prompt,
-                n=1,
-                api_key=self.api_key,
-                size="1024x1024",
-                user=user,
-                api_base=self.base_url,
-        )
-        response = await self._request_with_retries(image_partial)
-
-        images = []
-
-        for image in response.data:
-            image = requests.get(image.url).content
-            images.append(image)
-
-        return images, len(images)

src/gptbot/classes/trackingmore.py

@@ -1,9 +1,8 @@
 import trackingmore
-import requests
 
 from .logging import Logger
 
-from typing import Dict, List, Tuple, Generator, Optional
+from typing import Tuple, Optional
 
 class TrackingMore:
     api_key: str

src/gptbot/classes/wolframalpha.py

@@ -3,7 +3,7 @@ import requests
 
 from .logging import Logger
 
-from typing import Dict, List, Tuple, Generator, Optional
+from typing import Generator, Optional
 
 class WolframAlpha:
     api_key: str

src/gptbot/commands/__init__.py

@@ -22,6 +22,7 @@ for command in [
     "dice",
     "parcel",
     "space",
+    "tts",
 ]:
     function = getattr(import_module(
         "." + command, "gptbot.commands"), "command_" + command)

src/gptbot/commands/botinfo.py

@@ -3,21 +3,16 @@ from nio.rooms import MatrixRoom
 
 
 async def command_botinfo(room: MatrixRoom, event: RoomMessageText, bot):
-    logging("Showing bot info...")
+    bot.logger.log("Showing bot info...")
 
-    body = f"""GPT Info:
+    body = f"""GPT Room info:
 
-Model: {bot.model}
-Maximum context tokens: {bot.max_tokens}
-Maximum context messages: {bot.max_messages}
-
-Room info:
-
-Bot user ID: {bot.matrix_client.user_id}
-Current room ID: {room.room_id}
+Model: {await bot.get_room_model(room)}\n
+Maximum context tokens: {bot.chat_api.max_tokens}\n
+Maximum context messages: {bot.chat_api.max_messages}\n
+Bot user ID: {bot.matrix_client.user_id}\n
+Current room ID: {room.room_id}\n
 System message: {bot.get_system_message(room)}
-
-For usage statistics, run !gptbot stats
 """
 
     await bot.send_message(room, body, True)

src/gptbot/commands/calculate.py

@@ -23,14 +23,12 @@ async def command_calculate(room: MatrixRoom, event: RoomMessageText, bot):
         bot.logger.log("Querying calculation API...")
 
         for subpod in bot.calculation_api.generate_calculation_response(prompt, text, results_only, user=room.room_id):
-            bot.logger.log(f"Sending subpod...")
+            bot.logger.log("Sending subpod...")
             if isinstance(subpod, bytes):
                 await bot.send_image(room, subpod)
             else:
                 await bot.send_message(room, subpod, True)
 
-        bot.log_api_usage(event, room, f"{bot.calculation_api.api_code}-{bot.calculation_api.calculation_api}", tokens_used)
-
         return
 
     await bot.send_message(room, "You need to provide a prompt.", True)

src/gptbot/commands/dice.py

@@ -9,7 +9,7 @@ async def command_dice(room: MatrixRoom, event: RoomMessageText, bot):
 
     try:
         sides = int(event.body.split()[2])
-    except ValueError:
+    except (ValueError, IndexError):
         sides = 6
 
     if sides < 2:

src/gptbot/commands/help.py

@@ -8,17 +8,17 @@ async def command_help(room: MatrixRoom, event: RoomMessageText, bot):
 - !gptbot help - Show this message
 - !gptbot botinfo - Show information about the bot
 - !gptbot privacy - Show privacy information
-- !gptbot newroom \<room name\> - Create a new room and invite yourself to it
-- !gptbot stats - Show usage statistics for this room
-- !gptbot systemmessage \<message\> - Get or set the system message for this room
+- !gptbot newroom <room name> - Create a new room and invite yourself to it
+- !gptbot systemmessage <message> - Get or set the system message for this room
 - !gptbot space [enable|disable|update|invite] - Enable, disable, force update, or invite yourself to your space
 - !gptbot coin - Flip a coin (heads or tails)
 - !gptbot dice [number] - Roll a dice with the specified number of sides (default: 6)
-- !gptbot imagine \<prompt\> - Generate an image from a prompt
-- !gptbot calculate [--text] [--details] \<query\> - Calculate a result to a calculation, optionally forcing text output instead of an image, and optionally showing additional details like the input interpretation
-- !gptbot chat \<message\> - Send a message to the chat API
-- !gptbot classify \<message\> - Classify a message using the classification API
-- !gptbot custom \<message\> - Used for custom commands handled by the chat model and defined through the room's system message
+- !gptbot imagine <prompt> - Generate an image from a prompt
+- !gptbot calculate [--text] [--details] <query> - Calculate a result to a calculation, optionally forcing text output instead of an image, and optionally showing additional details like the input interpretation
+- !gptbot chat <message> - Send a message to the chat API
+- !gptbot classify <message> - Classify a message using the classification API
+- !gptbot custom <message> - Used for custom commands handled by the chat model and defined through the room's system message
+- !gptbot roomsettings [use_classification|use_timing|always_reply|system_message|tts] [true|false|<message>] - Get or set room settings
 - !gptbot ignoreolder - Ignore messages before this point as context
 """
 

src/gptbot/commands/imagine.py

@@ -16,10 +16,10 @@ async def command_imagine(room: MatrixRoom, event: RoomMessageText, bot):
             return
 
         for image in images:
-            bot.logger.log(f"Sending image...")
+            bot.logger.log("Sending image...")
             await bot.send_image(room, image)
 
-        bot.log_api_usage(event, room, f"{bot.image_api.api_code}-{bot.image_api.image_api}", tokens_used)
+        bot.log_api_usage(event, room, f"{bot.image_api.api_code}-{bot.image_api.image_model}", tokens_used)
 
         return
 

src/gptbot/commands/newroom.py

@@ -13,7 +13,7 @@ async def command_newroom(room: MatrixRoom, event: RoomMessageText, bot):
 
     if isinstance(new_room, RoomCreateError):
         bot.logger.log(f"Failed to create room: {new_room.message}")
-        await bot.send_message(room, f"Sorry, I was unable to create a new room. Please try again later, or create a room manually.", True)
+        await bot.send_message(room, "Sorry, I was unable to create a new room. Please try again later, or create a room manually.", True)
         return
 
     bot.logger.log(f"Inviting {event.sender} to new room...")
@@ -21,7 +21,7 @@ async def command_newroom(room: MatrixRoom, event: RoomMessageText, bot):
 
     if isinstance(invite, RoomInviteError):
         bot.logger.log(f"Failed to invite user: {invite.message}")
-        await bot.send_message(room, f"Sorry, I was unable to invite you to the new room. Please try again later, or create a room manually.", True)
+        await bot.send_message(room, "Sorry, I was unable to invite you to the new room. Please try again later, or create a room manually.", True)
         return
 
     with closing(bot.database.cursor()) as cursor:
@@ -43,4 +43,4 @@ async def command_newroom(room: MatrixRoom, event: RoomMessageText, bot):
 
     await bot.matrix_client.joined_rooms()
     await bot.send_message(room, f"Alright, I've created a new room called '{room_name}' and invited you to it. You can find it at {new_room.room_id}", True)
-    await bot.send_message(bot.matrix_client.rooms[new_room.room_id], f"Welcome to the new room! What can I do for you?")
+    await bot.send_message(bot.matrix_client.rooms[new_room.room_id], "Welcome to the new room! What can I do for you?")

src/gptbot/commands/privacy.py

@@ -11,7 +11,7 @@ async def command_privacy(room: MatrixRoom, event: RoomMessageText, bot):
         body += "- For chat requests: " + f"{bot.chat_api.operator}" + "\n"
     if bot.image_api:
         body += "- For image generation requests (!gptbot imagine): " + f"{bot.image_api.operator}" + "\n"
-    if bot.calculate_api:
-        body += "- For calculation requests (!gptbot calculate): " + f"{bot.calculate_api.operator}" + "\n"
+    if bot.calculation_api:
+        body += "- For calculation requests (!gptbot calculate): " + f"{bot.calculation_api.operator}" + "\n"
 
     await bot.send_message(room, body, True)

src/gptbot/commands/roomsettings.py

@@ -25,6 +25,8 @@ async def command_roomsettings(room: MatrixRoom, event: RoomMessageText, bot):
                     (room.room_id, "system_message", value, value)
                 )
 
+            bot.database.commit()
+
             await bot.send_message(room, f"Alright, I've stored the system message: '{value}'.", True)
             return
 
@@ -35,7 +37,7 @@ async def command_roomsettings(room: MatrixRoom, event: RoomMessageText, bot):
         await bot.send_message(room, f"The current system message is: '{system_message}'.", True)
         return
 
-    if setting in ("use_classification", "always_reply", "use_timing"):
+    if setting in ("use_classification", "always_reply", "use_timing", "tts", "stt"):
         if value:
             if value.lower() in ["true", "false"]:
                 value = value.lower() == "true"
@@ -49,6 +51,8 @@ async def command_roomsettings(room: MatrixRoom, event: RoomMessageText, bot):
                         (room.room_id, setting, "1" if value else "0", "1" if value else "0")
                     )
 
+                bot.database.commit()
+
                 await bot.send_message(room, f"Alright, I've set {setting} to: '{value}'.", True)
                 return
 
@@ -76,11 +80,51 @@ async def command_roomsettings(room: MatrixRoom, event: RoomMessageText, bot):
         await bot.send_message(room, f"The current {setting} status is: '{value}'.", True)
         return
 
-    message = f"""The following settings are available:
+    if bot.allow_model_override and setting == "model":
+        if value:
+            bot.logger.log(f"Setting chat model for {room.room_id} to {value}...")
+
+            with closing(bot.database.cursor()) as cur:
+                cur.execute(
+                    """INSERT INTO room_settings (room_id, setting, value) VALUES (?, ?, ?)
+                    ON CONFLICT (room_id, setting) DO UPDATE SET value = ?;""",
+                    (room.room_id, "model", value, value)
+                )
+
+            bot.database.commit()
+
+            await bot.send_message(room, f"Alright, I've set the chat model to: '{value}'.", True)
+            return
+
+        bot.logger.log(f"Retrieving chat model for {room.room_id}...")
+
+        with closing(bot.database.cursor()) as cur:
+            cur.execute(
+                """SELECT value FROM room_settings WHERE room_id = ? AND setting = ?;""",
+                (room.room_id, "model")
+            )
+
+            value = cur.fetchone()[0]
+
+            if not value:
+                value = bot.chat_api.chat_model
+            else:
+                value = str(value)
+
+        await bot.send_message(room, f"The current chat model is: '{value}'.", True)
+        return
+
+    message = """The following settings are available:
 
 - system_message [message]: Get or set the system message to be sent to the chat model
 - classification [true/false]: Get or set whether the room uses classification
 - always_reply [true/false]: Get or set whether the bot should reply to all messages (if false, only reply to mentions and commands)
+- tts [true/false]: Get or set whether the bot should generate audio files instead of sending text
+- stt [true/false]: Get or set whether the bot should attempt to process information from audio files
+- timing [true/false]: Get or set whether the bot should return information about the time it took to generate a response
 """
 
+    if bot.allow_model_override:
+        message += "- model [model]: Get or set the chat model to be used for this room"
+
     await bot.send_message(room, message, True)

src/gptbot/commands/space.py

@@ -120,7 +120,7 @@ async def command_space(room: MatrixRoom, event: RoomMessageText, bot):
 
             if isinstance(response, RoomInviteError):
                 bot.logger.log(
-                    f"Failed to invite user {user} to space {space}", "error")
+                    f"Failed to invite user {event.sender} to space {space}", "error")
                 await bot.send_message(
                     room, "Sorry, I couldn't invite you to the space. Please try again later.", True)
                 return

src/gptbot/commands/stats.py

@@ -5,16 +5,30 @@ from contextlib import closing
 
 
 async def command_stats(room: MatrixRoom, event: RoomMessageText, bot):
+    await bot.send_message(
+        room,
+        "The `stats` command is no longer supported. Sorry for the inconvenience.",
+        True,
+    )
+    return
+
+    # Yes, the code below is unreachable, but it's kept here for reference.
+
     bot.logger.log("Showing stats...")
 
     if not bot.database:
         bot.logger.log("No database connection - cannot show stats")
-        bot.send_message(room, "Sorry, I'm not connected to a database, so I don't have any statistics on your usage.", True)
-        return 
+        await bot.send_message(
+            room,
+            "Sorry, I'm not connected to a database, so I don't have any statistics on your usage.",
+            True,
+        )
+        return
 
     with closing(bot.database.cursor()) as cursor:
         cursor.execute(
-            "SELECT SUM(tokens) FROM token_usage WHERE room_id = ?", (room.room_id,))
+            "SELECT SUM(tokens) FROM token_usage WHERE room_id = ?", (room.room_id,)
+        )
         total_tokens = cursor.fetchone()[0] or 0
 
-    bot.send_message(room, f"Total tokens used: {total_tokens}", True)
+    await bot.send_message(room, f"Total tokens used: {total_tokens}", True)

src/gptbot/commands/tts.py

@@ -0,0 +1,23 @@
+from nio.events.room_events import RoomMessageText
+from nio.rooms import MatrixRoom
+
+
+async def command_tts(room: MatrixRoom, event: RoomMessageText, bot):
+    prompt = " ".join(event.body.split()[2:])
+
+    if prompt:
+        bot.logger.log("Generating speech...")
+
+        try:
+            content = await bot.tts_api.text_to_speech(prompt, user=room.room_id)
+        except Exception as e:
+            bot.logger.log(f"Error generating speech: {e}", "error")
+            await bot.send_message(room, "Sorry, I couldn't generate an audio file. Please try again later.", True)
+            return
+
+        bot.logger.log("Sending audio file...")
+        await bot.send_file(room, content, "audio.mp3", "audio/mpeg", "m.audio")
+
+        return
+
+    await bot.send_message(room, "You need to provide a prompt.", True)

src/gptbot/migrations/__init__.py

@@ -22,7 +22,7 @@ def get_version(db: SQLiteConnection) -> int:
 
     try:
         return int(db.execute("SELECT MAX(id) FROM migrations").fetchone()[0])
-    except:
+    except Exception:
         return 0
 
 def migrate(db: SQLiteConnection, from_version: Optional[int] = None, to_version: Optional[int] = None) -> None:

src/gptbot/migrations/migration_2.py

@@ -1,7 +1,5 @@
 # Migration for Matrix Store - No longer used
 
-from datetime import datetime
-from contextlib import closing
 
 def migration(conn):
     pass

src/gptbot/tools/__init__.py

@@ -0,0 +1,21 @@
+from importlib import import_module
+
+from .base import BaseTool, StopProcessing, Handover  # noqa: F401
+
+TOOLS = {}
+
+for tool in [
+    "weather",
+    "geocode",
+    "dice",
+    "websearch",
+    "webrequest",
+    "imagine",
+    "imagedescription",
+    "wikipedia",
+    "datetime",
+    "newroom",
+]:
+    tool_class = getattr(import_module(
+        "." + tool, "gptbot.tools"), tool.capitalize())
+    TOOLS[tool] = tool_class

src/gptbot/tools/base.py

@@ -0,0 +1,21 @@
+class BaseTool:
+    DESCRIPTION: str
+    PARAMETERS: dict
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        self.bot = kwargs.get("bot")
+        self.room = kwargs.get("room")
+        self.user = kwargs.get("user")
+        self.messages = kwargs.get("messages", [])
+
+    async def run(self):
+        raise NotImplementedError()
+
+class StopProcessing(Exception):
+    """Stop processing the message."""
+    pass
+
+class Handover(Exception):
+    """Handover to the original model, if applicable. Stop using tools."""
+    pass

src/gptbot/tools/datetime.py

@@ -0,0 +1,16 @@
+from .base import BaseTool
+
+from datetime import datetime
+
+class Datetime(BaseTool):
+    DESCRIPTION = "Get the current date and time."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+        },
+    }
+
+    async def run(self):
+        """Get the current date and time."""
+        return f"""**Current date and time (UTC)**
+{datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")}"""

src/gptbot/tools/dice.py

@@ -0,0 +1,26 @@
+from .base import BaseTool
+
+from random import SystemRandom
+
+class Dice(BaseTool):
+    DESCRIPTION = "Roll dice."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "dice": {
+                "type": "string",
+                "description": "The number of sides on the dice.",
+                "default": "6",
+            },
+        },
+        "required": [],
+    }
+
+    async def run(self):
+        """Roll dice."""
+        dice = int(self.kwargs.get("dice", 6))
+
+        return f"""**Dice roll**
+Used dice: {dice}
+Result: {SystemRandom().randint(1, dice)}
+"""

src/gptbot/tools/geocode.py

@@ -0,0 +1,34 @@
+from geopy.geocoders import Nominatim
+
+from .base import BaseTool
+
+class Geocode(BaseTool):
+    DESCRIPTION = "Get location information (latitude, longitude) for a given location name."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "location": {
+                "type": "string",
+                "description": "The location name.",
+            },
+        },
+        "required": ["location"],
+    }
+
+
+    async def run(self):
+        """Get location information for a given location."""
+        if not (location := self.kwargs.get("location")):
+            raise Exception('No location provided.')
+
+        geolocator = Nominatim(user_agent=self.bot.USER_AGENT)
+
+        location = geolocator.geocode(location)
+
+        if location:
+            return f"""**Location information for {location.address}**
+Latitude: {location.latitude}
+Longitude: {location.longitude}
+"""
+
+        raise Exception('Could not find location data for that location.')

src/gptbot/tools/imagedescription.py

@@ -0,0 +1,15 @@
+from .base import BaseTool
+
+class Imagedescription(BaseTool):
+    DESCRIPTION = "Describe the content of the images in the conversation."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+        },
+    }
+
+    async def run(self):
+        """Describe images in the conversation."""
+        image_api = self.bot.image_api
+
+        return (await image_api.describe_images(self.messages, self.user))[0]

src/gptbot/tools/imagine.py

@@ -0,0 +1,34 @@
+from .base import BaseTool, StopProcessing
+
+class Imagine(BaseTool):
+    DESCRIPTION = "Use generative AI to create images from text prompts."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "prompt": {
+                "type": "string",
+                "description": "The prompt to use.",
+            },
+            "orientation": {
+                "type": "string",
+                "description": "The orientation of the image.",
+                "enum": ["square", "landscape", "portrait"],
+                "default": "square",
+            },
+        },
+        "required": ["prompt"],
+    }
+
+    async def run(self):
+        """Use generative AI to create images from text prompts."""
+        if not (prompt := self.kwargs.get("prompt")):
+            raise Exception('No prompt provided.')
+
+        api = self.bot.image_api
+        orientation = self.kwargs.get("orientation", "square")
+        images, tokens = await api.generate_image(prompt, self.room, orientation=orientation)
+
+        for image in images:
+            await self.bot.send_image(self.room, image, prompt)
+
+        raise StopProcessing()

src/gptbot/tools/newroom.py

@@ -0,0 +1,57 @@
+from .base import BaseTool, StopProcessing
+
+from nio import RoomCreateError, RoomInviteError
+
+from contextlib import closing
+
+class Newroom(BaseTool):
+    DESCRIPTION = "Create a new Matrix room"
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "name": {
+                "type": "string",
+                "description": "The name of the room to create.",
+                "default": "GPTBot"
+            }
+        },   
+    }
+
+    async def run(self):
+        """Create a new Matrix room"""
+        name = self.kwargs.get("name", "GPTBot")
+
+        self.bot.logger.log("Creating new room...")
+        new_room = await self.bot.matrix_client.room_create(name=name)
+
+        if isinstance(new_room, RoomCreateError):
+            self.bot.logger.log(f"Failed to create room: {new_room.message}")
+            raise
+
+        self.bot.logger.log(f"Inviting {self.user} to new room...")
+        invite = await self.bot.matrix_client.room_invite(new_room.room_id, self.user)
+
+        if isinstance(invite, RoomInviteError):
+            self.bot.logger.log(f"Failed to invite user: {invite.message}")
+            raise
+
+        await self.bot.send_message(new_room.room_id, "Welcome to your new room! What can I do for you?")
+
+        with closing(self.bot.database.cursor()) as cursor:
+            cursor.execute(
+                "SELECT space_id FROM user_spaces WHERE user_id = ? AND active = TRUE", (self.user,))
+            space = cursor.fetchone()
+
+        if space:
+            self.bot.logger.log(f"Adding new room to space {space[0]}...")
+            await self.bot.add_rooms_to_space(space[0], [new_room.room_id])
+
+        if self.bot.logo_uri:
+            await self.bot.matrix_client.room_put_state(new_room, "m.room.avatar", {
+                "url": self.bot.logo_uri
+            }, "")
+
+        await self.bot.matrix_client.room_put_state(
+            new_room.room_id, "m.room.power_levels", {"users": {self.user: 100, self.bot.matrix_client.user_id: 100}})
+
+        raise StopProcessing("Created new Matrix room with ID " + new_room.room_id + " and invited user.")

src/gptbot/tools/weather.py

@@ -0,0 +1,58 @@
+import aiohttp
+
+from datetime import datetime
+
+from .base import BaseTool
+
+class Weather(BaseTool):
+    DESCRIPTION = "Get weather information for a given location."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "latitude": {
+                "type": "string",
+                "description": "The latitude of the location.",
+            },
+            "longitude": {
+                "type": "string",
+                "description": "The longitude of the location.",
+            },
+            "name": {
+                "type": "string",
+                "description": "A location name to include in the report. This is optional, latitude and longitude are always required."
+            }
+        },
+        "required": ["latitude", "longitude"],
+    }
+
+    async def run(self):
+        """Get weather information for a given location."""
+        if not (latitude := self.kwargs.get("latitude")) or not (longitude := self.kwargs.get("longitude")):
+            raise Exception('No location provided.')
+
+        name = self.kwargs.get("name")
+
+        weather_api_key = self.bot.config.get("OpenWeatherMap", "APIKey")
+
+        if not weather_api_key:
+            raise Exception('Weather API key not found.')
+
+        url = f'https://api.openweathermap.org/data/3.0/onecall?lat={latitude}&lon={longitude}&appid={weather_api_key}&units=metric'
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    return f"""**Weather report{f" for {name}" if name else ""}**
+Current: {data['current']['temp']}°C, {data['current']['weather'][0]['description']}
+Feels like: {data['current']['feels_like']}°C
+Humidity: {data['current']['humidity']}%
+Wind: {data['current']['wind_speed']}m/s
+Sunrise: {datetime.fromtimestamp(data['current']['sunrise']).strftime('%H:%M')}
+Sunset: {datetime.fromtimestamp(data['current']['sunset']).strftime('%H:%M')}
+
+Today: {data['daily'][0]['temp']['day']}°C, {data['daily'][0]['weather'][0]['description']}, {data['daily'][0]['summary']}
+Tomorrow: {data['daily'][1]['temp']['day']}°C, {data['daily'][1]['weather'][0]['description']}, {data['daily'][1]['summary']}
+"""
+                else:
+                    raise Exception(f'Could not connect to weather API: {response.status} {response.reason}')

src/gptbot/tools/webrequest.py

@@ -0,0 +1,59 @@
+from .base import BaseTool
+
+import aiohttp
+
+from bs4 import BeautifulSoup
+
+import re
+
+class Webrequest(BaseTool):
+    DESCRIPTION = "Browse an external website by URL."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "description": "The URL to request.",
+            },
+        },
+        "required": ["url"],
+    }
+
+    async def html_to_text(self, html):
+        # Parse the HTML content of the response
+        soup = BeautifulSoup(html, 'html.parser')
+
+        # Format the links within the text
+        for link in soup.find_all('a'):
+            link_text = link.get_text()
+            link_href = link.get('href')
+            new_link_text = f"{link_text} ({link_href})"
+            link.replace_with(new_link_text)
+
+        # Extract the plain text content of the website
+        plain_text_content = soup.get_text()
+
+        # Remove extra whitespace
+        plain_text_content = re.sub('\s+', ' ', plain_text_content).strip()
+
+        # Return the formatted text content of the website
+        return plain_text_content
+
+    async def run(self):
+        """Make a web request to a given URL."""
+        if not (url := self.kwargs.get("url")):
+            raise Exception('No URL provided.')
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    data = await response.text()
+
+                    output = await self.html_to_text(data)
+
+                    return f"""**Web request**
+URL: {url}
+Status: {response.status} {response.reason}
+
+{output}
+"""

src/gptbot/tools/websearch.py

@@ -0,0 +1,37 @@
+from .base import BaseTool
+
+import aiohttp
+
+from urllib.parse import quote_plus
+
+class Websearch(BaseTool):
+    DESCRIPTION = "Search the web for a given query."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "The query to search for.",
+            },
+        },
+        "required": ["query"],
+    }
+
+    async def run(self):
+        """Search the web for a given query."""
+        if not (query := self.kwargs.get("query")):
+            raise Exception('No query provided.')
+
+        query = quote_plus(query)
+
+        url = f'https://librey.private.coffee/api.php?q={query}'
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    response_text = "**Search results for {query}**"
+                    for result in data:
+                        response_text += f"\n{result['title']}\n{result['url']}\n{result['description']}\n"
+                    
+                    return response_text

src/gptbot/tools/wikipedia.py

@@ -0,0 +1,79 @@
+from .base import BaseTool
+
+from urllib.parse import urlencode
+
+import aiohttp
+
+class Wikipedia(BaseTool):
+    DESCRIPTION = "Get information from Wikipedia."
+    PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "The query to search for.",
+            },
+            "language": {
+                "type": "string",
+                "description": "The language to search in.",
+                "default": "en",
+            },
+            "extract": {
+                "type": "string",
+                "description": "What information to extract from the page. If not provided, the full page will be returned."
+            },
+            "summarize": {
+                "type": "boolean",
+                "description": "Whether to summarize the page or not.",
+                "default": False,
+            }
+        },
+        "required": ["query"],
+    }
+
+    async def run(self):
+        """Get information from Wikipedia."""
+        if not (query := self.kwargs.get("query")):
+            raise Exception('No query provided.')
+
+        language = self.kwargs.get("language", "en")
+        extract = self.kwargs.get("extract")
+        summarize = self.kwargs.get("summarize", False)
+
+        args = {
+            "action": "query",
+            "format": "json",
+            "titles": query,
+        }
+
+        args["prop"] = "revisions"
+        args["rvprop"] = "content"
+
+        url = f'https://{language}.wikipedia.org/w/api.php?{urlencode(args)}'
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    data = await response.json()
+
+                    try:
+                        pages = data['query']['pages']
+                        page = list(pages.values())[0]
+                        content = page['revisions'][0]['*']
+                    except KeyError:
+                        raise Exception(f'No results for {query} found in Wikipedia.')
+
+                    if extract:
+                        chat_messages = [{"role": "system", "content": f"Extract the following from the provided content: {extract}"}]
+                        chat_messages.append({"role": "user", "content": content})
+                        content, _ = await self.bot.chat_api.generate_chat_response(chat_messages, room=self.room, user=self.user, allow_override=False, use_tools=False)
+
+                    if summarize:
+                        chat_messages = [{"role": "system", "content": "Summarize the following content:"}]
+                        chat_messages.append({"role": "user", "content": content})
+                        content, _ = await self.bot.chat_api.generate_chat_response(chat_messages, room=self.room, user=self.user, allow_override=False, use_tools=False)
+
+                    return f"**Wikipedia: {page['title']}**\n{content}"
+
+                else:
+                    raise Exception(f'Could not connect to Wikipedia API: {response.status} {response.reason}')