From 2c991a42a404bff7bbcd622e250ce00d7d0971e9 Mon Sep 17 00:00:00 2001 From: Kumi Date: Sat, 18 May 2024 11:56:47 +0200 Subject: [PATCH] feat: fork for Synapse metrics exporter Refactored the project from a PostgreSQL connection metrics exporter to a Synapse metrics exporter. This change addresses the need for more specific metrics related to Synapse Matrix servers, instead of the general PostgreSQL connection metrics. Updated the project name, relevant variable names, and configurations to reflect the Synapse focus. Now, the exporter provides metrics such as local users by state, type, and moderation status, total devices, rooms, events, and federation destinations. This update shifts the project's direction to support Synapse server administrators by offering detailed insights into their server's usage and performance. No existing functionality was removed; instead, the project's aim was realigned to meet the more specialized requirements of Synapse metrics reporting. By focusing on Synapse, the exporter can offer valuable data that was not previously available through the general PostgreSQL connection metrics, potentially contributing to improved server management and user experience on the Synapse platform. --- CHANGELOG.md | 11 +- LICENSE | 2 +- README.md | 44 +-- contrib/postgres-connection-exporter.service | 13 - contrib/synapse-prometheus-exporter.service | 13 + pyproject.toml | 16 +- src/postgres_connection_exporter/__main__.py | 223 --------------- .../__init__.py | 0 src/synapse_prometheus_exporter/__main__.py | 259 ++++++++++++++++++ .../config.dist.yaml | 8 +- 10 files changed, 312 insertions(+), 277 deletions(-) delete mode 100644 contrib/postgres-connection-exporter.service create mode 100644 contrib/synapse-prometheus-exporter.service delete mode 100644 src/postgres_connection_exporter/__main__.py rename src/{postgres_connection_exporter => synapse_prometheus_exporter}/__init__.py (100%) create mode 100644 src/synapse_prometheus_exporter/__main__.py rename src/{postgres_connection_exporter => synapse_prometheus_exporter}/config.dist.yaml (68%) diff --git a/CHANGELOG.md b/CHANGELOG.md index eda59e9..a138261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,7 @@ # Changelog -## v0.0.2 - -### Added - -- Add `--create-config` flag to create a default configuration file -- Add `--config` flag to specify a configuration file -- Add `--host` and `--port` flags to specify the address to listen on -- Make listening address configurable in the `config.yaml` file - ## v0.0.1 ### Added -- Initial release +- Initial release, forked from [postgres-connection-exporter](https://git.private.coffee/kumi/postgres-connection-exporter) \ No newline at end of file diff --git a/LICENSE b/LICENSE index 32bb202..4bcbd05 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2024 Kumi Mitterer +Copyright (c) 2024 Kumi Mitterer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5d6e740..53d2bdb 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,27 @@ -# PostgreSQL Connection Exporter for Prometheus +# Synapse Exporter for Prometheus -This is a simple server that exports PostgreSQL connection metrics in a format that can be scraped by Prometheus. +This is an exporter for Prometheus that collects metrics from a Synapse PostgreSQL database. SQLite is not supported. -It outputs the following metrics: +It is designed to be used with the Synapse Matrix server and provides some additional metrics that are not exported by the internal Prometheus exporter. -- The number of connections per database -- The number of connections per user -- The number of connections per client address -- The number of connections per state +## Metrics + +The exporter currently provides the following metrics: + +- `synapse_local_users_state`: Number of local users by state (active, disabled) +- `synapse_local_users_type`: Number of local users by type (guest, user, appservice) +- `synapse_local_users_moderation`: Number of local users by moderation status (active, shadow_banned, locked) +- `synapse_total_devices`: Total number of devices +- `synapse_total_rooms`: Total number of rooms +- `synapse_total_events`: Total number of events +- `synapse_federation_destinations`: Number of federation destinations (i.e. federated servers) ## Installation You can install the exporter from PyPI. Within a virtual environment, run: ```bash -pip install postgres-connection-exporter +pip install synapse-prometheus-exporter ``` ## Configuration @@ -22,39 +29,38 @@ pip install postgres-connection-exporter The exporter is configured using a `config.yaml`. You can create a default configuration file in the current working directory with: ```bash -postgres-connection-exporter --create-config +synapse-prometheus-exporter --create-config ``` Now, edit the `config.yaml` file to match your PostgreSQL connection settings. Here is an example configuration: ```yaml hosts: - host: localhost - port: 5432 - user: postgres - password: postgres + - host: localhost + port: 5432 + user: postgres + password: postgres + database: synapse ``` -The user must have the `pg_monitor` role to access the `pg_stat_activity` view. - ## Usage After you have created your `config.yaml`, you can start the exporter with: ```bash -postgres-connection-exporter +synapse-prometheus-exporter ``` -By default, the exporter listens on `localhost:8989`. You can change the address in the `config.yaml` file, or using the `--host` and `--port` flags: +By default, the exporter listens on `localhost:8999`. You can change the address in the `config.yaml` file, or using the `--host` and `--port` flags: ```bash -postgres-connection-exporter --host 0.0.0.0 --port 9898 +synapse-prometheus-exporter --host 0.0.0.0 --port 9899 ``` You can also specify a different configuration file with the `--config` flag: ```bash -postgres-connection-exporter --config /path/to/config.yaml +synapse-prometheus-exporter --config /path/to/config.yaml ``` ## License diff --git a/contrib/postgres-connection-exporter.service b/contrib/postgres-connection-exporter.service deleted file mode 100644 index 733b727..0000000 --- a/contrib/postgres-connection-exporter.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Postgres Connection Exporter -After=network.target - -[Service] -User=postgres-connection-exporter -Group=postgres-connection-exporter -ExecStart=/srv/postgres-connection-exporter/bin/postgres-connection-exporter -Restart=always -WorkingDirectory=/srv/postgres-connection-exporter - -[Install] -WantedBy=multi-user.target diff --git a/contrib/synapse-prometheus-exporter.service b/contrib/synapse-prometheus-exporter.service new file mode 100644 index 0000000..d21f37d --- /dev/null +++ b/contrib/synapse-prometheus-exporter.service @@ -0,0 +1,13 @@ +[Unit] +Description=Synapse Prometheus Exporter +After=network.target + +[Service] +User=synapse-prometheus-exporter +Group=synapse-prometheus-exporter +ExecStart=/srv/synapse-prometheus-exporter/bin/synapse-prometheus-exporter +Restart=always +WorkingDirectory=/srv/synapse-prometheus-exporter + +[Install] +WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml index ebf60f1..4f72df7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,12 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "postgres-connection-exporter" -version = "0.0.2" +name = "synapse-prometheus-exporter" +version = "0.0.1" authors = [ - { name="Kumi Mitterer", email="postgres-connection-exporter@kumi.email" }, + { name="Kumi Mitterer", email="synapse-prometheus-exporter@kumi.email" }, ] -description = "A Prometheus exporter for PostgreSQL connection metrics" +description = "A Prometheus exporter for Synapse metrics" readme = "README.md" license = { file="LICENSE" } requires-python = ">=3.10" @@ -24,9 +24,9 @@ dependencies = [ ] [project.scripts] -postgres-connection-exporter = "postgres_connection_exporter.__main__:main" +synapse-prometheus-exporter = "synapse_prometheus_exporter.__main__:main" [project.urls] -"Homepage" = "https://git.private.coffee/kumi/postgres-connection-exporter" -"Bug Tracker" = "https://git.private.coffee/kumi/postgres-connection-exporter/issues" -"Source Code" = "https://git.private.coffee/kumi/postgres-connection-exporter" \ No newline at end of file +"Homepage" = "https://git.private.coffee/kumi/synapse-prometheus-exporter" +"Bug Tracker" = "https://git.private.coffee/kumi/synapse-prometheus-exporter/issues" +"Source Code" = "https://git.private.coffee/kumi/synapse-prometheus-exporter" \ No newline at end of file diff --git a/src/postgres_connection_exporter/__main__.py b/src/postgres_connection_exporter/__main__.py deleted file mode 100644 index 557104e..0000000 --- a/src/postgres_connection_exporter/__main__.py +++ /dev/null @@ -1,223 +0,0 @@ -import yaml -from prometheus_client import start_http_server, Gauge -import psycopg2 -import time -import argparse -import pathlib -import shutil -import sys - -CONNECTIONS_PER_DB = Gauge( - "db_connections_per_database", - "Number of current connections per database", - ["database", "host"], -) -CONNECTIONS_PER_USER = Gauge( - "db_connections_per_user", - "Number of current connections per user", - ["user", "host"], -) -CONNECTIONS_BY_STATE = Gauge( - "db_connections_by_state", - "Number of current connections by state", - ["state", "host"], -) -CONNECTIONS_PER_SOURCE = Gauge( - "db_connections_per_source", - "Number of current connections per source", - ["source", "host"], -) - - -def load_config(config_file="config.yaml"): - with open(config_file, "r") as f: - return yaml.safe_load(f) - - -def get_all_databases(db_config): - try: - conn = psycopg2.connect( - dbname="postgres", - user=db_config["user"], - password=db_config["password"], - host=db_config["host"], - port=db_config.get("port", 5432), - ) - cursor = conn.cursor() - - cursor.execute("SELECT datname FROM pg_database WHERE datistemplate = false;") - databases = cursor.fetchall() - - cursor.close() - conn.close() - - return [db[0] for db in databases] - - except Exception as e: - print(f"Error retrieving list of databases: {e}") - return [] - - -def get_db_connections(db_config): - try: - host_identifier = db_config.get("name") or db_config["host"] - - # Connect to the PostgreSQL database - conn = psycopg2.connect( - dbname="postgres", - user=db_config["user"], - password=db_config["password"], - host=db_config["host"], - port=db_config["port"], - ) - cursor = conn.cursor() - - # Query to get the number of connections per database - cursor.execute( - """ - SELECT datname, COUNT(*) - FROM pg_stat_activity - GROUP BY datname; - """ - ) - db_connections = cursor.fetchall() - for db, count in db_connections: - CONNECTIONS_PER_DB.labels(database=db, host=host_identifier).set(count) - - # Query to get the number of connections per user - cursor.execute( - """ - SELECT usename, COUNT(*) - FROM pg_stat_activity - GROUP BY usename; - """ - ) - user_connections = cursor.fetchall() - for user, count in user_connections: - CONNECTIONS_PER_USER.labels(user=user, host=host_identifier).set(count) - - # Query to get the number of connections by state - cursor.execute( - """ - SELECT state, COUNT(*) - FROM pg_stat_activity - GROUP BY state; - """ - ) - state_connections = cursor.fetchall() - for state, count in state_connections: - CONNECTIONS_BY_STATE.labels(state=state, host=host_identifier).set(count) - - # Query to get the number of connections per source - cursor.execute( - """ - SELECT client_addr, COUNT(*) - FROM pg_stat_activity - GROUP BY client_addr; - """ - ) - source_connections = cursor.fetchall() - for source, count in source_connections: - CONNECTIONS_PER_SOURCE.labels(source=source, host=host_identifier).set( - count - ) - - cursor.close() - conn.close() - except Exception as e: - print(f"Error retrieving data from {db_config['host']}: {e}") - - -def main(): - parser = argparse.ArgumentParser(description="PostgreSQL connection exporter") - parser.add_argument( - "--config", "-c", help="Path to the configuration file", default="config.yaml" - ) - parser.add_argument( - "--create", "-C", help="Create a new configuration file", action="store_true" - ) - parser.add_argument( - "--port", - "-p", - help="Port for the exporter to listen on (default: 8989, or the port specified in the configuration file)", - type=int, - ) - parser.add_argument( - "--host", - help="Host for the exporter to listen on (default: localhost, or the host specified in the configuration file)", - ) - args = parser.parse_args() - - if args.create: - config_file = pathlib.Path(args.config) - if config_file.exists(): - print("Configuration file already exists.") - sys.exit(1) - - template = pathlib.Path(__file__).parent / "config.dist.yaml" - try: - shutil.copy(template, config_file) - print(f"Configuration file created at {config_file}") - sys.exit(0) - except Exception as e: - print(f"Error creating configuration file: {e}") - sys.exit(1) - - config = load_config(args.config) - - if not ("hosts" in config and config["hosts"]): - print("No database hosts specified in the configuration file.") - sys.exit(1) - - databases_to_query = [] - - for host in config["hosts"]: - if not all(key in host for key in ["user", "password", "host", "port"]): - print("Database configuration is missing required fields.") - exit(1) - - db_config = { - "name": host.get("name"), - "user": host["user"], - "password": host["password"], - "host": host["host"], - "port": host["port"], - } - - databases_to_query.append(db_config) - - if not databases_to_query: - print("No databases to query.") - exit(1) - - exporter_port = ( - args.port - if args.port - else ( - config["exporter"]["port"] - if "exporter" in config and "port" in config["exporter"] - else 8989 - ) - ) - - exporter_host = ( - args.host - if args.host - else ( - config["exporter"]["host"] - if "exporter" in config and "host" in config["exporter"] - else "localhost" - ) - ) - - start_http_server(exporter_port, exporter_host) - print(f"Prometheus exporter started on {exporter_host}:{exporter_port}") - - while True: - for db in databases_to_query: - get_db_connections(db) - time.sleep(15) - - -if __name__ == "__main__": - main() diff --git a/src/postgres_connection_exporter/__init__.py b/src/synapse_prometheus_exporter/__init__.py similarity index 100% rename from src/postgres_connection_exporter/__init__.py rename to src/synapse_prometheus_exporter/__init__.py diff --git a/src/synapse_prometheus_exporter/__main__.py b/src/synapse_prometheus_exporter/__main__.py new file mode 100644 index 0000000..7a58830 --- /dev/null +++ b/src/synapse_prometheus_exporter/__main__.py @@ -0,0 +1,259 @@ +import yaml +from prometheus_client import start_http_server, Gauge +import psycopg2 +import time +import argparse +import pathlib +import shutil +import sys + +# Gauge for local users by state (active, disabled) +LOCAL_USERS_STATE = Gauge( + "synapse_local_users_state", "Number of local users by state", ["state", "host"] +) + +# Gauge for local users by type (guest, user, appservice) +LOCAL_USERS_TYPE = Gauge( + "synapse_local_users_type", "Number of local users by type", ["type", "host"] +) + +# Gauge for local users by moderation status (active, shadow_banned, locked +LOCAL_USERS_MODERATION = Gauge( + "synapse_local_users_moderation", + "Number of local users by moderation status", + ["status", "host"], +) + +# Gauge for total number of devices, rooms, and events +TOTAL_DEVICES = Gauge( + "synapse_total_devices", "Total number of devices in the database", ["host"] +) +TOTAL_ROOMS = Gauge( + "synapse_total_rooms", "Total number of rooms in the database", ["host"] +) +TOTAL_EVENTS = Gauge( + "synapse_total_events", "Total number of events in the database", ["host"] +) + +# Gauge for known remote servers +FEDERATION_DESTINATIONS = Gauge( + "synapse_federation_destinations", + "Number of federation destinations known to the server", + ["host"], +) + + +def load_config(config_file="config.yaml"): + with open(config_file, "r") as f: + return yaml.safe_load(f) + + +def get_synapse_stats(db_config): + try: + print(f"Retrieving data from {db_config['host']}") + host_identifier = db_config.get("name") or db_config["host"] + + # Connect to the PostgreSQL database + conn = psycopg2.connect( + dbname=db_config["database"], + user=db_config["user"], + password=db_config["password"], + host=db_config["host"], + port=db_config["port"], + ) + cursor = conn.cursor() + + # Get the number of local users by state + cursor.execute( + """ + SELECT is_guest, appservice_id, approved, deactivated, shadow_banned, locked FROM users + """ + ) + + local_users = cursor.fetchall() + local_users_state = { + "active": sum([1 for user in local_users if user[3] == 0]), + "disabled": sum([1 for user in local_users if user[3] == 1]), + } + local_users_type = { + "guest": sum([1 for user in local_users if user[0] == 1]), + "user": sum([1 for user in local_users if user[0] == 0 and not user[1]]), + "appservice": sum([1 for user in local_users if user[1]]), + } + local_users_moderation = { + "active": sum([1 for user in local_users if user[4] is False and user[5] is False]), + "shadow_banned": sum([1 for user in local_users if user[4] is True]), + "locked": sum([1 for user in local_users if user[5] is True]), + } + + + LOCAL_USERS_STATE.labels(state="active", host=host_identifier).set( + local_users_state["active"] + ) + LOCAL_USERS_STATE.labels(state="disabled", host=host_identifier).set( + local_users_state["disabled"] + ) + + LOCAL_USERS_TYPE.labels(type="guest", host=host_identifier).set( + local_users_type["guest"] + ) + LOCAL_USERS_TYPE.labels(type="user", host=host_identifier).set( + local_users_type["user"] + ) + LOCAL_USERS_TYPE.labels(type="appservice", host=host_identifier).set( + local_users_type["appservice"] + ) + + LOCAL_USERS_MODERATION.labels(status="active", host=host_identifier).set( + local_users_moderation["active"] + ) + LOCAL_USERS_MODERATION.labels(status="shadow_banned", host=host_identifier).set( + local_users_moderation["shadow_banned"] + ) + LOCAL_USERS_MODERATION.labels(status="locked", host=host_identifier).set( + local_users_moderation["locked"] + ) + + # Get the total number of devices, rooms, and events + cursor.execute( + """ + SELECT COUNT(*) FROM devices + """ + ) + + total_devices = cursor.fetchone()[0] + TOTAL_DEVICES.labels(host=host_identifier).set(total_devices) + + cursor.execute( + """ + SELECT COUNT(*) FROM rooms + """ + ) + + total_rooms = cursor.fetchone()[0] + TOTAL_ROOMS.labels(host=host_identifier).set(total_rooms) + + cursor.execute( + """ + SELECT COUNT(*) FROM events + """ + ) + + total_events = cursor.fetchone()[0] + TOTAL_EVENTS.labels(host=host_identifier).set(total_events) + + # Get the number of known federation destinations + cursor.execute( + """ + SELECT COUNT(*) FROM destinations + """ + ) + + federation_destinations = cursor.fetchone()[0] + FEDERATION_DESTINATIONS.labels(host=host_identifier).set( + federation_destinations + ) + + cursor.close() + conn.close() + print(f"Data retrieved from {db_config['host']}") + except Exception as e: + print(f"Error retrieving data from {db_config['host']}: {e}") + + +def main(): + parser = argparse.ArgumentParser(description="PostgreSQL connection exporter") + parser.add_argument( + "--config", "-c", help="Path to the configuration file", default="config.yaml" + ) + parser.add_argument( + "--create", "-C", help="Create a new configuration file", action="store_true" + ) + parser.add_argument( + "--port", + "-p", + help="Port for the exporter to listen on (default: 8999, or the port specified in the configuration file)", + type=int, + ) + parser.add_argument( + "--host", + help="Host for the exporter to listen on (default: localhost, or the host specified in the configuration file)", + ) + args = parser.parse_args() + + if args.create: + config_file = pathlib.Path(args.config) + if config_file.exists(): + print("Configuration file already exists.") + sys.exit(1) + + template = pathlib.Path(__file__).parent / "config.dist.yaml" + try: + shutil.copy(template, config_file) + print(f"Configuration file created at {config_file}") + sys.exit(0) + except Exception as e: + print(f"Error creating configuration file: {e}") + sys.exit(1) + + config = load_config(args.config) + + if not ("hosts" in config and config["hosts"]): + print("No database hosts specified in the configuration file.") + sys.exit(1) + + databases_to_query = [] + + for host in config["hosts"]: + if not all( + key in host for key in ["user", "password", "host", "port", "database"] + ): + print("Database configuration is missing required fields.") + exit(1) + + db_config = { + "name": host.get("name"), + "user": host["user"], + "password": host["password"], + "host": host["host"], + "port": host["port"], + "database": host["database"], + } + + databases_to_query.append(db_config) + + if not databases_to_query: + print("No databases to query.") + exit(1) + + exporter_port = ( + args.port + if args.port + else ( + config["exporter"]["port"] + if "exporter" in config and "port" in config["exporter"] + else 8999 + ) + ) + + exporter_host = ( + args.host + if args.host + else ( + config["exporter"]["host"] + if "exporter" in config and "host" in config["exporter"] + else "localhost" + ) + ) + + start_http_server(exporter_port, exporter_host) + print(f"Prometheus exporter started on {exporter_host}:{exporter_port}") + + while True: + for db in databases_to_query: + get_synapse_stats(db) + time.sleep(15) + + +if __name__ == "__main__": + main() diff --git a/src/postgres_connection_exporter/config.dist.yaml b/src/synapse_prometheus_exporter/config.dist.yaml similarity index 68% rename from src/postgres_connection_exporter/config.dist.yaml rename to src/synapse_prometheus_exporter/config.dist.yaml index 631295f..f8b6041 100644 --- a/src/postgres_connection_exporter/config.dist.yaml +++ b/src/synapse_prometheus_exporter/config.dist.yaml @@ -1,16 +1,18 @@ hosts: # List of database hosts - # Each host must have a user, password, host, and optionally port + # Each host must have a user, password, host, database, and optionally port (default is 5432) # A name can be provided to identify the host in the metrics instead of the host address - - name: "myserver" + - name: "mymatrix" user: "user1" password: "password1" host: "host1" port: "5432" + database: "synapse" - user: "user2" password: "password2" host: "host2" port: "5432" + database: "synapse" # As no name is provided, the host address "host2" will be used to identify the host in the metrics exporter: host: "localhost" # Network address on which the exporter will listen - port: 8989 # Port on which the exporter will listen + port: 8999 # Port on which the exporter will listen