diff --git a/CHANGELOG.md b/CHANGELOG.md index eda59e9..a138261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,7 @@ # Changelog -## v0.0.2 - -### Added - -- Add `--create-config` flag to create a default configuration file -- Add `--config` flag to specify a configuration file -- Add `--host` and `--port` flags to specify the address to listen on -- Make listening address configurable in the `config.yaml` file - ## v0.0.1 ### Added -- Initial release +- Initial release, forked from [postgres-connection-exporter](https://git.private.coffee/kumi/postgres-connection-exporter) \ No newline at end of file diff --git a/LICENSE b/LICENSE index 32bb202..4bcbd05 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2024 Kumi Mitterer +Copyright (c) 2024 Kumi Mitterer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5d6e740..53d2bdb 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,27 @@ -# PostgreSQL Connection Exporter for Prometheus +# Synapse Exporter for Prometheus -This is a simple server that exports PostgreSQL connection metrics in a format that can be scraped by Prometheus. +This is an exporter for Prometheus that collects metrics from a Synapse PostgreSQL database. SQLite is not supported. -It outputs the following metrics: +It is designed to be used with the Synapse Matrix server and provides some additional metrics that are not exported by the internal Prometheus exporter. -- The number of connections per database -- The number of connections per user -- The number of connections per client address -- The number of connections per state +## Metrics + +The exporter currently provides the following metrics: + +- `synapse_local_users_state`: Number of local users by state (active, disabled) +- `synapse_local_users_type`: Number of local users by type (guest, user, appservice) +- `synapse_local_users_moderation`: Number of local users by moderation status (active, shadow_banned, locked) +- `synapse_total_devices`: Total number of devices +- `synapse_total_rooms`: Total number of rooms +- `synapse_total_events`: Total number of events +- `synapse_federation_destinations`: Number of federation destinations (i.e. federated servers) ## Installation You can install the exporter from PyPI. Within a virtual environment, run: ```bash -pip install postgres-connection-exporter +pip install synapse-prometheus-exporter ``` ## Configuration @@ -22,39 +29,38 @@ pip install postgres-connection-exporter The exporter is configured using a `config.yaml`. You can create a default configuration file in the current working directory with: ```bash -postgres-connection-exporter --create-config +synapse-prometheus-exporter --create-config ``` Now, edit the `config.yaml` file to match your PostgreSQL connection settings. Here is an example configuration: ```yaml hosts: - host: localhost - port: 5432 - user: postgres - password: postgres + - host: localhost + port: 5432 + user: postgres + password: postgres + database: synapse ``` -The user must have the `pg_monitor` role to access the `pg_stat_activity` view. - ## Usage After you have created your `config.yaml`, you can start the exporter with: ```bash -postgres-connection-exporter +synapse-prometheus-exporter ``` -By default, the exporter listens on `localhost:8989`. You can change the address in the `config.yaml` file, or using the `--host` and `--port` flags: +By default, the exporter listens on `localhost:8999`. You can change the address in the `config.yaml` file, or using the `--host` and `--port` flags: ```bash -postgres-connection-exporter --host 0.0.0.0 --port 9898 +synapse-prometheus-exporter --host 0.0.0.0 --port 9899 ``` You can also specify a different configuration file with the `--config` flag: ```bash -postgres-connection-exporter --config /path/to/config.yaml +synapse-prometheus-exporter --config /path/to/config.yaml ``` ## License diff --git a/contrib/postgres-connection-exporter.service b/contrib/postgres-connection-exporter.service deleted file mode 100644 index 733b727..0000000 --- a/contrib/postgres-connection-exporter.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Postgres Connection Exporter -After=network.target - -[Service] -User=postgres-connection-exporter -Group=postgres-connection-exporter -ExecStart=/srv/postgres-connection-exporter/bin/postgres-connection-exporter -Restart=always -WorkingDirectory=/srv/postgres-connection-exporter - -[Install] -WantedBy=multi-user.target diff --git a/contrib/synapse-prometheus-exporter.service b/contrib/synapse-prometheus-exporter.service new file mode 100644 index 0000000..d21f37d --- /dev/null +++ b/contrib/synapse-prometheus-exporter.service @@ -0,0 +1,13 @@ +[Unit] +Description=Synapse Prometheus Exporter +After=network.target + +[Service] +User=synapse-prometheus-exporter +Group=synapse-prometheus-exporter +ExecStart=/srv/synapse-prometheus-exporter/bin/synapse-prometheus-exporter +Restart=always +WorkingDirectory=/srv/synapse-prometheus-exporter + +[Install] +WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml index ebf60f1..4f72df7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,12 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "postgres-connection-exporter" -version = "0.0.2" +name = "synapse-prometheus-exporter" +version = "0.0.1" authors = [ - { name="Kumi Mitterer", email="postgres-connection-exporter@kumi.email" }, + { name="Kumi Mitterer", email="synapse-prometheus-exporter@kumi.email" }, ] -description = "A Prometheus exporter for PostgreSQL connection metrics" +description = "A Prometheus exporter for Synapse metrics" readme = "README.md" license = { file="LICENSE" } requires-python = ">=3.10" @@ -24,9 +24,9 @@ dependencies = [ ] [project.scripts] -postgres-connection-exporter = "postgres_connection_exporter.__main__:main" +synapse-prometheus-exporter = "synapse_prometheus_exporter.__main__:main" [project.urls] -"Homepage" = "https://git.private.coffee/kumi/postgres-connection-exporter" -"Bug Tracker" = "https://git.private.coffee/kumi/postgres-connection-exporter/issues" -"Source Code" = "https://git.private.coffee/kumi/postgres-connection-exporter" \ No newline at end of file +"Homepage" = "https://git.private.coffee/kumi/synapse-prometheus-exporter" +"Bug Tracker" = "https://git.private.coffee/kumi/synapse-prometheus-exporter/issues" +"Source Code" = "https://git.private.coffee/kumi/synapse-prometheus-exporter" \ No newline at end of file diff --git a/src/postgres_connection_exporter/__main__.py b/src/postgres_connection_exporter/__main__.py deleted file mode 100644 index 557104e..0000000 --- a/src/postgres_connection_exporter/__main__.py +++ /dev/null @@ -1,223 +0,0 @@ -import yaml -from prometheus_client import start_http_server, Gauge -import psycopg2 -import time -import argparse -import pathlib -import shutil -import sys - -CONNECTIONS_PER_DB = Gauge( - "db_connections_per_database", - "Number of current connections per database", - ["database", "host"], -) -CONNECTIONS_PER_USER = Gauge( - "db_connections_per_user", - "Number of current connections per user", - ["user", "host"], -) -CONNECTIONS_BY_STATE = Gauge( - "db_connections_by_state", - "Number of current connections by state", - ["state", "host"], -) -CONNECTIONS_PER_SOURCE = Gauge( - "db_connections_per_source", - "Number of current connections per source", - ["source", "host"], -) - - -def load_config(config_file="config.yaml"): - with open(config_file, "r") as f: - return yaml.safe_load(f) - - -def get_all_databases(db_config): - try: - conn = psycopg2.connect( - dbname="postgres", - user=db_config["user"], - password=db_config["password"], - host=db_config["host"], - port=db_config.get("port", 5432), - ) - cursor = conn.cursor() - - cursor.execute("SELECT datname FROM pg_database WHERE datistemplate = false;") - databases = cursor.fetchall() - - cursor.close() - conn.close() - - return [db[0] for db in databases] - - except Exception as e: - print(f"Error retrieving list of databases: {e}") - return [] - - -def get_db_connections(db_config): - try: - host_identifier = db_config.get("name") or db_config["host"] - - # Connect to the PostgreSQL database - conn = psycopg2.connect( - dbname="postgres", - user=db_config["user"], - password=db_config["password"], - host=db_config["host"], - port=db_config["port"], - ) - cursor = conn.cursor() - - # Query to get the number of connections per database - cursor.execute( - """ - SELECT datname, COUNT(*) - FROM pg_stat_activity - GROUP BY datname; - """ - ) - db_connections = cursor.fetchall() - for db, count in db_connections: - CONNECTIONS_PER_DB.labels(database=db, host=host_identifier).set(count) - - # Query to get the number of connections per user - cursor.execute( - """ - SELECT usename, COUNT(*) - FROM pg_stat_activity - GROUP BY usename; - """ - ) - user_connections = cursor.fetchall() - for user, count in user_connections: - CONNECTIONS_PER_USER.labels(user=user, host=host_identifier).set(count) - - # Query to get the number of connections by state - cursor.execute( - """ - SELECT state, COUNT(*) - FROM pg_stat_activity - GROUP BY state; - """ - ) - state_connections = cursor.fetchall() - for state, count in state_connections: - CONNECTIONS_BY_STATE.labels(state=state, host=host_identifier).set(count) - - # Query to get the number of connections per source - cursor.execute( - """ - SELECT client_addr, COUNT(*) - FROM pg_stat_activity - GROUP BY client_addr; - """ - ) - source_connections = cursor.fetchall() - for source, count in source_connections: - CONNECTIONS_PER_SOURCE.labels(source=source, host=host_identifier).set( - count - ) - - cursor.close() - conn.close() - except Exception as e: - print(f"Error retrieving data from {db_config['host']}: {e}") - - -def main(): - parser = argparse.ArgumentParser(description="PostgreSQL connection exporter") - parser.add_argument( - "--config", "-c", help="Path to the configuration file", default="config.yaml" - ) - parser.add_argument( - "--create", "-C", help="Create a new configuration file", action="store_true" - ) - parser.add_argument( - "--port", - "-p", - help="Port for the exporter to listen on (default: 8989, or the port specified in the configuration file)", - type=int, - ) - parser.add_argument( - "--host", - help="Host for the exporter to listen on (default: localhost, or the host specified in the configuration file)", - ) - args = parser.parse_args() - - if args.create: - config_file = pathlib.Path(args.config) - if config_file.exists(): - print("Configuration file already exists.") - sys.exit(1) - - template = pathlib.Path(__file__).parent / "config.dist.yaml" - try: - shutil.copy(template, config_file) - print(f"Configuration file created at {config_file}") - sys.exit(0) - except Exception as e: - print(f"Error creating configuration file: {e}") - sys.exit(1) - - config = load_config(args.config) - - if not ("hosts" in config and config["hosts"]): - print("No database hosts specified in the configuration file.") - sys.exit(1) - - databases_to_query = [] - - for host in config["hosts"]: - if not all(key in host for key in ["user", "password", "host", "port"]): - print("Database configuration is missing required fields.") - exit(1) - - db_config = { - "name": host.get("name"), - "user": host["user"], - "password": host["password"], - "host": host["host"], - "port": host["port"], - } - - databases_to_query.append(db_config) - - if not databases_to_query: - print("No databases to query.") - exit(1) - - exporter_port = ( - args.port - if args.port - else ( - config["exporter"]["port"] - if "exporter" in config and "port" in config["exporter"] - else 8989 - ) - ) - - exporter_host = ( - args.host - if args.host - else ( - config["exporter"]["host"] - if "exporter" in config and "host" in config["exporter"] - else "localhost" - ) - ) - - start_http_server(exporter_port, exporter_host) - print(f"Prometheus exporter started on {exporter_host}:{exporter_port}") - - while True: - for db in databases_to_query: - get_db_connections(db) - time.sleep(15) - - -if __name__ == "__main__": - main() diff --git a/src/postgres_connection_exporter/__init__.py b/src/synapse_prometheus_exporter/__init__.py similarity index 100% rename from src/postgres_connection_exporter/__init__.py rename to src/synapse_prometheus_exporter/__init__.py diff --git a/src/synapse_prometheus_exporter/__main__.py b/src/synapse_prometheus_exporter/__main__.py new file mode 100644 index 0000000..7a58830 --- /dev/null +++ b/src/synapse_prometheus_exporter/__main__.py @@ -0,0 +1,259 @@ +import yaml +from prometheus_client import start_http_server, Gauge +import psycopg2 +import time +import argparse +import pathlib +import shutil +import sys + +# Gauge for local users by state (active, disabled) +LOCAL_USERS_STATE = Gauge( + "synapse_local_users_state", "Number of local users by state", ["state", "host"] +) + +# Gauge for local users by type (guest, user, appservice) +LOCAL_USERS_TYPE = Gauge( + "synapse_local_users_type", "Number of local users by type", ["type", "host"] +) + +# Gauge for local users by moderation status (active, shadow_banned, locked +LOCAL_USERS_MODERATION = Gauge( + "synapse_local_users_moderation", + "Number of local users by moderation status", + ["status", "host"], +) + +# Gauge for total number of devices, rooms, and events +TOTAL_DEVICES = Gauge( + "synapse_total_devices", "Total number of devices in the database", ["host"] +) +TOTAL_ROOMS = Gauge( + "synapse_total_rooms", "Total number of rooms in the database", ["host"] +) +TOTAL_EVENTS = Gauge( + "synapse_total_events", "Total number of events in the database", ["host"] +) + +# Gauge for known remote servers +FEDERATION_DESTINATIONS = Gauge( + "synapse_federation_destinations", + "Number of federation destinations known to the server", + ["host"], +) + + +def load_config(config_file="config.yaml"): + with open(config_file, "r") as f: + return yaml.safe_load(f) + + +def get_synapse_stats(db_config): + try: + print(f"Retrieving data from {db_config['host']}") + host_identifier = db_config.get("name") or db_config["host"] + + # Connect to the PostgreSQL database + conn = psycopg2.connect( + dbname=db_config["database"], + user=db_config["user"], + password=db_config["password"], + host=db_config["host"], + port=db_config["port"], + ) + cursor = conn.cursor() + + # Get the number of local users by state + cursor.execute( + """ + SELECT is_guest, appservice_id, approved, deactivated, shadow_banned, locked FROM users + """ + ) + + local_users = cursor.fetchall() + local_users_state = { + "active": sum([1 for user in local_users if user[3] == 0]), + "disabled": sum([1 for user in local_users if user[3] == 1]), + } + local_users_type = { + "guest": sum([1 for user in local_users if user[0] == 1]), + "user": sum([1 for user in local_users if user[0] == 0 and not user[1]]), + "appservice": sum([1 for user in local_users if user[1]]), + } + local_users_moderation = { + "active": sum([1 for user in local_users if user[4] is False and user[5] is False]), + "shadow_banned": sum([1 for user in local_users if user[4] is True]), + "locked": sum([1 for user in local_users if user[5] is True]), + } + + + LOCAL_USERS_STATE.labels(state="active", host=host_identifier).set( + local_users_state["active"] + ) + LOCAL_USERS_STATE.labels(state="disabled", host=host_identifier).set( + local_users_state["disabled"] + ) + + LOCAL_USERS_TYPE.labels(type="guest", host=host_identifier).set( + local_users_type["guest"] + ) + LOCAL_USERS_TYPE.labels(type="user", host=host_identifier).set( + local_users_type["user"] + ) + LOCAL_USERS_TYPE.labels(type="appservice", host=host_identifier).set( + local_users_type["appservice"] + ) + + LOCAL_USERS_MODERATION.labels(status="active", host=host_identifier).set( + local_users_moderation["active"] + ) + LOCAL_USERS_MODERATION.labels(status="shadow_banned", host=host_identifier).set( + local_users_moderation["shadow_banned"] + ) + LOCAL_USERS_MODERATION.labels(status="locked", host=host_identifier).set( + local_users_moderation["locked"] + ) + + # Get the total number of devices, rooms, and events + cursor.execute( + """ + SELECT COUNT(*) FROM devices + """ + ) + + total_devices = cursor.fetchone()[0] + TOTAL_DEVICES.labels(host=host_identifier).set(total_devices) + + cursor.execute( + """ + SELECT COUNT(*) FROM rooms + """ + ) + + total_rooms = cursor.fetchone()[0] + TOTAL_ROOMS.labels(host=host_identifier).set(total_rooms) + + cursor.execute( + """ + SELECT COUNT(*) FROM events + """ + ) + + total_events = cursor.fetchone()[0] + TOTAL_EVENTS.labels(host=host_identifier).set(total_events) + + # Get the number of known federation destinations + cursor.execute( + """ + SELECT COUNT(*) FROM destinations + """ + ) + + federation_destinations = cursor.fetchone()[0] + FEDERATION_DESTINATIONS.labels(host=host_identifier).set( + federation_destinations + ) + + cursor.close() + conn.close() + print(f"Data retrieved from {db_config['host']}") + except Exception as e: + print(f"Error retrieving data from {db_config['host']}: {e}") + + +def main(): + parser = argparse.ArgumentParser(description="PostgreSQL connection exporter") + parser.add_argument( + "--config", "-c", help="Path to the configuration file", default="config.yaml" + ) + parser.add_argument( + "--create", "-C", help="Create a new configuration file", action="store_true" + ) + parser.add_argument( + "--port", + "-p", + help="Port for the exporter to listen on (default: 8999, or the port specified in the configuration file)", + type=int, + ) + parser.add_argument( + "--host", + help="Host for the exporter to listen on (default: localhost, or the host specified in the configuration file)", + ) + args = parser.parse_args() + + if args.create: + config_file = pathlib.Path(args.config) + if config_file.exists(): + print("Configuration file already exists.") + sys.exit(1) + + template = pathlib.Path(__file__).parent / "config.dist.yaml" + try: + shutil.copy(template, config_file) + print(f"Configuration file created at {config_file}") + sys.exit(0) + except Exception as e: + print(f"Error creating configuration file: {e}") + sys.exit(1) + + config = load_config(args.config) + + if not ("hosts" in config and config["hosts"]): + print("No database hosts specified in the configuration file.") + sys.exit(1) + + databases_to_query = [] + + for host in config["hosts"]: + if not all( + key in host for key in ["user", "password", "host", "port", "database"] + ): + print("Database configuration is missing required fields.") + exit(1) + + db_config = { + "name": host.get("name"), + "user": host["user"], + "password": host["password"], + "host": host["host"], + "port": host["port"], + "database": host["database"], + } + + databases_to_query.append(db_config) + + if not databases_to_query: + print("No databases to query.") + exit(1) + + exporter_port = ( + args.port + if args.port + else ( + config["exporter"]["port"] + if "exporter" in config and "port" in config["exporter"] + else 8999 + ) + ) + + exporter_host = ( + args.host + if args.host + else ( + config["exporter"]["host"] + if "exporter" in config and "host" in config["exporter"] + else "localhost" + ) + ) + + start_http_server(exporter_port, exporter_host) + print(f"Prometheus exporter started on {exporter_host}:{exporter_port}") + + while True: + for db in databases_to_query: + get_synapse_stats(db) + time.sleep(15) + + +if __name__ == "__main__": + main() diff --git a/src/postgres_connection_exporter/config.dist.yaml b/src/synapse_prometheus_exporter/config.dist.yaml similarity index 68% rename from src/postgres_connection_exporter/config.dist.yaml rename to src/synapse_prometheus_exporter/config.dist.yaml index 631295f..f8b6041 100644 --- a/src/postgres_connection_exporter/config.dist.yaml +++ b/src/synapse_prometheus_exporter/config.dist.yaml @@ -1,16 +1,18 @@ hosts: # List of database hosts - # Each host must have a user, password, host, and optionally port + # Each host must have a user, password, host, database, and optionally port (default is 5432) # A name can be provided to identify the host in the metrics instead of the host address - - name: "myserver" + - name: "mymatrix" user: "user1" password: "password1" host: "host1" port: "5432" + database: "synapse" - user: "user2" password: "password2" host: "host2" port: "5432" + database: "synapse" # As no name is provided, the host address "host2" will be used to identify the host in the metrics exporter: host: "localhost" # Network address on which the exporter will listen - port: 8989 # Port on which the exporter will listen + port: 8999 # Port on which the exporter will listen