Implement dendrite & pinecone support in Playwright (#11943)

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
Michael Telatynski 2023-11-27 14:40:45 +00:00 committed by GitHub
parent a3cf11ac74
commit f7aab0e042
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 589 additions and 27 deletions

View file

@ -16,6 +16,7 @@ limitations under the License.
import { test, expect } from "../../element-web-test";
import { doTokenRegistration } from "./utils";
import { isDendrite } from "../../plugins/homeserver/dendrite";
test.describe("Login", () => {
test.describe("m.login.password", () => {
@ -79,6 +80,8 @@ test.describe("Login", () => {
// tests for old-style SSO login, in which we exchange tokens with Synapse, and Synapse talks to an auth server
test.describe("SSO login", () => {
test.skip(isDendrite, "does not yet support SSO");
test.use({
startHomeserverOpts: ({ oAuthServer }, use) =>
use({

View file

@ -18,7 +18,8 @@ import { Page } from "@playwright/test";
import { test, expect } from "../../element-web-test";
import { doTokenRegistration } from "./utils";
import { Credentials } from "../../plugins/utils/homeserver";
import { Credentials } from "../../plugins/homeserver";
import { isDendrite } from "../../plugins/homeserver/dendrite";
test.describe("Soft logout", () => {
test.use({
@ -54,6 +55,8 @@ test.describe("Soft logout", () => {
});
test.describe("with SSO user", () => {
test.skip(isDendrite, "does not yet support SSO");
test.use({
user: async ({ page, homeserver }, use) => {
const user = await doTokenRegistration(page, homeserver);

View file

@ -16,7 +16,7 @@ limitations under the License.
import { Page, expect } from "@playwright/test";
import { Credentials, HomeserverInstance } from "../../plugins/utils/homeserver";
import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
/** Visit the login page, choose to log in with "OAuth test", register a new account, and redirect back to Element
*/

View file

@ -16,8 +16,11 @@ limitations under the License.
import { test, expect } from "../../element-web-test";
import { MailHogServer } from "../../plugins/mailhog";
import { isDendrite } from "../../plugins/homeserver/dendrite";
test.describe("Email Registration", async () => {
test.skip(isDendrite, "not yet wired up");
test.use({
// eslint-disable-next-line no-empty-pattern
mailhog: async ({}, use) => {

View file

@ -20,8 +20,9 @@ import _ from "lodash";
import type mailhog from "mailhog";
import type { IConfigOptions } from "../src/IConfigOptions";
import { Credentials, HomeserverInstance, StartHomeserverOpts } from "./plugins/utils/homeserver";
import { Synapse } from "./plugins/synapse";
import { Credentials, Homeserver, HomeserverInstance, StartHomeserverOpts } from "./plugins/homeserver";
import { Synapse } from "./plugins/homeserver/synapse";
import { Dendrite, Pinecone } from "./plugins/homeserver/dendrite";
import { Instance } from "./plugins/mailhog";
import { ElementAppPage } from "./pages/ElementAppPage";
import { OAuthServer } from "./plugins/oauth_server";
@ -89,7 +90,19 @@ export const test = base.extend<
opts = { template: opts };
}
const server = new Synapse(request);
let server: Homeserver;
const homeserverName = process.env["PLAYWRIGHT_HOMESERVER"];
switch (homeserverName) {
case "dendrite":
server = new Dendrite(request);
break;
case "pinecone":
server = new Pinecone(request);
break;
default:
server = new Synapse(request);
}
await use(await server.start(opts));
await server.stop();
},

View file

@ -0,0 +1,151 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import * as path from "node:path";
import * as os from "node:os";
import * as fse from "fs-extra";
import { getFreePort } from "../../utils/port";
import { Homeserver, HomeserverConfig, HomeserverInstance, StartHomeserverOpts } from "../";
import { randB64Bytes } from "../../utils/rand";
import { Synapse } from "../synapse";
import { Docker } from "../../docker";
const dockerConfigDir = "/etc/dendrite/";
const dendriteConfigFile = "dendrite.yaml";
// Surprisingly, Dendrite implements the same register user Admin API Synapse, so we can just extend it
export class Dendrite extends Synapse implements Homeserver, HomeserverInstance {
public config: HomeserverConfig & { serverId: string };
protected image = "matrixdotorg/dendrite-monolith:main";
protected entrypoint = "/usr/bin/dendrite";
/**
* Start a dendrite instance: the template must be the name of one of the templates
* in the playwright/plugins/dendritedocker/templates directory
* @param opts
*/
public async start(opts: StartHomeserverOpts): Promise<HomeserverInstance> {
const denCfg = await cfgDirFromTemplate(this.image, opts);
console.log(`Starting dendrite with config dir ${denCfg.configDir}...`);
const dendriteId = await this.docker.run({
image: this.image,
params: [
"--rm",
"-v",
`${denCfg.configDir}:` + dockerConfigDir,
"-p",
`${denCfg.port}:8008/tcp`,
"--entrypoint",
this.entrypoint,
],
containerName: `react-sdk-playwright-dendrite`,
cmd: ["--config", dockerConfigDir + dendriteConfigFile, "--really-enable-open-registration", "true", "run"],
});
console.log(`Started dendrite with id ${dendriteId} on port ${denCfg.port}.`);
// Await Dendrite healthcheck
await this.docker.exec([
"curl",
"--connect-timeout",
"30",
"--retry",
"30",
"--retry-delay",
"1",
"--retry-all-errors",
"--silent",
"http://localhost:8008/_matrix/client/versions",
]);
this.config = {
...denCfg,
serverId: dendriteId,
};
return this;
}
public async stop(): Promise<void> {
if (!this.config) throw new Error("Missing existing dendrite instance, did you call stop() before start()?");
const dendriteLogsPath = path.join("playwright", "dendritelogs", this.config.serverId);
await fse.ensureDir(dendriteLogsPath);
await this.docker.persistLogsToFile({
stdoutFile: path.join(dendriteLogsPath, "stdout.log"),
stderrFile: path.join(dendriteLogsPath, "stderr.log"),
});
await this.docker.stop();
await fse.remove(this.config.configDir);
console.log(`Stopped dendrite id ${this.config.serverId}.`);
}
}
export class Pinecone extends Dendrite {
protected image = "matrixdotorg/dendrite-demo-pinecone:main";
protected entrypoint = "/usr/bin/dendrite-demo-pinecone";
}
async function cfgDirFromTemplate(dendriteImage: string, opts: StartHomeserverOpts): Promise<HomeserverConfig> {
const template = "default"; // XXX: for now we only have one template
const templateDir = path.join(__dirname, "templates", template);
const stats = await fse.stat(templateDir);
if (!stats?.isDirectory) {
throw new Error(`No such template: ${template}`);
}
const tempDir = await fse.mkdtemp(path.join(os.tmpdir(), "react-sdk-dendritedocker-"));
// copy the contents of the template dir, omitting homeserver.yaml as we'll template that
console.log(`Copy ${templateDir} -> ${tempDir}`);
await fse.copy(templateDir, tempDir, { filter: (f) => path.basename(f) !== dendriteConfigFile });
const registrationSecret = randB64Bytes(16);
const port = await getFreePort();
const baseUrl = `http://localhost:${port}`;
// now copy homeserver.yaml, applying substitutions
console.log(`Gen ${path.join(templateDir, dendriteConfigFile)}`);
let hsYaml = await fse.readFile(path.join(templateDir, dendriteConfigFile), "utf8");
hsYaml = hsYaml.replace(/{{REGISTRATION_SECRET}}/g, registrationSecret);
await fse.writeFile(path.join(tempDir, dendriteConfigFile), hsYaml);
const docker = new Docker();
await docker.run({
image: dendriteImage,
params: ["--rm", "--entrypoint=", "-v", `${tempDir}:/mnt`],
containerName: `react-sdk-playwright-dendrite-keygen`,
cmd: ["/usr/bin/generate-keys", "-private-key", "/mnt/matrix_key.pem"],
});
return {
port,
baseUrl,
configDir: tempDir,
registrationSecret,
};
}
export function isDendrite(): boolean {
return process.env["PLAYWRIGHT_HOMESERVER"] === "dendrite" || process.env["PLAYWRIGHT_HOMESERVER"] === "pinecone";
}

View file

@ -0,0 +1,378 @@
# This is the Dendrite configuration file.
#
# The configuration is split up into sections - each Dendrite component has a
# configuration section, in addition to the "global" section which applies to
# all components.
# The version of the configuration file.
version: 2
# Global Matrix configuration. This configuration applies to all components.
global:
# The domain name of this homeserver.
server_name: localhost
# The path to the signing private key file, used to sign requests and events.
# Note that this is NOT the same private key as used for TLS! To generate a
# signing key, use "./bin/generate-keys --private-key matrix_key.pem".
private_key: matrix_key.pem
# The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
# to old signing keys that were formerly in use on this domain name. These
# keys will not be used for federation request or event signing, but will be
# provided to any other homeserver that asks when trying to verify old events.
old_private_keys:
# If the old private key file is available:
# - private_key: old_matrix_key.pem
# expired_at: 1601024554498
# If only the public key (in base64 format) and key ID are known:
# - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
# key_id: ed25519:mykeyid
# expired_at: 1601024554498
# How long a remote server can cache our server signing key before requesting it
# again. Increasing this number will reduce the number of requests made by other
# servers for our key but increases the period that a compromised key will be
# considered valid by other homeservers.
key_validity_period: 168h0m0s
# Global database connection pool, for PostgreSQL monolith deployments only. If
# this section is populated then you can omit the "database" blocks in all other
# sections. For polylith deployments, or monolith deployments using SQLite databases,
# you must configure the "database" block for each component instead.
# database:
# connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
# max_open_conns: 90
# max_idle_conns: 5
# conn_max_lifetime: -1
# Configuration for in-memory caches. Caches can often improve performance by
# keeping frequently accessed items (like events, identifiers etc.) in memory
# rather than having to read them from the database.
cache:
# The estimated maximum size for the global cache in bytes, or in terabytes,
# gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
# 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
# memory limit for the entire process. A cache that is too small may ultimately
# provide little or no benefit.
max_size_estimated: 1gb
# The maximum amount of time that a cache entry can live for in memory before
# it will be evicted and/or refreshed from the database. Lower values result in
# easier admission of new cache entries but may also increase database load in
# comparison to higher values, so adjust conservatively. Higher values may make
# it harder for new items to make it into the cache, e.g. if new rooms suddenly
# become popular.
max_age: 1h
# The server name to delegate server-server communications to, with optional port
# e.g. localhost:443
well_known_server_name: ""
# The server name to delegate client-server communications to, with optional port
# e.g. localhost:443
well_known_client_name: ""
# Lists of domains that the server will trust as identity servers to verify third
# party identifiers such as phone numbers and email addresses.
trusted_third_party_id_servers:
- matrix.org
- vector.im
# Disables federation. Dendrite will not be able to communicate with other servers
# in the Matrix federation and the federation API will not be exposed.
disable_federation: false
# Configures the handling of presence events. Inbound controls whether we receive
# presence events from other servers, outbound controls whether we send presence
# events for our local users to other servers.
presence:
enable_inbound: false
enable_outbound: false
# Configures phone-home statistics reporting. These statistics contain the server
# name, number of active users and some information on your deployment config.
# We use this information to understand how Dendrite is being used in the wild.
report_stats:
enabled: false
endpoint: https://matrix.org/report-usage-stats/push
# Server notices allows server admins to send messages to all users on the server.
server_notices:
enabled: false
# The local part, display name and avatar URL (as a mxc:// URL) for the user that
# will send the server notices. These are visible to all users on the deployment.
local_part: "_server"
display_name: "Server Alerts"
avatar_url: ""
# The room name to be used when sending server notices. This room name will
# appear in user clients.
room_name: "Server Alerts"
# Configuration for NATS JetStream
jetstream:
# A list of NATS Server addresses to connect to. If none are specified, an
# internal NATS server will be started automatically when running Dendrite in
# monolith mode. For polylith deployments, it is required to specify the address
# of at least one NATS Server node.
addresses:
# - localhost:4222
# Disable the validation of TLS certificates of NATS. This is
# not recommended in production since it may allow NATS traffic
# to be sent to an insecure endpoint.
disable_tls_validation: false
# Persistent directory to store JetStream streams in. This directory should be
# preserved across Dendrite restarts.
storage_path: ./
# The prefix to use for stream names for this homeserver - really only useful
# if you are running more than one Dendrite server on the same NATS deployment.
topic_prefix: Dendrite
# Configuration for Prometheus metric collection.
metrics:
enabled: false
basic_auth:
username: metrics
password: metrics
# Optional DNS cache. The DNS cache may reduce the load on DNS servers if there
# is no local caching resolver available for use.
dns_cache:
enabled: false
cache_size: 256
cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration
# Configuration for the Appservice API.
app_service_api:
# Disable the validation of TLS certificates of appservices. This is
# not recommended in production since it may allow appservice traffic
# to be sent to an insecure endpoint.
disable_tls_validation: false
# Appservice configuration files to load into this homeserver.
config_files:
# - /path/to/appservice_registration.yaml
# Configuration for the Client API.
client_api:
# Prevents new users from being able to register on this homeserver, except when
# using the registration shared secret below.
registration_disabled: false
# Prevents new guest accounts from being created. Guest registration is also
# disabled implicitly by setting 'registration_disabled' above.
guests_disabled: true
# If set, allows registration by anyone who knows the shared secret, regardless
# of whether registration is otherwise disabled.
registration_shared_secret: "{{REGISTRATION_SECRET}}"
# Whether to require reCAPTCHA for registration. If you have enabled registration
# then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used
# for coordinated spam attacks.
enable_registration_captcha: false
# Settings for ReCAPTCHA.
recaptcha_public_key: ""
recaptcha_private_key: ""
recaptcha_bypass_secret: ""
# To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
# recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
# recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
# recaptcha_form_field: "h-captcha-response"
# recaptcha_sitekey_class: "h-captcha"
# TURN server information that this homeserver should send to clients.
turn:
turn_user_lifetime: "5m"
turn_uris:
# - turn:turn.server.org?transport=udp
# - turn:turn.server.org?transport=tcp
turn_shared_secret: ""
# If your TURN server requires static credentials, then you will need to enter
# them here instead of supplying a shared secret. Note that these credentials
# will be visible to clients!
# turn_username: ""
# turn_password: ""
# Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
# number of "slots" have been taken by requests from a specific host. Each "slot"
# will be released after the cooloff time in milliseconds. Server administrators
# and appservice users are exempt from rate limiting by default.
rate_limiting:
enabled: true
threshold: 20
cooloff_ms: 500
exempt_user_ids:
# - "@user:domain.com"
# Configuration for the Federation API.
federation_api:
# How many times we will try to resend a failed transaction to a specific server. The
# backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once
# the max retries are exceeded, Dendrite will no longer try to send transactions to
# that server until it comes back to life and connects to us again.
send_max_retries: 16
# Disable the validation of TLS certificates of remote federated homeservers. Do not
# enable this option in production as it presents a security risk!
disable_tls_validation: false
# Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
# keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
# more quickly than opening new connections each time. Disabling keepalives will close
# HTTP connections immediately after a successful request but may result in more CPU and
# memory being used on TLS handshakes for each new connection instead.
disable_http_keepalives: false
# Perspective keyservers to use as a backup when direct key fetches fail. This may
# be required to satisfy key requests for servers that are no longer online when
# joining some rooms.
key_perspectives:
- server_name: matrix.org
keys:
- key_id: ed25519:auto
public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
- key_id: ed25519:a_RXGa
public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
# This option will control whether Dendrite will prefer to look up keys directly
# or whether it should try perspective servers first, using direct fetches as a
# last resort.
prefer_direct_fetch: false
database:
connection_string: file:dendrite-federationapi.db
# Configuration for the Media API.
media_api:
# Storage path for uploaded media. May be relative or absolute.
base_path: ./media_store
# The maximum allowed file size (in bytes) for media uploads to this homeserver
# (0 = unlimited). If using a reverse proxy, ensure it allows requests at least
#this large (e.g. the client_max_body_size setting in nginx).
max_file_size_bytes: 10485760
# Whether to dynamically generate thumbnails if needed.
dynamic_thumbnails: false
# The maximum number of simultaneous thumbnail generators to run.
max_thumbnail_generators: 10
# A list of thumbnail sizes to be generated for media content.
thumbnail_sizes:
- width: 32
height: 32
method: crop
- width: 96
height: 96
method: crop
- width: 640
height: 480
method: scale
database:
connection_string: file:dendrite-mediaapi.db
# Configuration for enabling experimental MSCs on this homeserver.
mscs:
mscs:
# - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
# - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
database:
connection_string: file:dendrite-msc.db
# Configuration for the Sync API.
sync_api:
# This option controls which HTTP header to inspect to find the real remote IP
# address of the client. This is likely required if Dendrite is running behind
# a reverse proxy server.
# real_ip_header: X-Real-IP
# Configuration for the full-text search engine.
search:
# Whether or not search is enabled.
enabled: false
# The path where the search index will be created in.
index_path: "./searchindex"
# The language most likely to be used on the server - used when indexing, to
# ensure the returned results match expectations. A full list of possible languages
# can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
language: "en"
database:
connection_string: file:dendrite-syncapi.db
# Configuration for the User API.
user_api:
# The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
# See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
# Setting this lower makes registration/login consume less CPU resources at the cost
# of security should the database be compromised. Setting this higher makes registration/login
# consume more CPU resources but makes it harder to brute force password hashes. This value
# can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds).
bcrypt_cost: 10
# The length of time that a token issued for a relying party from
# /_matrix/client/r0/user/{userId}/openid/request_token endpoint
# is considered to be valid in milliseconds.
# The default lifetime is 3600000ms (60 minutes).
# openid_token_lifetime_ms: 3600000
# Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option.
# By default, any room aliases included in this list will be created as a publicly joinable room
# when the first user registers for the homeserver. If the room already exists,
# make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'.
# As Spaces are just rooms under the hood, Space aliases may also be used.
auto_join_rooms:
# - "#main:matrix.org"
account_database:
connection_string: file:dendrite-userapi.db
room_server:
database:
connection_string: file:dendrite-roomserverapi.db
key_server:
database:
connection_string: file:dendrite-keyserverapi.db
relay_api:
database:
connection_string: file:dendrite-relayapi.db
# Configuration for Opentracing.
# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
# how this works and how to set it up.
tracing:
enabled: false
jaeger:
serviceName: ""
disabled: false
rpc_metrics: false
tags: []
sampler: null
reporter: null
headers: null
baggage_restrictions: null
throttler: null
# Logging configuration. The "std" logging type controls the logs being sent to
# stdout. The "file" logging type controls logs being written to a log folder on
# the disk. Supported log levels are "debug", "info", "warn", "error".
logging:
- type: std
level: debug
- type: file
level: debug
params:
path: ./logs

View file

@ -18,6 +18,7 @@ export interface HomeserverConfig {
readonly configDir: string;
readonly baseUrl: string;
readonly port: number;
readonly registrationSecret: string;
}
export interface HomeserverInstance {

View file

@ -14,29 +14,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import * as path from "path";
import * as os from "os";
import * as crypto from "crypto";
import * as path from "node:path";
import * as os from "node:os";
import * as crypto from "node:crypto";
import * as fse from "fs-extra";
import { APIRequestContext } from "@playwright/test";
import { getFreePort } from "../utils/port";
import { Docker } from "../docker";
import {
HomeserverConfig,
HomeserverInstance,
Homeserver,
StartHomeserverOpts,
Credentials,
} from "../utils/homeserver";
import { getFreePort } from "../../utils/port";
import { Docker } from "../../docker";
import { HomeserverConfig, HomeserverInstance, Homeserver, StartHomeserverOpts, Credentials } from "..";
import { randB64Bytes } from "../../utils/rand";
function randB64Bytes(numBytes: number): string {
return crypto.randomBytes(numBytes).toString("base64").replace(/=*$/, "");
}
async function cfgDirFromTemplate(
opts: StartHomeserverOpts,
): Promise<HomeserverConfig & { registrationSecret: string }> {
async function cfgDirFromTemplate(opts: StartHomeserverOpts): Promise<HomeserverConfig> {
const templateDir = path.join(__dirname, "templates", opts.template);
const stats = await fse.stat(templateDir);
@ -104,8 +93,8 @@ async function cfgDirFromTemplate(
}
export class Synapse implements Homeserver, HomeserverInstance {
private docker: Docker = new Docker();
public config: HomeserverConfig & { serverId: string; registrationSecret: string };
protected docker: Docker = new Docker();
public config: HomeserverConfig & { serverId: string };
public constructor(private readonly request: APIRequestContext) {}

View file

@ -37,7 +37,7 @@ export class MailHogServer {
console.log(`Starting mailhog...`);
const containerId = await this.docker.run({
image: "mailhog/mailhog:latest",
containerName: `react-sdk-cypress-mailhog`,
containerName: `react-sdk-playwright-mailhog`,
params: ["--rm", "-p", `${smtpPort}:1025/tcp`, "-p", `${httpPort}:8025/tcp`],
});
console.log(`Started mailhog on ports smtp=${smtpPort} http=${httpPort}.`);

View file

@ -0,0 +1,21 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import crypto from "node:crypto";
export function randB64Bytes(numBytes: number): string {
return crypto.randomBytes(numBytes).toString("base64").replace(/=*$/, "");
}