Compare commits
4 commits
main
...
monitoring
Author | SHA1 | Date | |
---|---|---|---|
|
8f1026eae1 | ||
|
fff4e7581e | ||
|
49af0533b5 | ||
|
4f2a48a72e |
8 changed files with 786 additions and 1203 deletions
|
@ -254,6 +254,7 @@ module.exports.create = function (config) {
|
||||||
curvePublic: Nacl.util.encodeBase64(curve.publicKey),
|
curvePublic: Nacl.util.encodeBase64(curve.publicKey),
|
||||||
|
|
||||||
selfDestructTo: {},
|
selfDestructTo: {},
|
||||||
|
monitoring: {}
|
||||||
};
|
};
|
||||||
|
|
||||||
(function () {
|
(function () {
|
||||||
|
@ -417,6 +418,7 @@ const BAD = [
|
||||||
'limits',
|
'limits',
|
||||||
'customLimits',
|
'customLimits',
|
||||||
'scheduleDecree',
|
'scheduleDecree',
|
||||||
|
'monitoring',
|
||||||
|
|
||||||
'httpServer',
|
'httpServer',
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,9 @@ const BlobStore = require("./storage/blob");
|
||||||
const BlockStore = require("./storage/block");
|
const BlockStore = require("./storage/block");
|
||||||
const plugins = require("./plugin-manager");
|
const plugins = require("./plugin-manager");
|
||||||
|
|
||||||
|
const Prometheus = require('prom-client');
|
||||||
|
const Monitoring = require('./monitoring');
|
||||||
|
|
||||||
const DEFAULT_QUERY_TIMEOUT = 5000;
|
const DEFAULT_QUERY_TIMEOUT = 5000;
|
||||||
const PID = process.pid;
|
const PID = process.pid;
|
||||||
|
|
||||||
|
@ -66,6 +69,55 @@ Env.incrementBytesWritten = function () {};
|
||||||
|
|
||||||
const EVENTS = {};
|
const EVENTS = {};
|
||||||
|
|
||||||
|
// XXX Store in monitoring.js
|
||||||
|
const rssMetric = new Prometheus.Gauge({
|
||||||
|
name: `memory_rss`,
|
||||||
|
help: 'The amount of space occupied in the main memory device for the process.',
|
||||||
|
labelNames: ['pid', 'type']
|
||||||
|
});
|
||||||
|
const heapTotalMetric = new Prometheus.Gauge({
|
||||||
|
name: `memory_heap_total`,
|
||||||
|
help: "Total heap memory.",
|
||||||
|
labelNames: ['pid', 'type']
|
||||||
|
});
|
||||||
|
const heapUsedMetric = new Prometheus.Gauge({
|
||||||
|
name: `memory_heap_used`,
|
||||||
|
help: 'Used heap memory.',
|
||||||
|
labelNames: ['pid', 'type']
|
||||||
|
});
|
||||||
|
const externalMetric = new Prometheus.Gauge({
|
||||||
|
name: `memory_external`,
|
||||||
|
help: 'Memory usage of C++ objects bound to JavaScript objects managed by V8.',
|
||||||
|
labelNames: ['pid', 'type']
|
||||||
|
});
|
||||||
|
const arrayBufferMetric = new Prometheus.Gauge({
|
||||||
|
name: `memory_array_buffers`,
|
||||||
|
help: 'Memory allocated for ArrayBuffers and SharedArrayBuffers.',
|
||||||
|
labelNames: ['pid', 'type']
|
||||||
|
});
|
||||||
|
EVENTS.MONITORING = function (data) {
|
||||||
|
/*
|
||||||
|
{
|
||||||
|
main: {
|
||||||
|
rss: 1234
|
||||||
|
...
|
||||||
|
},
|
||||||
|
pid1: {
|
||||||
|
rss: 234
|
||||||
|
...
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
Object.keys(data).forEach(pid => {
|
||||||
|
rssMetric.set({pid: pid, type: data[pid].type}, data[pid].mem.rss);
|
||||||
|
heapTotalMetric.set({pid: pid, type: data[pid].type}, data[pid].mem.heapTotal);
|
||||||
|
heapUsedMetric.set({pid: pid, type: data[pid].type}, data[pid].mem.heapUsed);
|
||||||
|
externalMetric.set({pid: pid, type: data[pid].type}, data[pid].mem.external);
|
||||||
|
arrayBufferMetric.set({pid: pid, type: data[pid].type}, data[pid].mem.arrayBuffers);
|
||||||
|
});
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
EVENTS.ENV_UPDATE = function (data /*, cb */) {
|
EVENTS.ENV_UPDATE = function (data /*, cb */) {
|
||||||
try {
|
try {
|
||||||
Env = JSON.parse(data);
|
Env = JSON.parse(data);
|
||||||
|
@ -219,6 +271,12 @@ const wsProxy = createProxyMiddleware({
|
||||||
|
|
||||||
app.use('/cryptpad_websocket', wsProxy);
|
app.use('/cryptpad_websocket', wsProxy);
|
||||||
|
|
||||||
|
app.get('/metrics', (req, res) => {
|
||||||
|
Prometheus.register.metrics().then((data) => {
|
||||||
|
res.set('Content-Type', Prometheus.register.contentType);
|
||||||
|
res.send(data);
|
||||||
|
});
|
||||||
|
});
|
||||||
app.use('/ssoauth', (req, res, next) => {
|
app.use('/ssoauth', (req, res, next) => {
|
||||||
if (SSOUtils && req && req.body && req.body.SAMLResponse) {
|
if (SSOUtils && req && req.body && req.body.SAMLResponse) {
|
||||||
req.method = 'GET';
|
req.method = 'GET';
|
||||||
|
@ -763,7 +821,14 @@ nThen(function (w) {
|
||||||
}));
|
}));
|
||||||
}).nThen(function () {
|
}).nThen(function () {
|
||||||
// TODO inform the parent process that this worker is ready
|
// TODO inform the parent process that this worker is ready
|
||||||
|
setInterval(() => {
|
||||||
|
sendMessage({
|
||||||
|
command: 'MONITORING',
|
||||||
|
data: Monitoring.getData('http-worker')
|
||||||
|
}, () => {
|
||||||
|
// Done
|
||||||
|
});
|
||||||
|
}, Monitoring.interval);
|
||||||
});
|
});
|
||||||
|
|
||||||
process.on('uncaughtException', function (err) {
|
process.on('uncaughtException', function (err) {
|
||||||
|
|
35
lib/monitoring.js
Normal file
35
lib/monitoring.js
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
/*
|
||||||
|
globals process
|
||||||
|
*/
|
||||||
|
const VALUES = {};
|
||||||
|
VALUES.mem = () => {
|
||||||
|
return process.memoryUsage();
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyToEnv = (Env, data) => {
|
||||||
|
if (!Env) { return; }
|
||||||
|
Env.monitoring[data.pid] = data;
|
||||||
|
};
|
||||||
|
const getData = (type) => {
|
||||||
|
const value = {
|
||||||
|
pid: process.pid,
|
||||||
|
type: type
|
||||||
|
};
|
||||||
|
Object.keys(VALUES).forEach(key => {
|
||||||
|
value[key] = VALUES[key]();
|
||||||
|
});
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
const remove = (Env, pid) => {
|
||||||
|
if (Env && Env.monitoring && pid && Env.monitoring[pid]) {
|
||||||
|
delete Env.monitoring[pid];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
interval: 5000,
|
||||||
|
applyToEnv,
|
||||||
|
getData,
|
||||||
|
remove
|
||||||
|
};
|
|
@ -18,6 +18,7 @@ const Logger = require("../log");
|
||||||
const Tasks = require("../storage/tasks");
|
const Tasks = require("../storage/tasks");
|
||||||
const Nacl = require('tweetnacl/nacl-fast');
|
const Nacl = require('tweetnacl/nacl-fast');
|
||||||
const Eviction = require("../eviction");
|
const Eviction = require("../eviction");
|
||||||
|
const Monitoring = require('../monitoring');
|
||||||
|
|
||||||
const Env = {
|
const Env = {
|
||||||
Log: {},
|
Log: {},
|
||||||
|
@ -58,6 +59,13 @@ const init = function (config, _cb) {
|
||||||
Env.archiveRetentionTime = config.archiveRetentionTime;
|
Env.archiveRetentionTime = config.archiveRetentionTime;
|
||||||
Env.accountRetentionTime = config.accountRetentionTime;
|
Env.accountRetentionTime = config.accountRetentionTime;
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
process.send({
|
||||||
|
monitoring: true,
|
||||||
|
data: Monitoring.getData('db-worker')
|
||||||
|
});
|
||||||
|
}, Monitoring.interval);
|
||||||
|
|
||||||
nThen(function (w) {
|
nThen(function (w) {
|
||||||
Store.create(config, w(function (err, _store) {
|
Store.create(config, w(function (err, _store) {
|
||||||
if (err) {
|
if (err) {
|
||||||
|
|
|
@ -10,6 +10,7 @@ const { fork } = require('child_process');
|
||||||
const Workers = module.exports;
|
const Workers = module.exports;
|
||||||
const PID = process.pid;
|
const PID = process.pid;
|
||||||
const Block = require("../storage/block");
|
const Block = require("../storage/block");
|
||||||
|
const Monitoring = require('../monitoring');
|
||||||
|
|
||||||
const DB_PATH = 'lib/workers/db-worker';
|
const DB_PATH = 'lib/workers/db-worker';
|
||||||
const MAX_JOBS = 16;
|
const MAX_JOBS = 16;
|
||||||
|
@ -163,6 +164,13 @@ Workers.initialize = function (Env, config, _cb) {
|
||||||
if (res.log) {
|
if (res.log) {
|
||||||
return void handleLog(res.log, res.label, res.info);
|
return void handleLog(res.log, res.label, res.info);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handle monitoring data
|
||||||
|
if (res.monitoring) {
|
||||||
|
Monitoring.applyToEnv(Env, res.data);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// but don't bother handling things addressed to other processes
|
// but don't bother handling things addressed to other processes
|
||||||
// since it's basically guaranteed not to work
|
// since it's basically guaranteed not to work
|
||||||
if (res.pid !== PID) {
|
if (res.pid !== PID) {
|
||||||
|
@ -227,7 +235,9 @@ Workers.initialize = function (Env, config, _cb) {
|
||||||
handleResponse(state, res);
|
handleResponse(state, res);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let pid = worker.pid;
|
||||||
var substituteWorker = Util.once(function () {
|
var substituteWorker = Util.once(function () {
|
||||||
|
Monitoring.remove(Env, pid);
|
||||||
Env.Log.info("SUBSTITUTE_DB_WORKER", '');
|
Env.Log.info("SUBSTITUTE_DB_WORKER", '');
|
||||||
var idx = workers.indexOf(state);
|
var idx = workers.indexOf(state);
|
||||||
if (idx !== -1) {
|
if (idx !== -1) {
|
||||||
|
|
1837
package-lock.json
generated
1837
package-lock.json
generated
File diff suppressed because it is too large
Load diff
17
package.json
17
package.json
|
@ -13,18 +13,33 @@
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@mcrowe/minibloom": "^0.2.0",
|
"@mcrowe/minibloom": "^0.2.0",
|
||||||
|
"alertify.js": "1.0.11",
|
||||||
|
"bootstrap": "^4.0.0",
|
||||||
|
"bootstrap-tokenfield": "^0.12.0",
|
||||||
|
"chainpad": "^5.2.6",
|
||||||
"chainpad-crypto": "^0.2.5",
|
"chainpad-crypto": "^0.2.5",
|
||||||
|
"chainpad-listmap": "^1.0.0",
|
||||||
|
"chainpad-netflux": "^1.0.0",
|
||||||
"chainpad-server": "^5.1.0",
|
"chainpad-server": "^5.1.0",
|
||||||
"cookie-parser": "^1.4.6",
|
"cookie-parser": "^1.4.6",
|
||||||
"body-parser": "^1.20.2",
|
"body-parser": "^1.20.2",
|
||||||
"express": "~4.19.2",
|
"express": "~4.19.2",
|
||||||
"fs-extra": "^7.0.0",
|
"fs-extra": "^7.0.0",
|
||||||
"get-folder-size": "^2.0.1",
|
"get-folder-size": "^2.0.1",
|
||||||
"netflux-websocket": "^1.0.0",
|
"html2canvas": "^1.4.0",
|
||||||
"http-proxy-middleware": "^2.0.6",
|
"http-proxy-middleware": "^2.0.6",
|
||||||
|
"hyper-json": "~1.4.0",
|
||||||
|
"jquery": "3.6.0",
|
||||||
|
"json.sortify": "~2.1.0",
|
||||||
"jsonwebtoken": "^9.0.0",
|
"jsonwebtoken": "^9.0.0",
|
||||||
|
"jszip": "3.10.1",
|
||||||
|
"localforage": "^1.5.2",
|
||||||
|
"marked": "^4.3.0",
|
||||||
|
"mathjax": "3.0.5",
|
||||||
|
"netflux-websocket": "^1.0.0",
|
||||||
"notp": "^2.0.3",
|
"notp": "^2.0.3",
|
||||||
"nthen": "0.1.8",
|
"nthen": "0.1.8",
|
||||||
|
"prom-client": "^14.2.0",
|
||||||
"openid-client": "^5.4.2",
|
"openid-client": "^5.4.2",
|
||||||
"@node-saml/node-saml": "^4.0.5",
|
"@node-saml/node-saml": "^4.0.5",
|
||||||
"alertify.js": "1.0.11",
|
"alertify.js": "1.0.11",
|
||||||
|
|
13
server.js
13
server.js
|
@ -18,6 +18,7 @@ var config = require("./lib/load-config");
|
||||||
var Environment = require("./lib/env");
|
var Environment = require("./lib/env");
|
||||||
var Env = Environment.create(config);
|
var Env = Environment.create(config);
|
||||||
var Default = require("./lib/defaults");
|
var Default = require("./lib/defaults");
|
||||||
|
var Monitoring = require('./lib/monitoring');
|
||||||
|
|
||||||
var app = Express();
|
var app = Express();
|
||||||
|
|
||||||
|
@ -52,6 +53,11 @@ COMMANDS.GET_PROFILING_DATA = function (msg, cb) {
|
||||||
cb(void 0, Env.bytesWritten);
|
cb(void 0, Env.bytesWritten);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
COMMANDS.MONITORING = function (msg, cb) {
|
||||||
|
Monitoring.applyToEnv(Env, msg.data);
|
||||||
|
cb();
|
||||||
|
};
|
||||||
|
|
||||||
nThen(function (w) {
|
nThen(function (w) {
|
||||||
require("./lib/log").create(config, w(function (_log) {
|
require("./lib/log").create(config, w(function (_log) {
|
||||||
Env.Log = _log;
|
Env.Log = _log;
|
||||||
|
@ -103,6 +109,7 @@ nThen(function (w) {
|
||||||
|
|
||||||
var launchWorker = (online) => {
|
var launchWorker = (online) => {
|
||||||
var worker = Cluster.fork(workerState);
|
var worker = Cluster.fork(workerState);
|
||||||
|
var pid = worker.process.pid;
|
||||||
worker.on('online', () => {
|
worker.on('online', () => {
|
||||||
online();
|
online();
|
||||||
});
|
});
|
||||||
|
@ -132,6 +139,7 @@ nThen(function (w) {
|
||||||
});
|
});
|
||||||
|
|
||||||
worker.on('exit', (code, signal) => {
|
worker.on('exit', (code, signal) => {
|
||||||
|
Monitoring.remove(Env, pid);
|
||||||
if (!signal && code === 0) { return; }
|
if (!signal && code === 0) { return; }
|
||||||
// relaunch http workers if they crash
|
// relaunch http workers if they crash
|
||||||
Env.Log.error('HTTP_WORKER_EXIT', {
|
Env.Log.error('HTTP_WORKER_EXIT', {
|
||||||
|
@ -173,6 +181,11 @@ nThen(function (w) {
|
||||||
broadcast('FLUSH_CACHE', Env.FRESH_KEY);
|
broadcast('FLUSH_CACHE', Env.FRESH_KEY);
|
||||||
}, 250);
|
}, 250);
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
Monitoring.applyToEnv(Env, Monitoring.getData('main'));
|
||||||
|
broadcast('MONITORING', Env.monitoring);
|
||||||
|
}, Monitoring.interval);
|
||||||
|
|
||||||
Env.envUpdated.reg(throttledEnvChange);
|
Env.envUpdated.reg(throttledEnvChange);
|
||||||
Env.cacheFlushed.reg(throttledCacheFlush);
|
Env.cacheFlushed.reg(throttledCacheFlush);
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue