From 911c15df257a0cd64113f3535714cdf7c0720489 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 19 Jan 2023 08:56:32 +0530 Subject: [PATCH 1/4] fix incorrectly constructed archival path for blobs --- lib/storage/blob.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/storage/blob.js b/lib/storage/blob.js index a2d9930f3..7d55676d7 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -19,7 +19,11 @@ var isValidId = function (id) { // helpers var prependArchive = function (Env, path) { - return Path.join(Env.archivePath, path); + // Env has an absolute path to the blob storage + // we want the path to the blob relative to that + var relativePathToBlob = Path.relative(Env.blobPath, path); + // the new path structure is the same, but relative to the blob archive root + return Path.join(Env.archivePath, 'blob', relativePathToBlob); }; // /blob//// @@ -492,7 +496,7 @@ BlobStore.create = function (config, _cb) { if (e) { CB(e); } })); - Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) { + Fse.mkdirp(Path.join(Env.archivePath, './blob'), w(function (e) { if (e) { CB(e); } })); }).nThen(function (w) { From 15272a660421bb05d7edd221f059615dc77a38d2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 19 Jan 2023 08:57:39 +0530 Subject: [PATCH 2/4] relocate blobs that have been archived to the wrong location --- lib/eviction.js | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/lib/eviction.js b/lib/eviction.js index 673c7313f..32e1bdd56 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -3,6 +3,10 @@ var Bloom = require("@mcrowe/minibloom"); var Util = require("../lib/common-util"); var Pins = require("../lib/pins"); var Keys = require("./keys"); +var Path = require('node:path'); +var config = require("./load-config"); +var Fs = require("node:fs"); +var Fse = require("fs-extra"); var getNewestTime = function (stats) { return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { @@ -70,6 +74,103 @@ var evictArchived = function (Env, cb) { blobs = Env.blobStore; }; + var migrateBlobRoot = function (from, to) { + // only migrate subpaths, leave everything else alone + if (!Path.dirname(from).startsWith(Path.dirname(to))) { return; } + + // expects a directory + var recurse = function (relativePath) { + var src = Path.join(from, relativePath); + var children; + try { + children = Fs.readdirSync(src); + } catch (err) { + if (err.code === 'ENOENT') { return; } + // if you can't read a directory's contents + // then nothing else will work, so just abort + Log.verbose("EVICT_ARCHIVED_NOT_DIRECTORY", { + error: err, + }); + return; + } + + var dest; + if (children.length === 0) { + try { + Fse.removeSync(src); + } catch (err2) { + Log.error('EVICT_ARCHIVED_EMPTY_DIR_REMOVAL', { + error: err2, + }); + // removal is non-essential, so we can continue + } + } else { + // make an equivalent path in the target directory + dest = Path.join(to, relativePath); + + try { + Fse.mkdirpSync(dest); + } catch (err3) { + Log.error("EVICT_ARCHIVED_BLOB_MIGRATION", { + error: err3, + }); + + // failure to create the host directory + // will cause problems when we try to move + // so bail out here + return; + } + } + + children.forEach(function (child) { + var childSrcPath = Path.join(src, child); + var stat = Fs.statSync(childSrcPath); + if (stat.isDirectory()) { + return void recurse(Path.join(relativePath, child)); + } + + var childDestPath = Path.join(dest, child); + + try { + Log.verbose("EVICT_ARCHIVED_MOVE_FROM_DEPRECATED_PATH", { + from: childSrcPath, + to: childDestPath, + }); + Fse.moveSync(childSrcPath, childDestPath, { + overwrite: false, + }); + } catch (err4) { + Log.error('EVICT_ARCHIVED_MOVE_FAILURE', { + error: err4, + }); + } + }); + }; + recurse(''); + }; + +/* In CryptPad 5.2.0 we merged a patch which converted + all of CryptPad's root filepaths to their absolute form, + rather than the relative paths we'd been using until then. + Unfortunately, we overlooked a case where two absolute + paths were concatenated together, resulting in blobs being + archived to an incorrect path. + + This migration detects evidence of incorrect archivals + and moves such archived files to their intended location + before continuing with the normal eviction procedure. +*/ + var migrateIncorrectBlobs = function () { + var incorrectPaths = [ + Path.join(Env.paths.archive, config.blobPath), + Path.join(Env.paths.archive, Path.resolve(config.blobPath)) + ]; + var correctPath = Path.join(Env.paths.archive, 'blob'); + incorrectPaths.forEach(root => { + migrateBlobRoot(root, correctPath); + }); + }; + var removeArchivedChannels = function (w) { // this block will iterate over archived channels and removes them // if they've been in cold storage for longer than your configured archive time @@ -186,6 +287,7 @@ var evictArchived = function (Env, cb) { }; nThen(loadStorage) + .nThen(migrateIncorrectBlobs) .nThen(removeArchivedChannels) .nThen(removeArchivedBlobProofs) .nThen(removeArchivedBlobs) From c7dedf40541a8c818e21f90b0ba4990bab3b63d5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 19 Jan 2023 10:14:22 +0530 Subject: [PATCH 3/4] more logging for potentially long-running eviction processes --- lib/eviction.js | 58 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/lib/eviction.js b/lib/eviction.js index 32e1bdd56..1002a97b2 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -36,6 +36,7 @@ Env = { // the number of ms artificially introduced between CPU-intensive operations var THROTTLE_FACTOR = 10; +var PROGRESS_FACTOR = 1000; var evictArchived = function (Env, cb) { var Log; @@ -391,6 +392,12 @@ module.exports = function (Env, cb) { var active = 0; var handler = function (err, item, cb) { channels++; + if (channels % PROGRESS_FACTOR === 0) { + Log.info('EVICT_CHANNEL_CATEGORIZATION_PROGRESS', { + channels: channels, + }); + } + if (err) { Log.error('EVICT_CHANNEL_CATEGORIZATION', err); return void cb(); @@ -417,6 +424,7 @@ module.exports = function (Env, cb) { }); }; + Log.info('EVICT_CHANNEL_ACTIVITY_START', 'Assessing channel activity'); store.listChannels(handler, w(done)); }; @@ -424,9 +432,16 @@ module.exports = function (Env, cb) { var n_blobs = 0; var active = 0; + Log.info('EVICT_BLOBS_ACTIVITY_START', 'Assessing blob activity'); blobs.list.blobs(function (err, item, next) { next = Util.mkAsync(next, THROTTLE_FACTOR); n_blobs++; + if (n_blobs % PROGRESS_FACTOR === 0) { + Log.info('EVICT_BLOB_CATEGORIZATION_PROGRESS', { + blobs: n_blobs, + }); + } + if (err) { Log.error("EVICT_BLOB_CATEGORIZATION", err); return void next(); @@ -495,6 +510,11 @@ module.exports = function (Env, cb) { var handler = function (content, id, next) { next = Util.mkAsync(next, THROTTLE_FACTOR); accounts++; + if (accounts % PROGRESS_FACTOR === 0) { + Log.info('EVICT_ACCOUNT_CATEGORIZATION_PROGRESS', { + accounts: accounts, + }); + } var mtime = content.latest; var pinList = Object.keys(content.pins); @@ -551,6 +571,7 @@ module.exports = function (Env, cb) { }); }; + Log.info('EVICT_ACCOUNTS_ACTIVITY_START', 'Assessing account activity'); Pins.load(w(done), { pinPath: Env.paths.pin, handler: handler, @@ -562,6 +583,8 @@ module.exports = function (Env, cb) { // if they have not been accessed within the specified retention time var removed = 0; var total = 0; + + Log.info('EVICT_BLOB_START', {}); blobs.list.blobs(function (err, item, next) { next = Util.mkAsync(next, THROTTLE_FACTOR); if (err) { @@ -573,6 +596,12 @@ module.exports = function (Env, cb) { return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item); } total++; + if (total % PROGRESS_FACTOR === 0) { + Log.info("EVICT_BLOB_PROGRESS", { + blobs: total, + }); + } + if (pinnedDocs.test(item.blobId)) { return void next(); } if (activeDocs.test(item.blobId)) { return void next(); } @@ -606,6 +635,9 @@ module.exports = function (Env, cb) { // iterate over blob proofs and remove them // if they don't correspond to a pinned or active file var removed = 0; + var total = 0; + + Log.info("EVICT_ARCHIVE_INACTIVE_BLOB_PROOFS_START", {}); blobs.list.proofs(function (err, item, next) { next = Util.mkAsync(next, THROTTLE_FACTOR); if (err) { @@ -616,6 +648,14 @@ module.exports = function (Env, cb) { next(); return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item); } + total++; + + if (total % PROGRESS_FACTOR === 0) { + Log.info('EVICT_BLOB_PROOF_PROGRESS', { + proofs: total, + }); + } + if (pinnedDocs.test(item.blobId)) { return void next(); } if (getNewestTime(item) > inactiveTime) { return void next(); } nThen(function (w) { @@ -641,7 +681,10 @@ module.exports = function (Env, cb) { }); }); }, w(function () { - Log.info("EVICT_BLOB_PROOFS_REMOVED", removed); + Log.info("EVICT_BLOB_PROOFS_REMOVED", { + removed, + total, + }); })); }; @@ -652,6 +695,13 @@ module.exports = function (Env, cb) { var handler = function (err, item, cb) { cb = Util.mkAsync(cb, THROTTLE_FACTOR); channels++; + if (channels % PROGRESS_FACTOR === 0) { + Log.info('EVICT_INACTIVE_CHANNELS_PROGRESS', { + channels, + archived, + }); + } + if (err) { Log.error('EVICT_CHANNEL_ITERATION', err); return void cb(); @@ -708,9 +758,13 @@ module.exports = function (Env, cb) { var done = function () { report.channelsArchived = archived; - return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); + return void Log.info('EVICT_CHANNELS_ARCHIVED', { + channels, + archived, + }); }; + Log.info('EVICT_INACTIVE_CHANNELS_START', {}); store.listChannels(handler, w(done), true); // using a hacky "fast mode" since we only need the channel id }; From 91d781be140c8914a07e2cd688f523b1146867bf Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 20 Jan 2023 16:14:49 +0100 Subject: [PATCH 4/4] Fix locked spreadsheets for guest users on checkpoint required --- www/common/onlyoffice/inner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 66e26f60a..3e06c4818 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -2993,7 +2993,7 @@ Uncaught TypeError: Cannot read property 'calculatedType' of null var m = metadataMgr.getChannelMembers().filter(function (str) { return str.length === 32; }).length; - if ((m - v) === 1 && !readOnly) { + if ((m - v) === 1 && !readOnly && common.isLoggedIn()) { var needCp = ooChannel.queue.length > CHECKPOINT_INTERVAL; APP.initCheckpoint = needCp; }