blitzwing/cryptpad
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

judge blob activity based on mtime instead of atime

Browse source 
and output a basic overview when eviction completes
This commit is contained in:
ansuz 2023-01-23 08:29:36 +05:30
parent 4d78a26b0c
commit 7b65d3e8c8
3 changed files with 19 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
lib/eviction.js
View file

@ -244,7 +244,7 @@ var evictArchived = function (Env, cb) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
return void next();
}
if (item && getNewestTime(item) > retentionTime) { return void next(); }
if (item && item.mtime > retentionTime) { return void next(); }
blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
@ -271,7 +271,7 @@ var evictArchived = function (Env, cb) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next();
}
if (item && getNewestTime(item) > retentionTime) { return void next(); }
if (item && item.mtime > retentionTime) { return void next(); }
blobs.remove.archived.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
@ -293,7 +293,7 @@ var evictArchived = function (Env, cb) {
.nThen(removeArchivedBlobProofs)
.nThen(removeArchivedBlobs)
.nThen(function () {
cb();
cb(void 0, report);
});
};
@ -367,8 +367,8 @@ module.exports = function (Env, cb) {
TODO make this configurable ?
*/
var BLOOM_CAPACITY = (1 << 20) - 1; // over a million items
var BLOOM_ERROR = 1 / 10000; // an error rate of one in a thousand
var BLOOM_CAPACITY = (1 << 22) - 1; // over two million items
var BLOOM_ERROR = 1 / 10000; // an error rate of one in ten thousand
// we'll use one filter for the set of active documents
var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
@ -450,7 +450,7 @@ module.exports = function (Env, cb) {
next();
return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item);
}
if (getNewestTime(item) > inactiveTime) {
if (item.mtime > inactiveTime) {
activeDocs.add(item.blobId);
active++;
return void next();
@ -608,7 +608,7 @@ module.exports = function (Env, cb) {
// This seems redundant because we're already checking the bloom filter
// but we can't implement a 'fast mode' for the iterator
// unless we address this race condition with this last-minute double-check
if (getNewestTime(item) > inactiveTime) { return void next(); }
if (item.mtime > inactiveTime) { return void next(); }
removed++;
blobs.archive.blob(item.blobId, function (err) {
@ -657,7 +657,7 @@ module.exports = function (Env, cb) {
}
if (pinnedDocs.test(item.blobId)) { return void next(); }
if (getNewestTime(item) > inactiveTime) { return void next(); }
if (item.mtime > inactiveTime) { return void next(); }
nThen(function (w) {
blobs.size(item.blobId, w(function (err, size) {
if (err) {
@ -706,6 +706,10 @@ module.exports = function (Env, cb) {
Log.error('EVICT_CHANNEL_ITERATION', err);
return void cb();
}
// ignore the special admin broadcast channel
if (item.channel.length === 33) { return void cb(); }
// check if the database has any ephemeral channels
// if it does it's because of a bug, and they should be removed
if (item.channel.length === 34) {
@ -733,7 +737,7 @@ module.exports = function (Env, cb) {
// because it might have been created after the initial activity scan
store.getChannelStats(item.channel, w(function (err, newerItem) {
if (err) { return; }
if (item && getNewestTime(newerItem) > retentionTime) {
if (newerItem && getNewestTime(newerItem) > retentionTime) {
// it's actually active, so don't archive it.
w.abort();
cb();

5
scripts/evict-archived.js
View file

@ -98,7 +98,8 @@ nThen(function (w) {
}));
}).nThen(function (w) {
Eviction.archived(Env, w(function () {
Eviction.archived(Env, w(function (err, report) {
if (!report) { return; }
Env.Log.info('EVICT_ARCHIVED_FINAL_REPORT', report);
}));
});

5
scripts/evict-inactive.js
View file

@ -97,7 +97,8 @@ nThen(function (w) {
}));
}).nThen(function (w) {
Eviction(Env, w(function () {
Eviction(Env, w(function (err, report) {
if (!report) { return; }
Env.Log.info('EVICT_INACTIVE_FINAL_REPORT', report);
}));
});