blitzwing/cryptpad
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

introduce some artificial delays in the eviction script

Browse source 
...and reduce the margin of error in its bloom filters
This commit is contained in:
ansuz 2020-10-14 17:22:31 +05:30
parent 90f046f896
commit a29c5641b8
1 changed files with 9 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
lib/eviction.js
View file

@ -81,7 +81,9 @@ module.exports = function (Env, cb) {
TODO make this configurable ?
*/
var BLOOM_CAPACITY = (1 << 20) - 1; // over a million items
var BLOOM_ERROR = 1 / 1000; // an error rate of one in a thousand
var BLOOM_ERROR = 1 / 10000; // an error rate of one in a thousand
// the number of ms artificially introduced between CPU-intensive operations
var THROTTLE_FACTOR = 10;
// we'll use one filter for the set of active documents
var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
@ -165,6 +167,7 @@ module.exports = function (Env, cb) {
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
return void next();
@ -190,6 +193,7 @@ module.exports = function (Env, cb) {
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next();
@ -246,6 +250,7 @@ module.exports = function (Env, cb) {
var active = 0;
blobs.list.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
n_blobs++;
if (err) {
Log.error("EVICT_BLOB_CATEGORIZATION", err);
@ -311,6 +316,7 @@ module.exports = function (Env, cb) {
// otherwise, we'll only retain data from active accounts
// so we need more heuristics
var handler = function (content, id, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
accounts++;
var mtime = content.latest;
@ -416,6 +422,7 @@ module.exports = function (Env, cb) {
// if they don't correspond to a pinned or active file
var removed = 0;
blobs.list.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
@ -458,6 +465,7 @@ module.exports = function (Env, cb) {
var archived = 0;
var handler = function (err, item, cb) {
cb = Util.mkAsync(cb, THROTTLE_FACTOR);
channels++;
if (err) {
Log.error('EVICT_CHANNEL_ITERATION', err);