Merge branch 'main' into debower

.jshintignore

@@ -7,6 +7,7 @@ www/common/onlyoffice/v1
 www/common/onlyoffice/v2*
 www/common/onlyoffice/v4
 www/common/onlyoffice/v5
+www/common/onlyoffice/v*/
 
 www/scratch
 www/accounts

CHANGELOG.md

@@ -1,3 +1,72 @@
+# 5.3.0
+
+## Goals
+
+This release updates OnlyOffice applications to version 7.3.3. It improves the Form application and other areas of CryptPad with minor features and bug fixes. 
+
+## Features
+
+- Upgrade OnlyOffice applications (Sheet, Document, Presentation) to version 7.1
+
+- Forms
+  - New question type: Date [[#811](https://github.com/xwiki-labs/cryptpad/issues/811)]
+  - Add Condorcet voting results to ordered list responses
+
+- Default dark theme switch [[#759](https://github.com/xwiki-labs/cryptpad/issues/759)]: set dark theme as the default for the instance in `application_config.js`
+
+- New FreeBSD rc.d init script
+
+## Improvements
+
+- Auto-select document name on edit if it's still the default [thanks to [piemonkey](https://github.com/piemonkey)]
+
+- Forms
+  - Clarify button text to "Copy Public Link" [[#937](https://github.com/xwiki-labs/cryptpad/issues/937)]
+  - Clarify text on the document creation screen so that "Expiration date" (date at which the document will be destroyed) is not confused with the _closing date_ of the form [user feedback]
+  - Decimals are now allowed in text questions with type "number" [[Forum]](https://forum.cryptpad.org/d/88-decimals-in-number-type-text-field)
+
+- Rich Text
+  - Move width-toggle button out of the way of the text [[#957](https://github.com/xwiki-labs/cryptpad/issues/957)]
+
+- Deployment
+  - Systemd: Removed outdated logging directives and implemented sandboxing and other hardening best practices
+  - Nginx: Invert settings to forbid remote embedding by default
+
+- Removed unused dev dependencies
+
+## Bug Fixes
+
+- Forms and Kanban
+  - Fixed spacing issues with input fields
+
+- Forms
+  - Fixed ways to bypass "required" questions [[#1007](https://github.com/xwiki-labs/cryptpad/issues/1007) [#1014](https://github.com/xwiki-labs/cryptpad/issues/1014)]
+  - Fix missing notifications for responses
+  - Send response notifications to all owners
+
+- Rich Text
+  - Fix scroll issues when clicking on the table of contents 
+  - Fix double notification for mention + reply in a comment
+
+- Fix issues with deprecated cache 
+
+## Update notes
+
+If you are upgrading from a version older than `5.2.0` please read the upgrade notes of all versions between yours and `5.3.0` to avoid configuration issues.
+
+To upgrade:
+
+1. Stop your server
+2. Get the latest code with git
+
+```bash
+git fetch origin --tags
+git checkout 5.3
+```
+
+1. Restart your server
+2. Review your instance's checkup page to ensure that you are passing all tests
+   
 # 5.2.1
 
 ## Goals

customize.dist/ckeditor-contents.css

@@ -52,6 +52,12 @@ body > .non-realtime:first-child + * {
     margin-top: 0;
 }
 
+@media (max-width: 600px) {
+    body {
+        padding-bottom: 100px;
+    }
+}
+
 .cke_editable
 {
     font-size: 16px;

customize.dist/pages.js

@@ -95,7 +95,7 @@ define([
         return h('a', attrs, [icon, text]);
     };
 
-    Pages.versionString = "5.2.1";
+    Pages.versionString = "5.3.0";
 
     var customURLs = Pages.customURLs = {};
     (function () {

customize.dist/pages/install.js

@@ -14,11 +14,6 @@ define([
             return;
         }
 
-/*
-Msg.install_header = "CryptPad Install"; // XXX
-Msg.install_notes = "<ul class=\"cp-notes-list\"><li>Create your first admin account using this form.</li>" +
-                    "<li>Please note your password carefully. <span class=\"red\">If you lose it there is no way we can recover your data.</span></li></ul>"; // XXX
-*/
 Msg.install_token = "Install token";
 
         document.title = Msg.install_header;

docs/cryptpad.service

@@ -2,7 +2,7 @@
 Description=CryptPad API server
 
 [Service]
-ExecStart=/home/cryptpad/.nvm/versions/node/v12.14.0/bin/node /home/cryptpad/cryptpad/server.js
+ExecStart=/home/cryptpad/.nvm/versions/node/v19.8.1/bin/node /home/cryptpad/cryptpad/server.js
 # modify to match the location of your cryptpad repository
 WorkingDirectory=/home/cryptpad/cryptpad
 
@@ -10,10 +10,6 @@ Restart=always
 # Restart service after 10 seconds if node service crashes
 RestartSec=2
 
-# Output to syslog
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=cryptpad
 User=cryptpad
 Group=cryptpad
 # modify to match your working directory
@@ -24,5 +20,42 @@ Environment='PWD="/home/cryptpad/cryptpad"'
 # or risk EMFILE errors.
 LimitNOFILE=1000000
 
+# hardening directives as per https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+# check those with systemd-analyze security cryptpad.service for more information
+# Proc filesystem
+ProcSubset=all
+ProtectProc=invisible
+# Capabilities
+CapabilityBoundingSet=
+# Security
+NoNewPrivileges=true
+# Sandboxing
+ProtectSystem=strict
+PrivateTmp=true
+PrivateDevices=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_INET
+RestrictAddressFamilies=AF_INET6
+RestrictAddressFamilies=AF_NETLINK
+RestrictAddressFamilies=AF_UNIX
+RestrictNamespaces=true
+LockPersonality=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
+RemoveIPC=true
+PrivateMounts=true
+ProtectClock=true
+# System Call Filtering
+SystemCallArchitectures=native
+SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileged @setuid
+SystemCallFilter=@chown
+SystemCallFilter=pipe
+SystemCallFilter=pipe2
+ReadWritePaths=/home/cryptpad/cryptpad
+
 [Install]
 WantedBy=multi-user.target

docs/example.nginx.conf

@@ -23,15 +23,12 @@ server {
     set $main_domain "your-main-domain.com";
     set $sandbox_domain "your-sandbox-domain.com";
 
-    # By default CryptPad allows remote domains to embed CryptPad documents in iframes.
-    # This behaviour can be blocked by changing $allowed_origins from "*" to the
-    # sandbox domain, which must be permitted to load content from the main domain
-    # in order for CryptPad to work as expected.
-    #
-    # An example is given below which can be uncommented if you want to block
-    # remote sites from including content from your server
-    set $allowed_origins "*";
-    # set $allowed_origins "https://${sandbox_domain}";
+    # By default CryptPad forbids remote domains from embedding CryptPad documents in iframes.
+    # The sandbox domain must always be permitted in order for the platform to function.
+    # If you wish to enable remote embedding you may change the value below to "*"
+    # as per the commented value.
+    set $allowed_origins "https://${sandbox_domain}";
+    #set $allowed_origins "*";
 
     # CryptPad's dynamic content (websocket traffic and encrypted blobs)
     # can be served over separate domains. Using dedicated domains (or subdomains)

docs/rc.d-cryptpad

@@ -0,0 +1,37 @@
+!/bin/sh
+# $FreeBSD$
+# PROVIDE: cryptpad
+# REQUIRE: DAEMON nginx
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+
+name="cryptpad"
+start_cmd="start"
+stop_cmd="stop"
+rcvar=cryptpad_enable
+
+pidfile="/var/run/${name}.pid"
+
+desc="CryptPad Service"
+
+load_rc_config ${name}
+
+start() {
+
+/bin/mkdir -p /var/run/cryptpad
+/usr/sbin/chown cryptpad:cryptpad /var/run/cryptpad
+
+/usr/bin/su cryptpad -c "export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:~/bin && cd /home/cryptpad/cryptpad && /usr/sbin/daemon -T ${name} -P /var/run/cryptpad/${name}_supervisor.pid -p  /var/run/cryptpad/${name}.pid -f -S -r /usr/local/bin/node server"
+
+}
+
+stop() {
+
+/bin/kill -9 `cat /var/run/cryptpad/${name}_supervisor.pid`
+/bin/kill -15 `cat /var/run/cryptpad/${name}.pid`
+
+}
+
+run_rc_command "$1"
+

lib/commands/channel.js

@@ -224,7 +224,7 @@ Channel.deleteMailboxMessage = function (Env, data, cb) {
     Env.msgStore.deleteChannelLine(channelId, hash, function (msg) {
         // Check if you're allowed to delete this hash
         try {
-            const mySecret = new Uint8Array(32);
+            const mySecret = Env.curvePrivate;
             const msgBytes = Nacl.util.decodeBase64(msg).subarray(64); // Remove signature
             const theirPublic = msgBytes.subarray(24,56); // 0-24 = nonce; 24-56=publickey (32 bytes)
             const hashBytes = Nacl.box.open(proofBytes, nonce, theirPublic, mySecret);

lib/decrees.js

@@ -329,8 +329,6 @@ commands.ADD_INSTALL_TOKEN = function (Env, args) {
 
     var token = args[0];
 
-    // XXX check length, etc. ?
-
     Env.installToken = token;
 
     return true;

lib/env.js

@@ -13,6 +13,8 @@ const Util = require("./common-util");
 const Package = require("../package.json");
 const Path = require("path");
 
+const Nacl = require("tweetnacl/nacl-fast");
+
 var canonicalizeOrigin = function (s) {
     if (typeof(s) === 'undefined') { return; }
     return (s || '').trim().replace(/\/+$/, '');
@@ -68,6 +70,8 @@ module.exports.create = function (config) {
         permittedEmbedders = permittedEmbedders.trim();
     }
 
+    const curve = Nacl.box.keyPair();
+
     const Env = {
         protocol: new URL(httpUnsafeOrigin).protocol,
 
@@ -213,6 +217,9 @@ module.exports.create = function (config) {
         lastEviction: +new Date(),
         evictionReport: {},
         commandTimers: {},
+
+        curvePrivate: curve.secretKey,
+        curvePublic: Nacl.util.encodeBase64(curve.publicKey),
     };
 
     (function () {

lib/eviction.js

@@ -3,6 +3,10 @@ var Bloom = require("@mcrowe/minibloom");
 var Util = require("../lib/common-util");
 var Pins = require("../lib/pins");
 var Keys = require("./keys");
+var Path = require('node:path');
+var config = require("./load-config");
+var Fs = require("node:fs");
+var Fse = require("fs-extra");
 
 var getNewestTime = function (stats) {
     return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
@@ -32,6 +36,7 @@ Env = {
 
 // the number of ms artificially introduced between CPU-intensive operations
 var THROTTLE_FACTOR = 10;
+var PROGRESS_FACTOR = 1000;
 
 var evictArchived = function (Env, cb) {
     var Log;
@@ -70,6 +75,103 @@ var evictArchived = function (Env, cb) {
         blobs = Env.blobStore;
     };
 
+    var migrateBlobRoot = function (from, to) {
+        // only migrate subpaths, leave everything else alone
+        if (!Path.dirname(from).startsWith(Path.dirname(to))) { return; }
+
+        // expects a directory
+        var recurse = function (relativePath) {
+            var src = Path.join(from, relativePath);
+            var children;
+            try {
+                children = Fs.readdirSync(src);
+            } catch (err) {
+                if (err.code === 'ENOENT') { return; }
+                // if you can't read a directory's contents
+                // then nothing else will work, so just abort
+                Log.verbose("EVICT_ARCHIVED_NOT_DIRECTORY", {
+                    error: err,
+                });
+                return;
+            }
+
+            var dest;
+            if (children.length === 0) {
+                try {
+                    Fse.removeSync(src);
+                } catch (err2) {
+                    Log.error('EVICT_ARCHIVED_EMPTY_DIR_REMOVAL', {
+                        error: err2,
+                    });
+                    // removal is non-essential, so we can continue
+                }
+            } else {
+                // make an equivalent path in the target directory
+                dest = Path.join(to, relativePath);
+
+                try {
+                    Fse.mkdirpSync(dest);
+                } catch (err3) {
+                    Log.error("EVICT_ARCHIVED_BLOB_MIGRATION", {
+                        error: err3,
+                    });
+
+                    // failure to create the host directory
+                    // will cause problems when we try to move
+                    // so bail out here
+                    return;
+                }
+            }
+
+            children.forEach(function (child) {
+                var childSrcPath = Path.join(src, child);
+                var stat = Fs.statSync(childSrcPath);
+                if (stat.isDirectory()) {
+                    return void recurse(Path.join(relativePath, child));
+                }
+
+                var childDestPath = Path.join(dest, child);
+
+                try {
+                    Log.verbose("EVICT_ARCHIVED_MOVE_FROM_DEPRECATED_PATH", {
+                        from: childSrcPath,
+                        to: childDestPath,
+                    });
+                    Fse.moveSync(childSrcPath, childDestPath, {
+                        overwrite: false,
+                    });
+                } catch (err4) {
+                    Log.error('EVICT_ARCHIVED_MOVE_FAILURE', {
+                        error: err4,
+                    });
+                }
+            });
+        };
+        recurse('');
+    };
+
+/*  In CryptPad 5.2.0 we merged a patch which converted
+    all of CryptPad's root filepaths to their absolute form,
+    rather than the relative paths we'd been using until then.
+    Unfortunately, we overlooked a case where two absolute
+    paths were concatenated together, resulting in blobs being
+    archived to an incorrect path.
+
+    This migration detects evidence of incorrect archivals
+    and moves such archived files to their intended location
+    before continuing with the normal eviction procedure.
+*/
+    var migrateIncorrectBlobs = function () {
+        var incorrectPaths = [
+            Path.join(Env.paths.archive, config.blobPath),
+            Path.join(Env.paths.archive, Path.resolve(config.blobPath))
+        ];
+        var correctPath = Path.join(Env.paths.archive, 'blob');
+        incorrectPaths.forEach(root => {
+            migrateBlobRoot(root, correctPath);
+        });
+    };
+
     var removeArchivedChannels = function (w) {
         // this block will iterate over archived channels and removes them
         // if they've been in cold storage for longer than your configured archive time
@@ -142,7 +244,7 @@ var evictArchived = function (Env, cb) {
                 Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
                 return void next();
             }
-            if (item && getNewestTime(item) > retentionTime) { return void next(); }
+            if (item && item.mtime > retentionTime) { return void next(); }
             blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
                 if (err) {
                     Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
@@ -169,7 +271,7 @@ var evictArchived = function (Env, cb) {
                 Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
                 return void next();
             }
-            if (item && getNewestTime(item) > retentionTime) { return void next(); }
+            if (item && item.mtime > retentionTime) { return void next(); }
             blobs.remove.archived.blob(item.blobId, function (err) {
                 if (err) {
                     Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
@@ -186,11 +288,12 @@ var evictArchived = function (Env, cb) {
     };
 
     nThen(loadStorage)
+    .nThen(migrateIncorrectBlobs)
     .nThen(removeArchivedChannels)
     .nThen(removeArchivedBlobProofs)
     .nThen(removeArchivedBlobs)
     .nThen(function () {
-        cb();
+        cb(void 0, report);
     });
 };
 
@@ -264,8 +367,8 @@ module.exports = function (Env, cb) {
 
         TODO make this configurable ?
     */
-    var BLOOM_CAPACITY = (1 << 20) - 1; // over a million items
-    var BLOOM_ERROR = 1 / 10000;  // an error rate of one in a thousand
+    var BLOOM_CAPACITY = (1 << 22) - 1; // over two million items
+    var BLOOM_ERROR = 1 / 10000;  // an error rate of one in ten thousand
 
     // we'll use one filter for the set of active documents
     var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
@@ -289,6 +392,12 @@ module.exports = function (Env, cb) {
         var active = 0;
         var handler = function (err, item, cb) {
             channels++;
+            if (channels % PROGRESS_FACTOR === 0) {
+                Log.info('EVICT_CHANNEL_CATEGORIZATION_PROGRESS', {
+                    channels: channels,
+                });
+            }
+
             if (err) {
                 Log.error('EVICT_CHANNEL_CATEGORIZATION', err);
                 return void cb();
@@ -315,6 +424,7 @@ module.exports = function (Env, cb) {
             });
         };
 
+        Log.info('EVICT_CHANNEL_ACTIVITY_START', 'Assessing channel activity');
         store.listChannels(handler, w(done));
     };
 
@@ -322,9 +432,16 @@ module.exports = function (Env, cb) {
         var n_blobs = 0;
         var active = 0;
 
+        Log.info('EVICT_BLOBS_ACTIVITY_START', 'Assessing blob activity');
         blobs.list.blobs(function (err, item, next) {
             next = Util.mkAsync(next, THROTTLE_FACTOR);
             n_blobs++;
+            if (n_blobs % PROGRESS_FACTOR === 0) {
+                Log.info('EVICT_BLOB_CATEGORIZATION_PROGRESS', {
+                    blobs: n_blobs,
+                });
+            }
+
             if (err) {
                 Log.error("EVICT_BLOB_CATEGORIZATION", err);
                 return void next();
@@ -333,7 +450,7 @@ module.exports = function (Env, cb) {
                 next();
                 return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item);
             }
-            if (getNewestTime(item) > inactiveTime) {
+            if (item.mtime > inactiveTime) {
                 activeDocs.add(item.blobId);
                 active++;
                 return void next();
@@ -393,6 +510,11 @@ module.exports = function (Env, cb) {
         var handler = function (content, id, next) {
             next = Util.mkAsync(next, THROTTLE_FACTOR);
             accounts++;
+            if (accounts % PROGRESS_FACTOR === 0) {
+                Log.info('EVICT_ACCOUNT_CATEGORIZATION_PROGRESS', {
+                    accounts: accounts,
+                });
+            }
 
             var mtime = content.latest;
             var pinList = Object.keys(content.pins);
@@ -449,6 +571,7 @@ module.exports = function (Env, cb) {
             });
         };
 
+        Log.info('EVICT_ACCOUNTS_ACTIVITY_START', 'Assessing account activity');
         Pins.load(w(done), {
             pinPath: Env.paths.pin,
             handler: handler,
@@ -460,6 +583,8 @@ module.exports = function (Env, cb) {
     // if they have not been accessed within the specified retention time
         var removed = 0;
         var total = 0;
+
+        Log.info('EVICT_BLOB_START', {});
         blobs.list.blobs(function (err, item, next) {
             next = Util.mkAsync(next, THROTTLE_FACTOR);
             if (err) {
@@ -471,13 +596,19 @@ module.exports = function (Env, cb) {
                 return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item);
             }
             total++;
+            if (total % PROGRESS_FACTOR === 0) {
+                Log.info("EVICT_BLOB_PROGRESS", {
+                    blobs: total,
+                });
+            }
+
             if (pinnedDocs.test(item.blobId)) { return void next(); }
             if (activeDocs.test(item.blobId)) { return void next(); }
 
             // This seems redundant because we're already checking the bloom filter
             // but we can't implement a 'fast mode' for the iterator
             // unless we address this race condition with this last-minute double-check
-            if (getNewestTime(item) > inactiveTime) { return void next(); }
+            if (item.mtime > inactiveTime) { return void next(); }
 
             removed++;
             blobs.archive.blob(item.blobId, function (err) {
@@ -504,6 +635,9 @@ module.exports = function (Env, cb) {
     // iterate over blob proofs and remove them
     // if they don't correspond to a pinned or active file
         var removed = 0;
+        var total = 0;
+
+        Log.info("EVICT_ARCHIVE_INACTIVE_BLOB_PROOFS_START", {});
         blobs.list.proofs(function (err, item, next) {
             next = Util.mkAsync(next, THROTTLE_FACTOR);
             if (err) {
@@ -514,8 +648,16 @@ module.exports = function (Env, cb) {
                 next();
                 return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item);
             }
+            total++;
+
+            if (total % PROGRESS_FACTOR === 0) {
+                Log.info('EVICT_BLOB_PROOF_PROGRESS', {
+                    proofs: total,
+                });
+            }
+
             if (pinnedDocs.test(item.blobId)) { return void next(); }
-            if (getNewestTime(item) > inactiveTime) { return void next(); }
+            if (item.mtime > inactiveTime) { return void next(); }
             nThen(function (w) {
                 blobs.size(item.blobId, w(function (err, size) {
                     if (err) {
@@ -539,7 +681,10 @@ module.exports = function (Env, cb) {
                 });
             });
         }, w(function () {
-            Log.info("EVICT_BLOB_PROOFS_REMOVED", removed);
+            Log.info("EVICT_BLOB_PROOFS_REMOVED", {
+                removed,
+                total,
+            });
         }));
     };
 
@@ -550,10 +695,21 @@ module.exports = function (Env, cb) {
         var handler = function (err, item, cb) {
             cb = Util.mkAsync(cb, THROTTLE_FACTOR);
             channels++;
+            if (channels % PROGRESS_FACTOR === 0) {
+                Log.info('EVICT_INACTIVE_CHANNELS_PROGRESS', {
+                    channels,
+                    archived,
+                });
+            }
+
             if (err) {
                 Log.error('EVICT_CHANNEL_ITERATION', err);
                 return void cb();
             }
+
+            // ignore the special admin broadcast channel
+            if (item.channel.length === 33) { return void cb(); }
+
             // check if the database has any ephemeral channels
             // if it does it's because of a bug, and they should be removed
             if (item.channel.length === 34) {
@@ -581,7 +737,7 @@ module.exports = function (Env, cb) {
                 // because it might have been created after the initial activity scan
                 store.getChannelStats(item.channel, w(function (err, newerItem) {
                     if (err) { return; }
-                    if (item && getNewestTime(newerItem) > retentionTime) {
+                    if (newerItem && getNewestTime(newerItem) > retentionTime) {
                         // it's actually active, so don't archive it.
                         w.abort();
                         cb();
@@ -606,9 +762,13 @@ module.exports = function (Env, cb) {
 
         var done = function () {
             report.channelsArchived = archived;
-            return void Log.info('EVICT_CHANNELS_ARCHIVED', archived);
+            return void Log.info('EVICT_CHANNELS_ARCHIVED', {
+                channels,
+                archived,
+            });
         };
 
+        Log.info('EVICT_INACTIVE_CHANNELS_START', {});
         store.listChannels(handler, w(done), true); // using a hacky "fast mode" since we only need the channel id
     };
 

lib/hk-util.js

@@ -428,6 +428,7 @@ const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => {
 
     // lastKnownhash === -1 means we want the complete history
     if (lastKnownHash === -1) { return void cb(null, 0); }
+
     let offset = -1;
     nThen((waitFor) => {
         getIndex(Env, channelName, waitFor((err, index) => {
@@ -436,8 +437,16 @@ const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => {
             // check if the "hash" the client is requesting exists in the index
             const lkh = index.offsetByHash[lastKnownHash];
 
-            // fall through to the next block if the offset of the hash in question is not in memory
-            if (lastKnownHash && typeof(lkh) !== "number") { return; }
+            // lastKnownHash requested but not found in the index
+            if (lastKnownHash && typeof(lkh) !== "number") {
+                // No checkpoint: may be a non-chainpad channel
+                if (!index.cpIndex.length) {
+                    return;
+                }
+                // Hash too old or no longer exists, empty cache
+                waitFor.abort();
+                return void cb(new Error('EUNKNOWN'));
+            }
 
             // If we have a lastKnownHash or we didn't ask for one, we don't need the next blocks
             waitFor.abort();
@@ -465,18 +474,13 @@ const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => {
                 return void cb(new Error('EUNKNOWN'));
             }
 
-            // If we asked for a lastKnownHash but didn't find it AND if
-            // this channel has checkpoints, send EUNKNOWN so that the
-            // client can ask for normal history (without lastKnownHash)
-            if (lastKnownHash && !lkh && index.cpIndex.length) {
-                waitFor.abort();
-                return void cb(new Error('EUNKNOWN'));
-            }
-
             // Otherwise use our lastKnownHash
             cb(null, lkh);
         }));
     }).nThen((w) => {
+        // XXX entire block and getHashOffset to remove?
+
+
         // skip past this block if the offset is anything other than -1
         // this basically makes these first two nThen blocks behave like if-else
         if (offset !== -1) { return; }

lib/storage/blob.js

@@ -19,7 +19,11 @@ var isValidId = function (id) {
 // helpers
 
 var prependArchive = function (Env, path) {
-    return Path.join(Env.archivePath, path);
+    // Env has an absolute path to the blob storage
+    // we want the path to the blob relative to that
+    var relativePathToBlob = Path.relative(Env.blobPath, path);
+    // the new path structure is the same, but relative to the blob archive root
+    return Path.join(Env.archivePath, 'blob', relativePathToBlob);
 };
 
 // /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
@@ -492,7 +496,7 @@ BlobStore.create = function (config, _cb) {
             if (e) { CB(e); }
         }));
 
-        Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) {
+        Fse.mkdirp(Path.join(Env.archivePath, './blob'), w(function (e) {
             if (e) { CB(e); }
         }));
     }).nThen(function (w) {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "cryptpad",
   "description": "realtime collaborative visual editor with zero knowlege server",
-  "version": "5.2.1",
+  "version": "5.3.0",
   "license": "AGPL-3.0+",
   "repository": {
     "type": "git",
@@ -59,9 +59,11 @@
   },
   "devDependencies": {
     "jshint": "^2.13.4",
-    "less": "3.7.1",
-    "lesshint": "6.3.7",
-    "selenium-webdriver": "^3.6.0"
+    "lesshint": "6.3.7"
+  },
+  "overrides": {
+    "glob-parent": "5.1.2",
+    "set-value": "4.0.1"
   },
   "scripts": {
     "install:components": "node scripts/copy-components.js",

readme.md

@@ -81,6 +81,8 @@ We love Open Source and we love contribution. Learn more about [contributing](ht
 
 If you have any questions or comments, or if you're interested in contributing to CryptPad, come say hi in our [Matrix channel](https://app.element.io/#/room/#cryptpad:matrix.xwiki.com).
 
+This project is tested with [BrowserStack](https://www.browserstack.com/).
+
 # License
 
 ![AGPL logo](https://www.gnu.org/graphics/agplv3-155x51.png "GNU Affero General Public License")

scripts/build.js

@@ -229,7 +229,7 @@ appIndexesToBuild.forEach(function (app) {
 
     write(built, `./www/${app}/index.html`);
 
-    // XXX preloading version for inner.html
+    // TODO preloading version for inner.html
 });
 
 var instance;

scripts/evict-archived.js

@@ -98,7 +98,8 @@ nThen(function (w) {
 
     }));
 }).nThen(function (w) {
-    Eviction.archived(Env, w(function () {
-
+    Eviction.archived(Env, w(function (err, report) {
+        if (!report) { return; }
+        Env.Log.info('EVICT_ARCHIVED_FINAL_REPORT', report);
     }));
 });

scripts/evict-inactive.js

@@ -97,7 +97,8 @@ nThen(function (w) {
 
     }));
 }).nThen(function (w) {
-    Eviction(Env, w(function () {
-
+    Eviction(Env, w(function (err, report) {
+        if (!report) { return; }
+        Env.Log.info('EVICT_INACTIVE_FINAL_REPORT', report);
     }));
 });

scripts/install.js

@@ -22,7 +22,6 @@ nThen(function (w) {
             console.log('Existing token');
             token = Env.installToken;
         }
-        // XXX IF ADMINS ABORT?
     }));
 }).nThen(function (w) {
     if (Env.installToken) { return; }

scripts/translations/unused-translations.js

@@ -32,6 +32,7 @@ var grep = function (pattern, cb) {
         'www/common/onlyoffice/v2b*',
         'www/common/onlyoffice/v4*',
         'www/common/onlyoffice/v5*',
+        'www/common/onlyoffice/v6*',
         'www/common/onlyoffice/x2t/*',
         //'www/common/onlyoffice/build/*',
         'www/lib/*',

server.js

@@ -256,6 +256,7 @@ var serveBroadcast = makeRouteCache(function () {
     return [
         'define(function(){',
         'return ' + JSON.stringify({
+            curvePublic: Env.curvePublic, // XXX could be in api/config but issue with static config
             lastBroadcastHash: Env.lastBroadcastHash,
             surveyURL: Env.surveyURL,
             maintenance: maintenance

www/calendar/inner.js

@@ -1397,8 +1397,8 @@ ICS ==> create a new event with the same UID and a RECURRENCE-ID field (with a v
         if (updatedOn) { delete APP.recurrenceRule._next; }
         APP.wasRecurrent = Boolean(APP.recurrenceRule);
 
-// XXX TEST
 /*
+// Test data:
 APP.recurrenceRule = {
     freq: 'yearly',
     interval: 2,

www/code/app-code.less

@@ -146,6 +146,9 @@
                     flex: 1;
                     max-width: 100%;
                     resize: none;
+                    .CodeMirror-sizer > div {
+                        padding-bottom: 100px;
+                    }
                 }
                 #cp-app-code-preview {
                     display: none !important;

www/common/application_config_internal.js

@@ -280,7 +280,7 @@ define(function() {
     // the driveless mode by changing the following value to "false"
     AppConfig.allowDrivelessMode = true;
 
-    AppConfig.emojiAvatars = '🙈 🦀 🐞 🦋 🐬 🐋 🐢 🦉 🦆 🐧 🦡 🦘 🦨 🦦 🦥 🐼 🐻 🦝 🦓 🐄 💮️ 🐙️ 🌸️ 🌻️ 🐝️ 🐐 🦙 🦒 🐘 🦏 🐁 🐹 🐰 🦫 🦔 🐨 🐱 🐺 👺 👹 👽 👾 🤖'.split(/\s+/);
+    AppConfig.emojiAvatars = '🐵 🐒 🐶 🐩 🐺 🐱 🐯 🐴 🐎 🐮 🐷 🐗 🐑 🐫 🐘 🐭 🐹 🐰 🐻 🐨 🐼 🐔 🐣 🐥 🐢 🐍 🐲 🐳 🐬 🐟 🐠 🐡 🐙 🐚 🐌 🐛 🐝 🐞 💐 🌸 💮 🌹 🌺 🌻 🌼 🌷 🌱 🌴 🌵 🌾 🌿 🍀 🍁 🍂 🍃 🍄 💫 🌛 ⛄ 🔥 💧 🌊 🎃 👹 👺 👻 👽 👾'.split(/\s+/);
 
     return AppConfig;
 });

www/common/boot2.js

@@ -1,40 +1,50 @@
-(function () {
-try {
-    var isDarkOS = function () {
-        try {
-            return window.matchMedia('(prefers-color-scheme: dark)').matches;
-        } catch (e) { return false; }
-    };
-    var flush = window.CryptPad_flushCache = function () {
-        Object.keys(localStorage).forEach(function (k) {
-            if (k.indexOf('CRYPTPAD_CACHE|') !== 0 && k.indexOf('LESS_CACHE') !== 0) { return; }
-            delete localStorage[k];
-        });
-    };
-    var os = isDarkOS() ? 'dark' : 'light';
-    var key = 'CRYPTPAD_STORE|colortheme';
-    window.CryptPad_theme = localStorage[key] || os;
-    if (!localStorage[key]) {
-        // We're using OS theme, check if we need to change
-        if (os !== localStorage[key+'_default']) {
-            console.warn('New OS theme, flush cache');
-            flush();
-            localStorage[key+'_default'] = os;
-        }
-    }
-    if (window.CryptPad_theme === 'dark') {
-        var s = document.createElement('style');
-        s.innerHTML = 'body { background: black; }';
-        document.body.appendChild(s);
-    }
-} catch (e) { console.error(e); }
-})();
-
-// This is stage 1, it can be changed but you must bump the version of the project.
 define([
-    '/common/requireconfig.js'
-], function (RequireConfig) {
+    '/common/requireconfig.js', 
+    '/customize/application_config.js'
+], function (RequireConfig, AppConfig) {
 
+    // if an AppConfig.defaultDarkTheme variable is added to application_config.js and set to true, this sets the theme to dark by default irrespective of browser settings
+    var checkDefaultDarkTheme = function () {
+        if (AppConfig.defaultDarkTheme) {
+            return 'dark';
+        }
+        var isDarkOS = function () {
+            try {
+                return window.matchMedia('(prefers-color-scheme: dark)').matches;
+            } catch (e) { return false; }
+        };
+        return isDarkOS() ? 'dark' : 'light';
+    };
+    var os = checkDefaultDarkTheme();
+
+    try {
+        var flush = window.CryptPad_flushCache = function () {
+            Object.keys(localStorage).forEach(function (k) {
+                if (k.indexOf('CRYPTPAD_CACHE|') !== 0 && k.indexOf('LESS_CACHE') !== 0) { return; }
+                delete localStorage[k];
+            });
+        };
+
+        var key = 'CRYPTPAD_STORE|colortheme';
+        window.CryptPad_theme = localStorage[key] || os;
+        if (!localStorage[key]) {
+            // We're using OS theme, check if we need to change
+            if (os !== localStorage[key+'_default']) {
+                console.warn('New OS theme, flush cache');
+                flush();
+                localStorage[key+'_default'] = os;
+            }
+        }
+        if (window.CryptPad_theme === 'dark') {
+            var s = document.createElement('style');
+            s.innerHTML = 'body { background: black; }';
+            document.body.appendChild(s);
+        }
+    } catch (e) { console.error(e); }
+
+
+
+    // This is stage 1, it can be changed but you must bump the version of the project.
     require.config(RequireConfig());
 
     // most of CryptPad breaks if you don't support isArray

www/common/common-ui-elements.js

@@ -2591,14 +2591,12 @@ define([
                     icon: h('span.cptools.cptools-new-template')
                 });
             }*/
-            if (!privateData.newTemplate) {
-                allData.unshift({
-                    name: Messages.creation_noTemplate,
-                    id: 0,
-                    //icon: h('span.fa.fa-file')
-                    icon: UI.getFileIcon({type: type})
-                });
-            }
+            allData.unshift({
+                name: Messages.creation_noTemplate,
+                id: 0,
+                //icon: h('span.fa.fa-file')
+                icon: UI.getFileIcon({type: type})
+            });
             var redraw = function (index) {
                 if (index < 0) { i = 0; }
                 else if (index > allData.length - 1) { return; }

www/common/cryptpad-common.js

@@ -219,10 +219,12 @@ define([
             var n = Nthen;
             var nacl, theirs;
             n = n(function (waitFor) {
-                require(['/components/tweetnacl/nacl-fast.min.js'], waitFor(function () {
+                require([
+                    '/api/broadcast?'+ (+new Date()),
+                    '/components/tweetnacl/nacl-fast.min.js'
+                ], waitFor(function (Broadcast) {
                     nacl = window.nacl;
-                    var s = new Uint8Array(32);
-                    theirs = nacl.box.keyPair.fromSecretKey(s);
+                    theirs = nacl.util.decodeBase64(Broadcast.curvePublic);
                 }));
             }).nThen;
             var toDelete = [];
@@ -236,7 +238,7 @@ define([
                     var curve = answer.curvePrivate;
                     var mySecret = nacl.util.decodeBase64(curve);
                     var nonce = nacl.randomBytes(24);
-                    var proofBytes = nacl.box(h, nonce, theirs.publicKey, mySecret);
+                    var proofBytes = nacl.box(h, nonce, theirs, mySecret);
                     var proof = nacl.util.encodeBase64(nonce) +'|'+ nacl.util.encodeBase64(proofBytes);
                     var lineData = {
                         channel: data.channel,

www/common/inner/access.js

@@ -332,6 +332,12 @@ define([
                                                                           : Messages.error;
                             return void UI.warn(text);
                         }
+                        sframeChan.query('Q_ACCEPT_OWNERSHIP', data, function (err, res) {
+                            if (err || (res && res.error)) {
+                                return void console.error(err || res.error);
+                            }
+                            UI.log(Messages.saved);
+                        });
                     }));
                 }
             }).nThen(function (waitFor) {
@@ -867,7 +873,7 @@ define([
             // In the properties, we should have the edit href if we know it.
             // We should know it because the pad is stored, but it's better to check...
             //if (!data.noEditPassword && !opts.noEditPassword && owned && data.href) {
-            if (!data.noEditPassword && !opts.noEditPassword && owned && data.href && parsed.type !== "form") { // XXX password change in forms block responses (validation & decryption)
+            if (!data.noEditPassword && !opts.noEditPassword && owned && data.href && parsed.type !== "form") { // TODO password change in forms block responses (validation & decryption)
                 var isOO = parsed.type === 'sheet';
                 var isFile = parsed.hashData.type === 'file';
                 var isSharedFolder = parsed.type === 'drive';

www/common/onlyoffice/inner.js

@@ -61,7 +61,7 @@ define([
     var CHECKPOINT_INTERVAL = 100;
     var FORCE_CHECKPOINT_INTERVAL = 10000;
     var DISPLAY_RESTORE_BUTTON = false;
-    var NEW_VERSION = 5; // version of the .bin, patches and ChainPad formats
+    var NEW_VERSION = 6; // version of the .bin, patches and ChainPad formats
     var PENDING_TIMEOUT = 30000;
     var CURRENT_VERSION = X2T.CURRENT_VERSION;
 
@@ -1349,6 +1349,8 @@ define([
                 type: "saveChanges",
                 changes: parseChanges(JSON.stringify(aRes)),
                 changesIndex: ooChannel.cpIndex || 0,
+                startSaveChanges: true,
+                endSaveChanges: true,
                 locks: getUserLock(getId(), true),
                 excelAdditionalInfo: null
             };
@@ -2134,9 +2136,7 @@ Uncaught TypeError: Cannot read property 'calculatedType' of null
         var exportXLSXFile = function() {
             var text = getContent();
             var suggestion = Title.suggestTitle(Title.defaultTitle);
-            var ext = ['.xlsx', '.ods', '.bin',
-            //'.csv', // XXX 4.11.0
-            '.pdf'];
+            var ext = ['.xlsx', '.ods', '.bin', '.pdf'];
             var type = common.getMetadataMgr().getPrivateData().ooType;
             var warning = '';
             if (type==="presentation") {
@@ -2943,6 +2943,23 @@ Uncaught TypeError: Cannot read property 'calculatedType' of null
                     }
                     readOnly = true;
                 }
+            } else if (content && content.version <= 5) {
+                version = 'v5/';
+                APP.migrate = true;
+                // Registedred ~~users~~ editors can start the migration
+                if (common.isLoggedIn() && !readOnly) {
+                    content.migration = true;
+                    APP.onLocal();
+                } else {
+                    msg = h('div.alert.alert-warning.cp-burn-after-reading', Messages.oo_sheetMigration_anonymousEditor);
+                    if (APP.helpMenu) {
+                        $(APP.helpMenu.menu).after(msg);
+                    } else {
+                        $('#cp-app-oo-editor').prepend(msg);
+                    }
+                    readOnly = true;
+                }
+
             }
             // NOTE: don't forget to also update the version in 'EV_OOIFRAME_REFRESH'