diff --git a/CHANGELOG.md b/CHANGELOG.md index 566758ad4..137c1665b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,44 @@ +# Quagga release (3.16.0) + +## Goals + +We've continued to keep a close eye on server performance since our last release while making minimal changes. Our goal for this release has been to improve server scalability further while also addressing user needs with updates to our client code. + +We were pleasantly surprised to receive a pull request implementing a basic version of [author colors](https://github.com/xwiki-labs/cryptpad/issues/41) in our code editor. Since it was nearly ready to go we set some time aside to polish it up a little bit to include it in this release. + +## Update notes + +We've updated the example nginx config in order to include an `Access-Control-Allow-Origin` header that was not included. We've also added a new configuration point in response to [this issue](https://github.com/xwiki-labs/cryptpad/issues/529) about the server's child processes using too many threads. Administrators may not set a maximum number of child processes via `config.js` using `maxWorkers: `. We recommend using one less than the number of available cores, though one worker should be sufficient as long as your server is not under heavy load. + +As usual, updating from the previous release can be accomplished by: + +1. stopping your server +2. pulling the latest code with git +3. installing clientside dependencies with `bower update` +4. installing serverside dependencies with `npm i` +5. restarting your server + +## Features + +* As mentioned above, we've built upon a very helpful PR to introduce author colors in our code editor. It's still experimental, but you can test it out by enabling author colors in a code pad via the pad's properties modal. +* Serverside performance optimizations + * Automatically expiring pads work by creating a task to be run at the target date. This process involves a little bit of hashing, so we've changed it to be run in the worker. + * The act of deleting a file from the server actually moves it to an archive which is not publicly accessible. These archived files are regularly cleaned up if you run `scripts/evict-inactive.js`. Unfortunately, moving files is more expensive than deletion, so we've noticed spikes in CPU when users delete many files at once (like when emptying the trash from their drive). To avoid such spikes while the server is already under load we've implemented per-user queues for deletion. + * We've also noticed that when we restart our server while it is under heavy load some queries can time out due to many users requesting history at once. We've implemented another queue to delegate tasks to workers in the order that they are received. We need to observe how this system performs in practice, so there might be small tweaks as we get more data. + * As noted above, we've made the number of workers configurable. At the same time we unified two types of workers into one, cutting the number of workers in half. +* We've added a new admin RPC call to request some information about the server's memory usage to help us debug what seems to be a small memory leak. +* Most of our editors were previously loaded with two more iframes on the page in addition to our main sandboxed iframe. These separate frames ensure that encryption keys are not exposed to the same iframe responsible for displaying the rest of CryptPad's UI. One was responsible for loading the "filepicker" for inserting media into your documents, the other was responsible for handling encryption keys for the share modal. Since we wanted to add two new functions using iframes in the same manner we took the opportunity to come up with a generic solution using only one iframe for these separate modals, since they all have the same level of privilege to the sensitive data we're trying to protect. +* Our mermaidjs integration has been customized to be a little easier on the eyes. We focused in particular on GANTT charts, though other charts should be more appealing as well, especially in the new "lightbox" UI introduced in our last release. +* We now prompt unregistered users to register or log in when they use the spreadsheet editor. For context, unregistered users don't benefit from all of the same features as registered users, and this makes a few performance optimizations impossible. +* Finally, we've continued to receive translations from contributors in Catalan, German, and Dutch. + +## Bug fixes + +* We noticed that under certain conditions clients were sending metadata queries to the server for documents that don't have metadata. We've implemented some stricter checks to prevent these useless queries. +* We've implemented a temporary fix for our rich text editor to solve [this issue](https://github.com/xwiki-labs/cryptpad/issues/526) related to conflicting font-size and header styles. +* We also accepted [this PR](https://github.com/xwiki-labs/cryptpad/pull/525) to tolerate server configurations specifying a `defaultStorageLimit` of 0. +* Finally, we noticed that embedded media occasionally stopped responding correctly to right-click events due to a problem with our in-memory cache. It has since been fixed. + # PigFootedBandicoot release (3.15.0) ## Goals diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index 867dbacc9..1bd592d63 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -74,6 +74,7 @@ } } media-tag { + cursor: pointer; * { max-width: 100%; } @@ -93,6 +94,7 @@ pre.mermaid { svg { max-width: 100%; + cursor: pointer; } } } diff --git a/customize.dist/src/less2/include/modals-ui-elements.less b/customize.dist/src/less2/include/modals-ui-elements.less index 0cc580ec0..c407c8523 100644 --- a/customize.dist/src/less2/include/modals-ui-elements.less +++ b/customize.dist/src/less2/include/modals-ui-elements.less @@ -26,6 +26,11 @@ // Properties modal .cp-app-prop { margin-bottom: 10px; + .cp-app-prop-hint { + color: @cryptpad_text_col; + font-size: 0.8em; + margin-bottom: 5px; + } .cp-app-prop-size-container { height: 20px; background-color: @colortheme_logo-2; diff --git a/customize.dist/src/less2/include/toolbar.less b/customize.dist/src/less2/include/toolbar.less index 73e608c4a..542d3f35b 100644 --- a/customize.dist/src/less2/include/toolbar.less +++ b/customize.dist/src/less2/include/toolbar.less @@ -1181,6 +1181,7 @@ &.fa-print { order: 4; } &.fa-arrows-h { order: 5; } &.fa-cog { order: 5; } + &.fa-paint-brush { order: 5; } &.fa-info-circle { order: 6; } &.fa-help { order: 7; } } diff --git a/www/code/inner.js b/www/code/inner.js index 28b90b07c..7b654a3b1 100644 --- a/www/code/inner.js +++ b/www/code/inner.js @@ -3,16 +3,20 @@ define([ '/common/diffMarked.js', '/bower_components/nthen/index.js', '/common/sframe-common.js', + '/common/hyperscript.js', '/common/sframe-app-framework.js', '/common/sframe-common-codemirror.js', + '/common/common-interface.js', '/common/common-util.js', '/common/common-hash.js', + '/code/markers.js', '/common/modes.js', '/common/visible.js', '/common/TypingTests.js', '/customize/messages.js', 'cm/lib/codemirror', + 'css!cm/lib/codemirror.css', 'css!cm/addon/dialog/dialog.css', 'css!cm/addon/fold/foldgutter.css', @@ -46,10 +50,13 @@ define([ DiffMd, nThen, SFCommon, + h, Framework, SFCodeMirror, + UI, Util, Hash, + Markers, Modes, Visible, TypingTest, @@ -273,6 +280,54 @@ define([ }; }; + var mkColorByAuthor = function (framework, markers) { + var common = framework._.sfCommon; + var $cbaButton = framework._.sfCommon.createButton(null, true, { + icon: 'fa-paint-brush', + text: Messages.cba_title, + name: 'cba' + }, function () { + var div = h('div'); + var $div = $(div); + var content = h('div', [ + h('h4', Messages.cba_properties), + h('p', Messages.cba_hint), + div + ]); + var setButton = function (state) { + var button = h('button.btn'); + var $button = $(button); + $div.html('').append($button); + if (state) { + // Add "enable" button + $button.addClass('btn-secondary').text(Messages.cba_enable); + UI.confirmButton(button, { + classes: 'btn-primary' + }, function () { + $button.remove(); + markers.setState(true); + common.setAttribute(['code', 'enableColors'], true); + setButton(false); + }); + return; + } + // Add "disable" button + $button.addClass('btn-danger-alt').text(Messages.cba_disable); + UI.confirmButton(button, { + classes: 'btn-danger' + }, function () { + $button.remove(); + markers.setState(false); + common.setAttribute(['code', 'enableColors'], false); + setButton(true); + }); + }; + setButton(!markers.getState()); + UI.alert(content); + }); + framework._.toolbar.$drawer.append($cbaButton); + }; + var mkFilePicker = function (framework, editor, evModeChange) { evModeChange.reg(function (mode) { if (MEDIA_TAG_MODES.indexOf(mode) !== -1) { @@ -301,6 +356,20 @@ define([ var previewPane = mkPreviewPane(editor, CodeMirror, framework, isPresentMode); var markdownTb = mkMarkdownTb(editor, framework); + var markers = Markers.create({ + common: common, + framework: framework, + CodeMirror: CodeMirror, + devMode: privateData.devMode, + editor: editor + }); + + var $showAuthorColorsButton = framework._.sfCommon.createButton('', true, { + icon: 'fa-paint-brush', + }).hide(); + framework._.toolbar.$rightside.append($showAuthorColorsButton); + markers.setButton($showAuthorColorsButton); + var $print = $('#cp-app-code-print'); var $content = $('#cp-app-code-preview-content'); mkPrintButton(framework, $content, $print); @@ -323,15 +392,23 @@ define([ CodeMirror.configureTheme(common); } - //// - framework.onContentUpdate(function (newContent) { var highlightMode = newContent.highlightMode; if (highlightMode && highlightMode !== CodeMirror.highlightMode) { CodeMirror.setMode(highlightMode, evModeChange.fire); } + + // Fix the markers offsets + markers.checkMarks(newContent); + + // Apply the text content CodeMirror.contentUpdate(newContent); previewPane.draw(); + + // Apply the markers + markers.setMarks(); + + framework.localChange(); }); framework.setContentGetter(function () { @@ -339,6 +416,10 @@ define([ var content = CodeMirror.getContent(); content.highlightMode = CodeMirror.highlightMode; previewPane.draw(); + + markers.updateAuthorMarks(); + content.authormarks = markers.getAuthorMarks(); + return content; }); @@ -368,6 +449,18 @@ define([ //console.log("%s => %s", CodeMirror.highlightMode, CodeMirror.$language.val()); } + markers.ready(); + common.getPadMetadata(null, function (md) { + if (md && md.error) { return; } + if (!common.isOwned(md.owners)) { return; } + // We're the owner: add the button and enable the colors if needed + mkColorByAuthor(framework, markers); + if (newPad && Util.find(privateData, ['settings', 'code', 'enableColors'])) { + markers.setState(true); + } + }); + + var fmConfig = { dropArea: $('.CodeMirror'), body: $('body'), @@ -385,7 +478,7 @@ define([ }); framework.onDefaultContentNeeded(function () { - editor.setValue(''); //Messages.codeInitialState); + editor.setValue(''); }); framework.setFileExporter(CodeMirror.getContentExtension, CodeMirror.fileExporter); @@ -402,11 +495,14 @@ define([ framework.setNormalizer(function (c) { return { content: c.content, - highlightMode: c.highlightMode + highlightMode: c.highlightMode, + authormarks: c.authormarks }; }); - editor.on('change', framework.localChange); + editor.on('change', function( cm, change ) { + markers.localChange(change, framework.localChange); + }); framework.start(); diff --git a/www/code/markers.js b/www/code/markers.js new file mode 100644 index 000000000..fb44707e0 --- /dev/null +++ b/www/code/markers.js @@ -0,0 +1,750 @@ +define([ + '/common/common-util.js', + '/common/sframe-common-codemirror.js', + '/customize/messages.js', + '/bower_components/chainpad/chainpad.dist.js', +], function (Util, SFCodeMirror, Messages, ChainPad) { + var Markers = {}; + + /* TODO Known Issues + * 1. ChainPad diff is not completely accurate: we're not aware of the other user's cursor + position so if they insert an "a" in the middle of "aaaaa", the diff will think that + the "a" was inserted at the end of this sequence. This is not an issue for the content + but it will cause issues for the colors + 2. ChainPad doesn't always provide the good result in case of conflict (?) + e.g. Alice is inserting "pew" at offset 10, Bob is removing 1 character at offset 10 + The expected result is to have "pew" and the following character deleted + In some cases, the result is "ew" inserted and the following character not deleted + */ + + var debug = function () {}; + + var MARK_OPACITY = 0.5; + var DEFAULT = { + authors: {}, + marks: [[-1, 0, 0, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER]] + }; + + var addMark = function (Env, from, to, uid) { + if (!Env.enabled) { return; } + var author = Env.authormarks.authors[uid] || {}; + if (uid === -1) { + return void Env.editor.markText(from, to, { + css: "background-color: transparent", + attributes: { + 'data-type': 'authormark', + 'data-uid': uid + } + }); + } + uid = Number(uid); + var name = Util.fixHTML(author.name || Messages.anonymous); + var col = Util.hexToRGB(author.color); + var rgba = 'rgba('+col[0]+','+col[1]+','+col[2]+','+Env.opacity+');'; + return Env.editor.markText(from, to, { + inclusiveLeft: uid === Env.myAuthorId, + inclusiveRight: uid === Env.myAuthorId, + css: "background-color: " + rgba, + attributes: { + title: Env.opacity ? Messages._getKey('cba_writtenBy', [name]) : undefined, + 'data-type': 'authormark', + 'data-uid': uid + } + }); + }; + var sortMarks = function (a, b) { + if (!Array.isArray(b)) { return -1; } + if (!Array.isArray(a)) { return 1; } + // Check line + if (a[1] < b[1]) { return -1; } + if (a[1] > b[1]) { return 1; } + // Same line: check start offset + if (a[2] < b[2]) { return -1; } + if (a[2] > b[2]) { return 1; } + return 0; + }; + + /* Formats: + [uid, startLine, startCh, endLine, endCh] (multi line) + [uid, startLine, startCh, endCh] (single line) + [uid, startLine, startCh] (single character) + */ + var parseMark = Markers.parseMark = function (array) { + if (!Array.isArray(array)) { return {}; } + var multiline = typeof(array[4]) !== "undefined"; + var singleChar = typeof(array[3]) === "undefined"; + return { + uid: array[0], + startLine: array[1], + startCh: array[2], + endLine: multiline ? array[3] : array[1], + endCh: singleChar ? (array[2]+1) : (multiline ? array[4] : array[3]) + }; + }; + + var setAuthorMarks = function (Env, authormarks) { + if (!Env.enabled) { + Env.authormarks = {}; + return; + } + authormarks = authormarks || {}; + if (!authormarks.marks) { authormarks.marks = Util.clone(DEFAULT.marks); } + if (!authormarks.authors) { authormarks.authors = Util.clone(DEFAULT.authors); } + Env.oldMarks = Env.authormarks; + Env.authormarks = authormarks; + }; + + var getAuthorMarks = function (Env) { + return Env.authormarks; + }; + + var updateAuthorMarks = function (Env) { + if (!Env.enabled) { return; } + + // get author marks + var _marks = []; + var all = []; + + var i = 0; + Env.editor.getAllMarks().forEach(function (mark) { + var pos = mark.find(); + var attributes = mark.attributes || {}; + if (!pos || attributes['data-type'] !== 'authormark') { return; } + + + var uid = Number(attributes['data-uid']) || 0; + + all.forEach(function (obj) { + if (obj.uid !== uid) { return; } + if (obj.removed) { return; } + // Merge left + if (obj.pos.to.line === pos.from.line && obj.pos.to.ch === pos.from.ch) { + obj.removed = true; + _marks[obj.index] = undefined; + obj.mark.clear(); + mark.clear(); + mark = addMark(Env, obj.pos.from, pos.to, uid); + pos.from = obj.pos.from; + return; + } + // Merge right + if (obj.pos.from.line === pos.to.line && obj.pos.from.ch === pos.to.ch) { + obj.removed = true; + _marks[obj.index] = undefined; + obj.mark.clear(); + mark.clear(); + mark = addMark(Env, pos.from, obj.pos.to, uid); + pos.to = obj.pos.to; + } + }); + + var array = [uid, pos.from.line, pos.from.ch]; + if (pos.from.line === pos.to.line && pos.to.ch > (pos.from.ch+1)) { + // If there is more than 1 character, add the "to" character + array.push(pos.to.ch); + } else if (pos.from.line !== pos.to.line) { + // If the mark is on more than one line, add the "to" line data + Array.prototype.push.apply(array, [pos.to.line, pos.to.ch]); + } + _marks.push(array); + all.push({ + uid: uid, + pos: pos, + mark: mark, + index: i + }); + i++; + }); + _marks.sort(sortMarks); + debug('warn', _marks); + Env.authormarks.marks = _marks.filter(Boolean); + }; + + // Fix all marks located after the given operation in the provided document + var fixMarksFromOp = function (Env, op, marks, doc) { + var pos = SFCodeMirror.posToCursor(op.offset, doc); // pos of start offset + var rPos = SFCodeMirror.posToCursor(op.offset + op.toRemove, doc); // end of removed content + var removed = doc.slice(op.offset, op.offset + op.toRemove).split('\n'); // removed content + var added = op.toInsert.split('\n'); // added content + var posEndLine = pos.line + added.length - 1; // end line after op + var posEndCh = added[added.length - 1].length; // end ch after op + var addLine = added.length - removed.length; + var addCh = added[added.length - 1].length - removed[removed.length - 1].length; + if (addLine > 0) { addCh -= pos.ch; } + else if (addLine < 0) { addCh += pos.ch; } + else { posEndCh += pos.ch; } + + var splitted; + + marks.forEach(function (mark, i) { + if (!mark) { return; } + var p = parseMark(mark); + // Don't update marks located before the operation + if (p.endLine < pos.line || (p.endLine === pos.line && p.endCh < pos.ch)) { return; } + // Remove markers that have been deleted by my changes + if ((p.startLine > pos.line || (p.startLine === pos.line && p.startCh >= pos.ch)) && + (p.endLine < rPos.line || (p.endLine === rPos.line && p.endCh <= rPos.ch))) { + marks[i] = undefined; + return; + } + // Update markers that have been cropped right + if (p.endLine < rPos.line || (p.endLine === rPos.line && p.endCh <= rPos.ch)) { + mark[3] = pos.line; + mark[4] = pos.ch; + return; + } + // Update markers that have been cropped left. This markers will be affected by + // my toInsert so don't abort + if (p.startLine < rPos.line || (p.startLine === rPos.line && p.startCh < rPos.ch)) { + // If our change will split an existing mark, put the existing mark after the change + // and create a new mark before + if (p.startLine < pos.line || (p.startLine === pos.line && p.startCh < pos.ch)) { + splitted = [mark[0], mark[1], mark[2], pos.line, pos.ch]; + } + mark[1] = rPos.line; + mark[2] = rPos.ch; + } + // Apply my toInsert the to remaining marks + mark[1] += addLine; + if (typeof(mark[4]) !== "undefined") { mark[3] += addLine; } + + if (mark[1] === posEndLine) { + mark[2] += addCh; + if (typeof(mark[4]) === "undefined" && typeof(mark[3]) !== "undefined") { + mark[3] += addCh; + } else if (typeof(mark[4]) !== "undefined" && mark[3] === posEndLine) { + mark[4] += addCh; + } + } + }); + if (op.toInsert.length) { + marks.push([Env.myAuthorId, pos.line, pos.ch, posEndLine, posEndCh]); + } + if (splitted) { + marks.push(splitted); + } + marks.sort(sortMarks); + }; + + // Remove marks added by OT and fix the incorrect ones + // first: data about the change with the lowest offset + // last: data about the change with the latest offset + // in the comments, "I" am "first" + var fixMarks = function (Env, first, last, content, toKeepEnd) { + var toKeep = []; + var toJoin = {}; + + debug('error', "Fix marks"); + debug('warn', first); + debug('warn', last); + + if (first.me !== last.me) { + // Get their start position compared to the authDoc + var lastAuthOffset = last.offset + last.total; + var lastAuthPos = SFCodeMirror.posToCursor(lastAuthOffset, last.doc); + // Get their start position compared to the localDoc + var lastLocalOffset = last.offset + first.total; + var lastLocalPos = SFCodeMirror.posToCursor(lastLocalOffset, first.doc); + + // Keep their changes in the marks (after their offset) + last.marks.some(function (array, i) { + var p = parseMark(array); + // End of the mark before offset? ignore + if (p.endLine < lastAuthPos.line) { return; } + // Take everything from the first mark ending after the pos + if (p.endLine > lastAuthPos.line || p.endCh >= lastAuthPos.ch) { + toKeep = last.marks.slice(i); + last.marks.splice(i); + return true; + } + }); + // Keep my marks (based on currentDoc) before their changes + first.marks.some(function (array, i) { + var p = parseMark(array); + // End of the mark before offset? ignore + if (p.endLine < lastLocalPos.line) { return; } + // Take everything from the first mark ending after the pos + if (p.endLine > lastLocalPos.line || p.endCh >= lastLocalPos.ch) { + first.marks.splice(i); + return true; + } + }); + } + + // If we still have markers in "first", store the last one so that we can "join" + // everything at the end + if (first.marks.length) { + var toJoinMark = first.marks[first.marks.length - 1].slice(); + toJoin = parseMark(toJoinMark); + } + + + // Add the new markers to the result + Array.prototype.unshift.apply(toKeepEnd, toKeep); + + debug('warn', toJoin); + debug('warn', toKeep); + debug('warn', toKeepEnd); + + // Fix their offset: compute added lines and added characters on the last line + // using the chainpad operation data (toInsert and toRemove) + var pos = SFCodeMirror.posToCursor(first.offset, content); + var removed = content.slice(first.offset, first.offset + first.toRemove).split('\n'); + var added = first.toInsert.split('\n'); + var posEndLine = pos.line + added.length - 1; // end line after op + var addLine = added.length - removed.length; + var addCh = added[added.length - 1].length - removed[removed.length - 1].length; + if (addLine > 0) { addCh -= pos.ch; } + if (addLine < 0) { addCh += pos.ch; } + toKeepEnd.forEach(function (array) { + // Push to correct lines + array[1] += addLine; + if (typeof(array[4]) !== "undefined") { array[3] += addLine; } + // If they have markers on my end line, push their "ch" + if (array[1] === posEndLine) { + array[2] += addCh; + // If they have no end line, it means end line === start line, + // so we also push their end offset + if (typeof(array[4]) === "undefined" && typeof(array[3]) !== "undefined") { + array[3] += addCh; + } else if (typeof(array[4]) !== "undefined" && array[3] === posEndLine) { + array[4] += addCh; + } + } + }); + + if (toKeep.length && toJoin && typeof(toJoin.endLine) !== "undefined" + && typeof(toJoin.endCh) !== "undefined") { + // Make sure the marks are joined correctly: + // fix the start position of the marks to keep + // Note: we must preserve the same end for this mark if it was single line! + if (typeof(toKeepEnd[0][4]) === "undefined") { // Single line + toKeepEnd[0][4] = toKeepEnd[0][3] || (toKeepEnd[0][2]+1); // preserve end ch + toKeepEnd[0][3] = toKeepEnd[0][1]; // preserve end line + } + toKeepEnd[0][1] = toJoin.endLine; + toKeepEnd[0][2] = toJoin.endCh; + } + + debug('log', 'Fixed'); + debug('warn', toKeepEnd); + }; + + var checkMarks = function (Env, userDoc) { + + var chainpad = Env.framework._.cpNfInner.chainpad; + var editor = Env.editor; + var CodeMirror = Env.CodeMirror; + + Env.enabled = Boolean(userDoc.authormarks && userDoc.authormarks.marks); + setAuthorMarks(Env, userDoc.authormarks); + + if (!Env.enabled) { return; } + + debug('error', 'Check marks'); + + var authDoc = JSON.parse(chainpad.getAuthDoc() || '{}'); + if (!authDoc.content || !userDoc.content) { return; } + + var authPatch = chainpad.getAuthBlock(); + if (authPatch.isFromMe) { + debug('log', 'Switch branch, from me'); + debug('log', authDoc.content); + debug('log', authDoc.authormarks.marks); + debug('log', userDoc.content); + // We're switching to a different branch that was created by us. + // We can't trust localDoc anymore because it contains data from the other branch + // It means the only changes that we need to consider are ours. + // Diff between userDoc and authDoc to see what we changed + var _myOps = ChainPad.Diff.diff(authDoc.content, userDoc.content).reverse(); + var authormarks = Util.clone(authDoc.authormarks); + _myOps.forEach(function (op) { + fixMarksFromOp(Env, op, authormarks.marks, authDoc.content); + }); + authormarks.marks = authormarks.marks.filter(Boolean); + debug('log', 'Fixed marks'); + debug('warn', authormarks.marks); + setAuthorMarks(Env, authormarks); + return; + } + + + var oldMarks = Env.oldMarks; + + + if (authDoc.content === userDoc.content) { return; } // No uncommitted work + + if (!userDoc.authormarks || !Array.isArray(userDoc.authormarks.marks)) { return; } + + debug('warn', 'Begin...'); + + var localDoc = CodeMirror.canonicalize(editor.getValue()); + + var commonParent = chainpad.getAuthBlock().getParent().getContent().doc; + var content = JSON.parse(commonParent || '{}').content || ''; + + var theirOps = ChainPad.Diff.diff(content, authDoc.content); + var myOps = ChainPad.Diff.diff(content, localDoc); + + debug('log', theirOps); + debug('log', myOps); + + if (!myOps.length || !theirOps.length) { return; } + + // If I have uncommited content when receiving a remote patch, all the operations + // placed after someone else's changes will create marker issues. We have to fix it + var sorted = []; + + var myTotal = 0; + var theirTotal = 0; + var parseOp = function (me) { + return function (op) { + var size = (op.toInsert.length - op.toRemove); + + sorted.push({ + me: me, + offset: op.offset, + toInsert: op.toInsert, + toRemove: op.toRemove, + size: size, + marks: (me ? (oldMarks && oldMarks.marks) + : (authDoc.authormarks && authDoc.authormarks.marks)) || [], + doc: me ? localDoc : authDoc.content + }); + + if (me) { myTotal += size; } + else { theirTotal += size; } + }; + }; + myOps.forEach(parseOp(true)); + theirOps.forEach(parseOp(false)); + + // Sort the operation in reverse order of offset + // If an operation from them has the same offset than an operation from me, put mine first + sorted.sort(function (a, b) { + if (a.offset === b.offset) { + return a.me ? -1 : 1; + } + return b.offset - a.offset; + }); + + debug('log', sorted); + + // We start from the end so that we don't have to fix the offsets everytime + var prev; + var toKeepEnd = []; + sorted.forEach(function (op) { + + // Not the same author? fix! + if (prev) { + // Provide the new "totals" + prev.total = prev.me ? myTotal : theirTotal; + op.total = op.me ? myTotal : theirTotal; + // Fix the markers + fixMarks(Env, op, prev, content, toKeepEnd); + } + + if (op.me) { myTotal -= op.size; } + else { theirTotal -= op.size; } + prev = op; + }); + + debug('log', toKeepEnd); + + // We now have all the markers located after the first operation (ordered by offset). + // Prepend the markers placed before this operation + var first = sorted[sorted.length - 1]; + if (first) { Array.prototype.unshift.apply(toKeepEnd, first.marks); } + + // Commit our new markers + Env.authormarks.marks = toKeepEnd; + + debug('warn', toKeepEnd); + debug('warn', '...End'); + }; + + // Reset marks displayed in CodeMirror to the marks stored in Env + var setMarks = function (Env) { + // on remote update: remove all marks, add new marks if colors are enabled + Env.editor.getAllMarks().forEach(function (marker) { + if (marker.attributes && marker.attributes['data-type'] === 'authormark') { + marker.clear(); + } + }); + + if (!Env.enabled) { return; } + + debug('error', 'setMarks'); + debug('log', Env.authormarks.marks); + + var authormarks = Env.authormarks; + authormarks.marks.forEach(function (mark) { + var uid = mark[0]; + if (uid !== -1 && (!authormarks.authors || !authormarks.authors[uid])) { return; } + var from = {}; + var to = {}; + from.line = mark[1]; + from.ch = mark[2]; + if (mark.length === 3) { + to.line = mark[1]; + to.ch = mark[2]+1; + } else if (mark.length === 4) { + to.line = mark[1]; + to.ch = mark[3]; + } else if (mark.length === 5) { + to.line = mark[3]; + to.ch = mark[4]; + } + + // Remove marks that are placed under this one + try { + Env.editor.findMarks(from, to).forEach(function (mark) { + if (!mark || !mark.attributes || mark.attributes['data-type'] !== 'authormark') { return; } + mark.clear(); + }); + } catch (e) { + console.warn(mark, JSON.stringify(authormarks.marks)); + console.error(from, to); + console.error(e); + } + + addMark(Env, from, to, uid); + }); + }; + + var setMyData = function (Env) { + if (!Env.enabled) { return; } + + var userData = Env.common.getMetadataMgr().getUserData(); + var old = Env.authormarks.authors[Env.myAuthorId]; + Env.authormarks.authors[Env.myAuthorId] = { + name: userData.name, + curvePublic: userData.curvePublic, + color: userData.color + }; + if (!old || (old.name === userData.name && old.color === userData.color)) { return; } + return true; + }; + + var localChange = function (Env, change, cb) { + cb = cb || function () {}; + + if (!Env.enabled) { return void cb(); } + + debug('error', 'Local change'); + debug('log', change, true); + + if (change.origin === "setValue") { + // If the content is changed from a remote patch, we call localChange + // in "onContentUpdate" directly + return; + } + if (change.text === undefined || ['+input', 'paste'].indexOf(change.origin) === -1) { + return void cb(); + } + + // add new author mark if text is added. marks from removed text are removed automatically + + // change.to is not always correct, fix it! + var to_add = { + line: change.from.line + change.text.length-1, + }; + if (change.text.length > 1) { + // Multiple lines => take the length of the text added to the last line + to_add.ch = change.text[change.text.length-1].length; + } else { + // Single line => use the "from" position and add the length of the text + to_add.ch = change.from.ch + change.text[change.text.length-1].length; + } + + // If my text is inside an existing mark: + // * if it's my mark, do nothing + // * if it's someone else's mark, break it + // We can only have one author mark at a given position, but there may be + // another mark (cursor selection...) at this position so we use ".some" + var toSplit, abort; + + + Env.editor.findMarks(change.from, to_add).some(function (mark) { + if (!mark.attributes) { return; } + if (mark.attributes['data-type'] !== 'authormark') { return; } + if (mark.attributes['data-uid'] !== Env.myAuthorId) { + toSplit = { + mark: mark, + uid: mark.attributes['data-uid'] + }; + } else { + // This is our mark: abort to avoid making a new one + abort = true; + } + + return true; + }); + if (abort) { return void cb(); } + + // Add my data to the doc if it's missing + if (!Env.authormarks.authors[Env.myAuthorId]) { + setMyData(Env); + } + + if (toSplit && toSplit.mark && typeof(toSplit.uid) !== "undefined") { + // Break the other user's mark if needed + var _pos = toSplit.mark.find(); + toSplit.mark.clear(); + addMark(Env, _pos.from, change.from, toSplit.uid); // their mark, 1st part + addMark(Env, change.from, to_add, Env.myAuthorId); // my mark + addMark(Env, to_add, _pos.to, toSplit.uid); // their mark, 2nd part + } else { + // Add my mark + addMark(Env, change.from, to_add, Env.myAuthorId); + } + + cb(); + }; + + var setButton = function (Env, $button) { + var toggle = function () { + var tippy = $button[0] && $button[0]._tippy; + if (Env.opacity) { + Env.opacity = 0; + if (tippy) { tippy.title = Messages.cba_show; } + else { $button.attr('title', Messages.cba_show); } + $button.removeClass("cp-toolbar-button-active"); + } else { + Env.opacity = MARK_OPACITY; + if (tippy) { tippy.title = Messages.cba_hide; } + else { $button.attr('title', Messages.cba_hide); } + $button.addClass("cp-toolbar-button-active"); + } + }; + toggle(); + Env.$button = $button; + $button.click(function() { + toggle(); + setMarks(Env); + }); + }; + + var authorUid = function (existing) { + if (!Array.isArray(existing)) { existing = []; } + var n; + var i = 0; + while (!n || existing.indexOf(n) !== -1 && i++ < 1000) { + n = Math.floor(Math.random() * 1000000); + } + // If we can't find a valid number in 1000 iterations, use 0... + if (existing.indexOf(n) !== -1) { n = 0; } + return n; + }; + var getAuthorId = function (Env) { + var existing = Object.keys(Env.authormarks.authors || {}).map(Number); + if (!Env.common.isLoggedIn()) { return authorUid(existing); } + + var userData = Env.common.getMetadataMgr().getUserData(); + var uid; + existing.some(function (id) { + var author = Env.authormarks.authors[id] || {}; + if (author.curvePublic !== userData.curvePublic) { return; } + uid = Number(id); + return true; + }); + return uid || authorUid(existing); + }; + var ready = function (Env) { + Env.ready = true; + Env.myAuthorId = getAuthorId(Env); + + if (!Env.enabled) { return; } + if (Env.$button) { Env.$button.show(); } + if (!Env.authormarks.marks || !Env.authormarks.marks.length) { + Env.authormarks = Util.clone(DEFAULT); + } + setMarks(Env); + }; + + var getState = function (Env) { + return Boolean(Env.authormarks && Env.authormarks.marks); + }; + var setState = function (Env, enabled) { + // If the state has changed in the pad, change the Env too + if (!Env.ready) { return; } + if (Env.enabled === enabled) { return; } + Env.enabled = enabled; + if (!Env.enabled) { + // Reset marks + Env.authormarks = {}; + setMarks(Env); + if (Env.$button) { Env.$button.hide(); } + } else { + Env.myAuthorId = getAuthorId(Env); + // If it's a reset, add initial marker + if (!Env.authormarks.marks || !Env.authormarks.marks.length) { + Env.authormarks = Util.clone(DEFAULT); + setMarks(Env); + } + if (Env.$button) { Env.$button.show(); } + } + if (Env.ready) { Env.framework.localChange(); } + }; + + Markers.create = function (config) { + var Env = config; + Env.authormarks = Util.clone(DEFAULT); + Env.enabled = false; + Env.myAuthorId = 0; + + if (Env.devMode) { + debug = function (level, obj, logObject) { + var f = console.log; + if (typeof(console[level]) === "function") { + f = console[level]; + } + if (logObject) { return void f(obj); } + }; + } + + var metadataMgr = Env.common.getMetadataMgr(); + metadataMgr.onChange(function () { + // If the markers are disabled or if I haven't pushed content since the last reset, + // don't update my data + if (!Env.enabled || !Env.myAuthorId || !Env.authormarks.authors || + !Env.authormarks.authors[Env.myAuthorId]) { + return; + } + + // Update my data + var changed = setMyData(Env); + if (changed) { + setMarks(Env); + Env.framework.localChange(); + } + }); + + var call = function (f) { + return function () { + try { + [].unshift.call(arguments, Env); + return f.apply(null, arguments); + } catch (e) { + console.error(e); + } + }; + }; + + + return { + addMark: call(addMark), + getAuthorMarks: call(getAuthorMarks), + updateAuthorMarks: call(updateAuthorMarks), + checkMarks: call(checkMarks), + setMarks: call(setMarks), + localChange: call(localChange), + ready: call(ready), + setButton: call(setButton), + getState: call(getState), + setState: call(setState), + }; + }; + + return Markers; +}); diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 677243399..25b1a320f 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -1708,14 +1708,17 @@ define([ var ver = arr[1]; if (!ver) { return; } var verArr = ver.split('.'); - verArr[2] = 0; + //verArr[2] = 0; if (verArr.length !== 3) { return; } var stored = currentVersion || '0.0.0'; var storedArr = stored.split('.'); - storedArr[2] = 0; + //storedArr[2] = 0; + var shouldUpdate = JSON.stringify(verArr) !== JSON.stringify(storedArr); +/* var shouldUpdate = parseInt(verArr[0]) !== parseInt(storedArr[0]) || (parseInt(verArr[0]) === parseInt(storedArr[0]) && parseInt(verArr[1]) !== parseInt(storedArr[1])); +*/ if (!shouldUpdate) { return; } currentVersion = ver; localStorage[CRYPTPAD_VERSION] = ver; diff --git a/www/common/inner/common-mediatag.js b/www/common/inner/common-mediatag.js index 40ae19059..3e1f2e75b 100644 --- a/www/common/inner/common-mediatag.js +++ b/www/common/inner/common-mediatag.js @@ -284,10 +284,12 @@ define([ 'data-crypto-key': key }); $inner.append(tag); - MediaTag(tag).on('error', function () { - locked = false; - $spinner.hide(); - UI.log(Messages.error); + setTimeout(function () { + MediaTag(tag).on('error', function () { + locked = false; + $spinner.hide(); + UI.log(Messages.error); + }); }); } diff --git a/www/common/inner/common-modal.js b/www/common/inner/common-modal.js index 33ac14d9f..2131719cb 100644 --- a/www/common/inner/common-modal.js +++ b/www/common/inner/common-modal.js @@ -69,23 +69,7 @@ define([ Modal.isOwned = function (Env, data) { var common = Env.common; data = data || {}; - var priv = common.getMetadataMgr().getPrivateData(); - var edPublic = priv.edPublic; - var owned = false; - if (Array.isArray(data.owners) && data.owners.length) { - if (data.owners.indexOf(edPublic) !== -1) { - owned = true; - } else { - Object.keys(priv.teams || {}).some(function (id) { - var team = priv.teams[id] || {}; - if (team.viewer) { return; } - if (data.owners.indexOf(team.edPublic) === -1) { return; } - owned = Number(id); - return true; - }); - } - } - return owned; + return common.isOwned(data.owners); }; var blocked = false; diff --git a/www/common/metadata-manager.js b/www/common/metadata-manager.js index 717705d84..70e9263e5 100644 --- a/www/common/metadata-manager.js +++ b/www/common/metadata-manager.js @@ -48,7 +48,6 @@ define(['json.sortify'], function (Sortify) { //title: meta.doc.defaultTitle, type: meta.doc.type, users: {}, - authors: {} }; metadataLazyObj = JSON.parse(JSON.stringify(metadataObj)); } @@ -69,6 +68,10 @@ define(['json.sortify'], function (Sortify) { } metadataObj.users = mdo; + // Clean old data + delete metadataObj.authors; + delete metadataLazyObj.authors; + // Always update the userlist in the lazy object, otherwise it may be outdated // and metadataMgr.updateMetadata() won't do anything, and so we won't push events // to the userlist UI ==> phantom viewers @@ -96,27 +99,6 @@ define(['json.sortify'], function (Sortify) { checkUpdate(lazy); }); }; - var addAuthor = function () { - if (!meta.user || !meta.user.netfluxId || !priv || !priv.edPublic) { return; } - var authors = metadataObj.authors || {}; - var old = Sortify(authors); - if (!authors[priv.edPublic]) { - authors[priv.edPublic] = { - nId: [meta.user.netfluxId], - name: meta.user.name - }; - } else { - authors[priv.edPublic].name = meta.user.name; - if (authors[priv.edPublic].nId.indexOf(meta.user.netfluxId) === -1) { - authors[priv.edPublic].nId.push(meta.user.netfluxId); - } - } - if (Sortify(authors) !== old) { - metadataObj.authors = authors; - metadataLazyObj.authors = JSON.parse(JSON.stringify(authors)); - change(); - } - }; var netfluxId; var isReady = false; @@ -225,7 +207,6 @@ define(['json.sortify'], function (Sortify) { if (isReady) { return void f(); } readyHandlers.push(f); }, - addAuthor: addAuthor, }); }; return Object.freeze({ create: create }); diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index fb7b2cc8e..2841d8130 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -1890,7 +1890,15 @@ define([ if (msg) { msg = msg.replace(/cp\|(([A-Za-z0-9+\/=]+)\|)?/, ''); //var decryptedMsg = crypto.decrypt(msg, true); - msgs.push(msg); + if (data.debug) { + msgs.push({ + msg: msg, + author: parsed[1][1], + time: parsed[1][5] + }); + } else { + msgs.push(msg); + } } }; network.on('message', onMsg); diff --git a/www/common/sframe-app-framework.js b/www/common/sframe-app-framework.js index fc149b2cc..24e7d0abd 100644 --- a/www/common/sframe-app-framework.js +++ b/www/common/sframe-app-framework.js @@ -290,7 +290,7 @@ define([ } if (padChange && hasChanged(content)) { - cpNfInner.metadataMgr.addAuthor(); + //cpNfInner.metadataMgr.addAuthor(); } oldContent = content; diff --git a/www/common/sframe-common-codemirror.js b/www/common/sframe-common-codemirror.js index 2192f168c..dc046cd62 100644 --- a/www/common/sframe-common-codemirror.js +++ b/www/common/sframe-common-codemirror.js @@ -12,7 +12,7 @@ define([ ], function ($, Modes, Themes, Messages, UIElements, MT, Hash, Util, TextCursor, ChainPad) { var module = {}; - var cursorToPos = function(cursor, oldText) { + var cursorToPos = module.cursorToPos = function(cursor, oldText) { var cLine = cursor.line; var cCh = cursor.ch; var pos = 0; @@ -28,7 +28,7 @@ define([ return pos; }; - var posToCursor = function(position, newText) { + var posToCursor = module.posToCursor = function(position, newText) { var cursor = { line: 0, ch: 0 @@ -218,6 +218,24 @@ define([ }); editor.focus(); + // Fix cursor and scroll position after undo/redo + var undoData; + editor.on('beforeChange', function (editor, change) { + if (change.origin !== "undo" && change.origin !== "redo") { return; } + undoData = editor.getValue(); + }); + editor.on('change', function (editor, change) { + if (change.origin !== "undo" && change.origin !== "redo") { return; } + if (typeof(undoData) === "undefined") { return; } + var doc = editor.getValue(); + var ops = ChainPad.Diff.diff(undoData, doc); + undoData = undefined; + if (!ops.length) { return; } + var cursor = posToCursor(ops[0].offset, doc); + editor.setCursor(cursor); + editor.scrollIntoView(cursor); + }); + var setMode = exp.setMode = function (mode, cb) { exp.highlightMode = mode; if (mode === 'markdown') { mode = 'gfm'; } @@ -415,8 +433,7 @@ define([ ///// - var canonicalize = function (t) { return t.replace(/\r\n/g, '\n'); }; - + var canonicalize = exp.canonicalize = function (t) { return t.replace(/\r\n/g, '\n'); }; exp.contentUpdate = function (newContent) { @@ -424,6 +441,7 @@ define([ var remoteDoc = newContent.content; // setValueAndCursor triggers onLocal, even if we don't make any change to the content // and it may revert other changes (metadata) + if (oldDoc === remoteDoc) { return; } exp.setValueAndCursor(oldDoc, remoteDoc); }; diff --git a/www/common/sframe-common-history.js b/www/common/sframe-common-history.js index be2793128..0217fee7a 100644 --- a/www/common/sframe-common-history.js +++ b/www/common/sframe-common-history.js @@ -71,7 +71,10 @@ define([ lastKnownHash = data.lastKnownHash; isComplete = data.isFull; var messages = (data.messages || []).map(function (obj) { - return obj.msg; + if (!config.debug) { + return obj.msg; + } + return obj; }); if (config.debug) { console.log(data.messages); } Array.prototype.unshift.apply(allMessages, messages); // Destructive concat diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 2ae6f789a..681701129 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -395,6 +395,7 @@ define([ var forceCreationScreen = cfg.useCreationScreen && sessionStorage[Utils.Constants.displayPadCreationScreen]; delete sessionStorage[Utils.Constants.displayPadCreationScreen]; + var isSafe = ['debug', 'profile', 'drive'].indexOf(currentPad.app) !== -1; var updateMeta = function () { //console.log('EV_METADATA_UPDATE'); var metaObj; @@ -459,7 +460,7 @@ define([ additionalPriv.registeredOnly = true; } - if (['debug', 'profile'].indexOf(currentPad.app) !== -1) { + if (isSafe) { additionalPriv.hashes = hashes; } @@ -620,7 +621,7 @@ define([ }); }; - addCommonRpc(sframeChan); + addCommonRpc(sframeChan, isSafe); var currentTitle; var currentTabTitle; @@ -838,17 +839,26 @@ define([ sframeChan.on('Q_GET_FULL_HISTORY', function (data, cb) { var crypto = Crypto.createEncryptor(secret.keys); Cryptpad.getFullHistory({ + debug: data && data.debug, channel: secret.channel, validateKey: secret.keys.validateKey }, function (encryptedMsgs) { var nt = nThen; var decryptedMsgs = []; var total = encryptedMsgs.length; - encryptedMsgs.forEach(function (msg, i) { + encryptedMsgs.forEach(function (_msg, i) { nt = nt(function (waitFor) { // The 3rd parameter "true" means we're going to skip signature validation. // We don't need it since the message is already validated serverside by hk - decryptedMsgs.push(crypto.decrypt(msg, true, true)); + if (typeof(_msg) === "object") { + decryptedMsgs.push({ + author: _msg.author, + time: _msg.time, + msg: crypto.decrypt(_msg.msg, true, true) + }); + } else { + decryptedMsgs.push(crypto.decrypt(_msg, true, true)); + } setTimeout(waitFor(function () { sframeChan.event('EV_FULL_HISTORY_STATUS', (i+1)/total); })); @@ -992,12 +1002,12 @@ define([ config.onAction = function (data) { if (typeof(SecureModal.cb) !== "function") { return; } SecureModal.cb(data); - SecureModal.$iframe.hide(); }; config.onClose = function () { SecureModal.$iframe.hide(); }; config.data = { + app: parsed.type, hashes: hashes, password: password, isTemplate: isTemplate @@ -1010,12 +1020,12 @@ define([ }; SecureModal.$iframe = $('