Merge remote-tracking branch 'origin/develop' into develop

src/autocomplete/CommandProvider.js

@@ -20,7 +20,7 @@ limitations under the License.
 import React from 'react';
 import {_t} from '../languageHandler';
 import AutocompleteProvider from './AutocompleteProvider';
-import FuzzyMatcher from './FuzzyMatcher';
+import QueryMatcher from './QueryMatcher';
 import {TextualCompletion} from './Components';
 import type {Completion, SelectionRange} from "./Autocompleter";
 import {CommandMap} from '../SlashCommands';
@@ -32,7 +32,7 @@ const COMMAND_RE = /(^\/\w*)(?: .*)?/g;
 export default class CommandProvider extends AutocompleteProvider {
     constructor() {
         super(COMMAND_RE);
-        this.matcher = new FuzzyMatcher(COMMANDS, {
+        this.matcher = new QueryMatcher(COMMANDS, {
            keys: ['command', 'args', 'description'],
         });
     }

src/autocomplete/CommunityProvider.js

@@ -19,7 +19,7 @@ import React from 'react';
 import { _t } from '../languageHandler';
 import AutocompleteProvider from './AutocompleteProvider';
 import MatrixClientPeg from '../MatrixClientPeg';
-import FuzzyMatcher from './FuzzyMatcher';
+import QueryMatcher from './QueryMatcher';
 import {PillCompletion} from './Components';
 import sdk from '../index';
 import _sortBy from 'lodash/sortBy';
@@ -41,7 +41,7 @@ function score(query, space) {
 export default class CommunityProvider extends AutocompleteProvider {
     constructor() {
         super(COMMUNITY_REGEX);
-        this.matcher = new FuzzyMatcher([], {
+        this.matcher = new QueryMatcher([], {
             keys: ['groupId', 'name', 'shortDescription'],
         });
     }

src/autocomplete/EmojiProvider.js

@@ -20,7 +20,7 @@ import React from 'react';
 import { _t } from '../languageHandler';
 import AutocompleteProvider from './AutocompleteProvider';
 import {shortnameToUnicode, asciiRegexp, unicodeRegexp} from 'emojione';
-import FuzzyMatcher from './FuzzyMatcher';
+import QueryMatcher from './QueryMatcher';
 import sdk from '../index';
 import {PillCompletion} from './Components';
 import type {Completion, SelectionRange} from './Autocompleter';
@@ -84,12 +84,12 @@ function score(query, space) {
 export default class EmojiProvider extends AutocompleteProvider {
     constructor() {
         super(EMOJI_REGEX);
-        this.matcher = new FuzzyMatcher(EMOJI_SHORTNAMES, {
+        this.matcher = new QueryMatcher(EMOJI_SHORTNAMES, {
             keys: ['aliases_ascii', 'shortname', 'aliases'],
             // For matching against ascii equivalents
             shouldMatchWordsOnly: false,
         });
-        this.nameMatcher = new FuzzyMatcher(EMOJI_SHORTNAMES, {
+        this.nameMatcher = new QueryMatcher(EMOJI_SHORTNAMES, {
             keys: ['name'],
             // For removing punctuation
             shouldMatchWordsOnly: true,

src/autocomplete/FuzzyMatcher.js

@@ -1,107 +0,0 @@
-/*
-Copyright 2017 Aviral Dasgupta
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-//import Levenshtein from 'liblevenshtein';
-//import _at from 'lodash/at';
-//import _flatMap from 'lodash/flatMap';
-//import _sortBy from 'lodash/sortBy';
-//import _sortedUniq from 'lodash/sortedUniq';
-//import _keys from 'lodash/keys';
-//
-//class KeyMap {
-//    keys: Array<String>;
-//    objectMap: {[String]: Array<Object>};
-//    priorityMap: {[String]: number}
-//}
-//
-//const DEFAULT_RESULT_COUNT = 10;
-//const DEFAULT_DISTANCE = 5;
-
-// FIXME Until Fuzzy matching works better, we use prefix matching.
-
-import PrefixMatcher from './QueryMatcher';
-export default PrefixMatcher;
-
-//class FuzzyMatcher { // eslint-disable-line no-unused-vars
-//    /**
-//     * @param {object[]} objects the objects to perform a match on
-//     * @param {string[]} keys an array of keys within each object to match on
-//     * Keys can refer to object properties by name and as in JavaScript (for nested properties)
-//     *
-//     * To use, simply presort objects by required criteria, run through this function and create a FuzzyMatcher with the
-//     * resulting KeyMap.
-//     *
-//     * TODO: Handle arrays and objects (Fuse did this, RoomProvider uses it)
-//     * @return {KeyMap}
-//     */
-//    static valuesToKeyMap(objects: Array<Object>, keys: Array<String>): KeyMap {
-//        const keyMap = new KeyMap();
-//        const map = {};
-//        const priorities = {};
-//
-//        objects.forEach((object, i) => {
-//            const keyValues = _at(object, keys);
-//            console.log(object, keyValues, keys);
-//            for (const keyValue of keyValues) {
-//                if (!map.hasOwnProperty(keyValue)) {
-//                   map[keyValue] = [];
-//                }
-//                map[keyValue].push(object);
-//            }
-//            priorities[object] = i;
-//        });
-//
-//        keyMap.objectMap = map;
-//        keyMap.priorityMap = priorities;
-//        keyMap.keys = _sortBy(_keys(map), [(value) => priorities[value]]);
-//        return keyMap;
-//    }
-//
-//    constructor(objects: Array<Object>, options: {[Object]: Object} = {}) {
-//        this.options = options;
-//        this.keys = options.keys;
-//        this.setObjects(objects);
-//    }
-//
-//    setObjects(objects: Array<Object>) {
-//        this.keyMap = FuzzyMatcher.valuesToKeyMap(objects, this.keys);
-//        console.log(this.keyMap.keys);
-//        this.matcher = new Levenshtein.Builder()
-//            .dictionary(this.keyMap.keys, true)
-//            .algorithm('transposition')
-//            .sort_candidates(false)
-//            .case_insensitive_sort(true)
-//            .include_distance(true)
-//            .maximum_candidates(this.options.resultCount || DEFAULT_RESULT_COUNT) // result count 0 doesn't make much sense
-//            .build();
-//    }
-//
-//    match(query: String): Array<Object> {
-//        const candidates = this.matcher.transduce(query, this.options.distance || DEFAULT_DISTANCE);
-//        // TODO FIXME This is hideous. Clean up when possible.
-//        const val = _sortedUniq(_sortBy(_flatMap(candidates, (candidate) => {
-//                return this.keyMap.objectMap[candidate[0]].map((value) => {
-//                    return {
-//                        distance: candidate[1],
-//                        ...value,
-//                    };
-//                });
-//            }),
-//            [(candidate) => candidate.distance, (candidate) => this.keyMap.priorityMap[candidate]]));
-//        console.log(val);
-//        return val;
-//    }
-//}

src/autocomplete/QueryMatcher.js

@@ -2,6 +2,7 @@
 /*
 Copyright 2017 Aviral Dasgupta
 Copyright 2018 Michael Telatynski <7t3chguy@gmail.com>
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,99 +21,99 @@ import _at from 'lodash/at';
 import _flatMap from 'lodash/flatMap';
 import _sortBy from 'lodash/sortBy';
 import _uniq from 'lodash/uniq';
-import _keys from 'lodash/keys';
-
-class KeyMap {
-    keys: Array<String>;
-    objectMap: {[String]: Array<Object>};
-    priorityMap = new Map();
-}
 
 function stripDiacritics(str: string): string {
     return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
 }
 
+/**
+ * Simple search matcher that matches any results with the query string anywhere
+ * in the search string. Returns matches in the order the query string appears
+ * in the search key, earliest first, then in the order the items appeared in
+ * the source array.
+ *
+ * @param {Object[]} objects Initial list of objects. Equivalent to calling
+ *     setObjects() after construction
+ * @param {Object} options Options object
+ * @param {string[]} options.keys List of keys to use as indexes on the objects
+ * @param {function[]} options.funcs List of functions that when called with the
+ *     object as an arg will return a string to use as an index
+ */
 export default class QueryMatcher {
-    /**
-     * @param {object[]} objects the objects to perform a match on
-     * @param {string[]} keys an array of keys within each object to match on
-     * Keys can refer to object properties by name and as in JavaScript (for nested properties)
-     *
-     * To use, simply presort objects by required criteria, run through this function and create a QueryMatcher with the
-     * resulting KeyMap.
-     *
-     * TODO: Handle arrays and objects (Fuse did this, RoomProvider uses it)
-     * @return {KeyMap}
-     */
-    static valuesToKeyMap(objects: Array<Object>, keys: Array<String>): KeyMap {
-        const keyMap = new KeyMap();
-        const map = {};
-
-        objects.forEach((object, i) => {
-            const keyValues = _at(object, keys);
-            for (const keyValue of keyValues) {
-                const key = stripDiacritics(keyValue).toLowerCase();
-                if (!map.hasOwnProperty(key)) {
-                    map[key] = [];
-                }
-                map[key].push(object);
-            }
-            keyMap.priorityMap.set(object, i);
-        });
-
-        keyMap.objectMap = map;
-        keyMap.keys = _keys(map);
-        return keyMap;
-    }
-
     constructor(objects: Array<Object>, options: {[Object]: Object} = {}) {
-        this.options = options;
+        this._options = options;
-        this.keys = options.keys;
+        this._keys = options.keys;
+        this._funcs = options.funcs || [];
+
         this.setObjects(objects);
 
         // By default, we remove any non-alphanumeric characters ([^A-Za-z0-9_]) from the
         // query and the value being queried before matching
-        if (this.options.shouldMatchWordsOnly === undefined) {
+        if (this._options.shouldMatchWordsOnly === undefined) {
-            this.options.shouldMatchWordsOnly = true;
+            this._options.shouldMatchWordsOnly = true;
         }
 
         // By default, match anywhere in the string being searched. If enabled, only return
         // matches that are prefixed with the query.
-        if (this.options.shouldMatchPrefix === undefined) {
+        if (this._options.shouldMatchPrefix === undefined) {
-            this.options.shouldMatchPrefix = false;
+            this._options.shouldMatchPrefix = false;
         }
     }
 
     setObjects(objects: Array<Object>) {
-        this.keyMap = QueryMatcher.valuesToKeyMap(objects, this.keys);
+        this._items = new Map();
+
+        for (const object of objects) {
+            const keyValues = _at(object, this._keys);
+
+            for (const f of this._funcs) {
+                keyValues.push(f(object));
+            }
+
+            for (const keyValue of keyValues) {
+                const key = stripDiacritics(keyValue).toLowerCase();
+                if (!this._items.has(key)) {
+                    this._items.set(key, []);
+                }
+                this._items.get(key).push(object);
+            }
+        }
     }
 
     match(query: String): Array<Object> {
         query = stripDiacritics(query).toLowerCase();
-        if (this.options.shouldMatchWordsOnly) {
+        if (this._options.shouldMatchWordsOnly) {
             query = query.replace(/[^\w]/g, '');
         }
         if (query.length === 0) {
             return [];
         }
         const results = [];
-        this.keyMap.keys.forEach((key) => {
+        // Iterate through the map & check each key.
+        // ES6 Map iteration order is defined to be insertion order, so results
+        // here will come out in the order they were put in.
+        for (const key of this._items.keys()) {
             let resultKey = key;
-            if (this.options.shouldMatchWordsOnly) {
+            if (this._options.shouldMatchWordsOnly) {
                 resultKey = resultKey.replace(/[^\w]/g, '');
             }
             const index = resultKey.indexOf(query);
-            if (index !== -1 && (!this.options.shouldMatchPrefix || index === 0)) {
+            if (index !== -1 && (!this._options.shouldMatchPrefix || index === 0)) {
                 results.push({key, index});
             }
+        }
+
+        // Sort them by where the query appeared in the search key
+        // lodash sortBy is a stable sort, so results where the query
+        // appeared in the same place will retain their order with
+        // respect to each other.
+        const sortedResults = _sortBy(results, (candidate) => {
+            return candidate.index;
         });
 
-        return _uniq(_flatMap(_sortBy(results, (candidate) => {
+        // Now map the keys to the result objects. Each result object is a list, so
-            return candidate.index;
+        // flatMap will flatten those lists out into a single list. Also remove any
-        }).map((candidate) => {
+        // duplicates.
-            // return an array of objects (those given to setObjects) that have the given
+        return _uniq(_flatMap(sortedResults, (candidate) => this._items.get(candidate.key)));
-            // key as a property.
-            return this.keyMap.objectMap[candidate.key];
-        })));
     }
 }

src/autocomplete/RoomProvider.js

@@ -21,7 +21,7 @@ import React from 'react';
 import { _t } from '../languageHandler';
 import AutocompleteProvider from './AutocompleteProvider';
 import MatrixClientPeg from '../MatrixClientPeg';
-import FuzzyMatcher from './FuzzyMatcher';
+import QueryMatcher from './QueryMatcher';
 import {PillCompletion} from './Components';
 import {getDisplayAliasForRoom} from '../Rooms';
 import sdk from '../index';
@@ -43,7 +43,7 @@ function score(query, space) {
 export default class RoomProvider extends AutocompleteProvider {
     constructor() {
         super(ROOM_REGEX);
-        this.matcher = new FuzzyMatcher([], {
+        this.matcher = new QueryMatcher([], {
             keys: ['displayedAlias', 'name'],
         });
     }

src/autocomplete/UserProvider.js

@@ -23,7 +23,7 @@ import { _t } from '../languageHandler';
 import AutocompleteProvider from './AutocompleteProvider';
 import {PillCompletion} from './Components';
 import sdk from '../index';
-import FuzzyMatcher from './FuzzyMatcher';
+import QueryMatcher from './QueryMatcher';
 import _sortBy from 'lodash/sortBy';
 import MatrixClientPeg from '../MatrixClientPeg';
 
@@ -44,8 +44,9 @@ export default class UserProvider extends AutocompleteProvider {
     constructor(room) {
         super(USER_REGEX, FORCED_USER_REGEX);
         this.room = room;
-        this.matcher = new FuzzyMatcher([], {
+        this.matcher = new QueryMatcher([], {
-            keys: ['name', 'userId'],
+            keys: ['name'],
+            funcs: [obj => obj.userId.slice(1)], // index by user id minus the leading '@'
             shouldMatchPrefix: true,
             shouldMatchWordsOnly: false,
         });
@@ -104,7 +105,9 @@ export default class UserProvider extends AutocompleteProvider {
         const fullMatch = command[0];
         // Don't search if the query is a single "@"
         if (fullMatch && fullMatch !== '@') {
-            completions = this.matcher.match(fullMatch).map((user) => {
+            // Don't include the '@' in our search query - it's only used as a way to trigger completion
+            const query = fullMatch.startsWith('@') ? fullMatch.substring(1) : fullMatch;
+            completions = this.matcher.match(query).map((user) => {
                 const displayName = (user.name || user.userId || '');
                 return {
                     // Length of completion should equal length of text in decorator. draft-js

test/autocomplete/QueryMatcher-test.js

@@ -0,0 +1,175 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import expect from 'expect';
+
+import QueryMatcher from '../../src/autocomplete/QueryMatcher';
+
+const OBJECTS = [
+    { name: "Mel B", nick: "Scary" },
+    { name: "Mel C", nick: "Sporty" },
+    { name: "Emma", nick: "Baby" },
+    { name: "Geri", nick: "Ginger" },
+    { name: "Victoria", nick: "Posh" },
+];
+
+const NONWORDOBJECTS = [
+    { name: "B.O.B" },
+    { name: "bob" },
+];
+
+describe('QueryMatcher', function() {
+    it('Returns results by key', function() {
+        const qm = new QueryMatcher(OBJECTS, {keys: ["name"]});
+        const results = qm.match('Geri');
+
+        expect(results.length).toBe(1);
+        expect(results[0].name).toBe('Geri');
+    });
+
+    it('Returns results by prefix', function() {
+        const qm = new QueryMatcher(OBJECTS, {keys: ["name"]});
+        const results = qm.match('Ge');
+
+        expect(results.length).toBe(1);
+        expect(results[0].name).toBe('Geri');
+    });
+
+    it('Matches case-insensitive', function() {
+        const qm = new QueryMatcher(OBJECTS, {keys: ["name"]});
+        const results = qm.match('geri');
+
+        expect(results.length).toBe(1);
+        expect(results[0].name).toBe('Geri');
+    });
+
+    it('Matches ignoring accents', function() {
+        const qm = new QueryMatcher([{name: "Gëri", foo: 46}], {keys: ["name"]});
+        const results = qm.match('geri');
+
+        expect(results.length).toBe(1);
+        expect(results[0].foo).toBe(46);
+    });
+
+    it('Returns multiple results in order of search string appearance', function() {
+        const qm = new QueryMatcher(OBJECTS, {keys: ["name", "nick"]});
+        const results = qm.match('or');
+
+        expect(results.length).toBe(2);
+        expect(results[0].name).toBe('Mel C');
+        expect(results[1].name).toBe('Victoria');
+
+
+        qm.setObjects(OBJECTS.slice().reverse());
+        const reverseResults = qm.match('or');
+
+        // should still be in the same order: search string position
+        // takes precedence over input order
+        expect(reverseResults.length).toBe(2);
+        expect(reverseResults[0].name).toBe('Mel C');
+        expect(reverseResults[1].name).toBe('Victoria');
+    });
+
+    it('Returns results with search string in same place in insertion order', function() {
+        const qm = new QueryMatcher(OBJECTS, {keys: ["name"]});
+        const results = qm.match('Mel');
+
+        expect(results.length).toBe(2);
+        expect(results[0].name).toBe('Mel B');
+        expect(results[1].name).toBe('Mel C');
+
+
+        qm.setObjects(OBJECTS.slice().reverse());
+
+        const reverseResults = qm.match('Mel');
+
+        expect(reverseResults.length).toBe(2);
+        expect(reverseResults[0].name).toBe('Mel C');
+        expect(reverseResults[1].name).toBe('Mel B');
+    });
+
+    it('Returns numeric results in correct order (input pos)', function() {
+        // regression test for depending on object iteration order
+        const qm = new QueryMatcher([
+            {name: "123456badger"},
+            {name: "123456"},
+        ], {keys: ["name"]});
+        const results = qm.match('123456');
+
+        expect(results.length).toBe(2);
+        expect(results[0].name).toBe('123456badger');
+        expect(results[1].name).toBe('123456');
+    });
+
+    it('Returns numeric results in correct order (query pos)', function() {
+        const qm = new QueryMatcher([
+            {name: "999999123456"},
+            {name: "123456badger"},
+        ], {keys: ["name"]});
+        const results = qm.match('123456');
+
+        expect(results.length).toBe(2);
+        expect(results[0].name).toBe('123456badger');
+        expect(results[1].name).toBe('999999123456');
+    });
+
+    it('Returns results by function', function() {
+        const qm = new QueryMatcher(OBJECTS, {
+            keys: ["name"],
+            funcs: [x => x.name.replace('Mel', 'Emma')],
+        });
+
+        const results = qm.match('Emma');
+        expect(results.length).toBe(3);
+        expect(results[0].name).toBe('Mel B');
+        expect(results[1].name).toBe('Mel C');
+        expect(results[2].name).toBe('Emma');
+    });
+
+    it('Matches words only by default', function() {
+        const qm = new QueryMatcher(NONWORDOBJECTS, { keys: ["name"] });
+
+        const results = qm.match('bob');
+        expect(results.length).toBe(2);
+        expect(results[0].name).toBe('B.O.B');
+        expect(results[1].name).toBe('bob');
+    });
+
+    it('Matches all chars with words-only off', function() {
+        const qm = new QueryMatcher(NONWORDOBJECTS, {
+            keys: ["name"],
+            shouldMatchWordsOnly: false,
+         });
+
+        const results = qm.match('bob');
+        expect(results.length).toBe(1);
+        expect(results[0].name).toBe('bob');
+    });
+
+    it('Matches only by prefix with shouldMatchPrefix on', function() {
+        const qm = new QueryMatcher([
+            {name: "Victoria"},
+            {name: "Tori"},
+        ], {
+            keys: ["name"],
+            shouldMatchPrefix: true,
+         });
+
+        const results = qm.match('tori');
+        expect(results.length).toBe(1);
+        expect(results[0].name).toBe('Tori');
+    });
+});