make EmojiProvider and stripped-emoji.json work

res/themes/light/css/_light.scss

@@ -5,11 +5,11 @@
    Arial empirically gets it right, hence prioritising Arial here. */
 /* We fall through to Twemoji for emoji rather than falling through
    to native Emoji fonts (if any) to ensure cross-browser consistency */
-$font-family: Nunito, Arial, Helvetica, Sans-Serif, 'Twemoji Mozilla';
+$font-family: Nunito, 'Twemoji Mozilla', Arial, Helvetica, Sans-Serif;
 
 // XXX: In theory this should be Fira, but it's a bit ugly.
 // TODO: make it consistent cross-browser
-$monospace-font-family: Consolas, 'Liberation Mono', Menlo, Courier, monospace, 'Twemoji Mozilla';
+$monospace-font-family: Consolas, 'Liberation Mono', Courier, 'Twemoji Mozilla', monospace;
 
 // unified palette
 // try to use these colors when possible

scripts/emoji-data-strip.js

@@ -3,34 +3,27 @@
 // This generates src/stripped-emoji.json as used by the EmojiProvider autocomplete
 // provider.
 
-// FIXME: we no longer depends on emojione, so this generation script no longer
+const EMOJIBASE = require('emojibase-data/en/compact.json');
-// works, but the expectation is that we will shift to using emojimart or
-// similar as an emoji picker before this next needs to be run again.
 
-const EMOJI_DATA = require('emojione/emoji.json');
-const EMOJI_SUPPORTED = Object.keys(require('emojione').emojioneList);
 const fs = require('fs');
 
-const output = Object.keys(EMOJI_DATA).map(
+const output = EMOJIBASE.map(
-    (key) => {
+    (datum) => {
-        const datum = EMOJI_DATA[key];
         const newDatum = {
-            name: datum.name,
+            name: datum.annotation,
-            shortname: datum.shortname,
+            shortname: `:${datum.shortcodes[0]}:`,
-            category: datum.category,
+            category: datum.group,
-            emoji_order: datum.emoji_order,
+            emoji_order: datum.order,
         };
-        if (datum.aliases.length > 0) {
+        if (datum.shortcodes.length > 1) {
-            newDatum.aliases = datum.aliases;
+            newDatum.aliases = datum.shortcodes.slice(1).map(s => `:${s}:`);
         }
-        if (datum.aliases_ascii.length > 0) {
+        if (datum.emoticon) {
-            newDatum.aliases_ascii = datum.aliases_ascii;
+            newDatum.aliases_ascii = [ datum.emoticon ];
         }
         return newDatum;
     }
-).filter((datum) => {
+);
-    return EMOJI_SUPPORTED.includes(datum.shortname);
-});
 
 // Write to a file in src. Changes should be checked into git. This file is copied by
 // babel using --copy-files

src/HtmlUtils.js

@@ -74,19 +74,20 @@ export function containsEmoji(str) {
  * @return {String} The shortcode (such as :thumbup:)
  */
 export function unicodeToShortcode(char) {
-    const data = EMOJIBASE.find((e)=>{ e.unicode === char });
+    const data = EMOJIBASE.find(e => e.unicode === char);
-    return (data && data.shortcodes ? data.shortcodes[0] : '');
+    return (data && data.shortcodes ? `:${data.shortcodes[0]}:` : '');
 }
 
 /**
  * Returns the unicode character for an emoji shortcode
  *
  * @param {String} shortcode The shortcode (such as :thumbup:)
- * @return {String} The emoji character
+ * @return {String} The emoji character; null if none exists
  */
 export function shortcodeToUnicode(shortcode) {
-    const data = EMOJIBASE.find((e)=>{ e.shortcodes && e.shortcodes.contains(shortcode) });
+    shortcode = shortcode.slice(1, shortcode.length - 1);
-    return data.unicode;
+    const data = EMOJIBASE.find(e => e.shortcodes && e.shortcodes.includes(shortcode));
+    return data ? data.unicode : null;
 }
 
 export function processHtmlForSending   (html: string): string {

src/autocomplete/AutocompleteProvider.js

@@ -61,6 +61,7 @@ export default class AutocompleteProvider {
 
         let match;
         while ((match = commandRegex.exec(query)) != null) {
+            console.log('Matched ' + JSON.stringify(match));
             const start = match.index;
             const end = start + match[0].length;
             if (selection.start <= end && selection.end >= start) {

src/autocomplete/EmojiProvider.js

@@ -34,35 +34,16 @@ import SHORTCODE_REGEX from 'emojibase-regex/shortcode';
 import EmojiData from '../stripped-emoji.json';
 
 const LIMIT = 20;
-const CATEGORY_ORDER = [
-    'people',
-    'food',
-    'objects',
-    'activity',
-    'nature',
-    'travel',
-    'flags',
-    'regional',
-    'symbols',
-    'modifier',
-];
 
-// Match for ":wink:" or ascii-style ";-)" provided by emojibase
+// Match for ascii-style ";-)" emoticons or ":wink:" shortcodes provided by emojibase
-// (^|\s|(emojiUnicode)) to make sure we're either at the start of the string or there's a
+const EMOJI_REGEX = new RegExp('(' + EMOTICON_REGEX.source + '|:[+-\\w]*:?)$', 'g');
-// whitespace character or an emoji before the emoji. The reason for unicodeRegexp is
-// that we need to support inputting multiple emoji with no space between them.
-const EMOJI_REGEX = new RegExp('(?:^|\\s|' + UNICODE_REGEX.source + ')(' + EMOTICON_REGEX.source + '|:[+-\\w]*:?)$', 'g');
-
-// We also need to match the non-zero-length prefixes to remove them from the final match,
-// and update the range so that we don't replace the whitespace or the previous emoji.
-const MATCH_PREFIX_REGEX = new RegExp('(\\s|' + UNICODE_REGEX.source + ')');
 
 const EMOJI_SHORTNAMES = Object.keys(EmojiData).map((key) => EmojiData[key]).sort(
     (a, b) => {
         if (a.category === b.category) {
             return a.emoji_order - b.emoji_order;
         }
-        return CATEGORY_ORDER.indexOf(a.category) - CATEGORY_ORDER.indexOf(b.category);
+        return a.category - b.category;
     },
 ).map((a, index) => {
     return {
@@ -108,20 +89,18 @@ export default class EmojiProvider extends AutocompleteProvider {
         const {command, range} = this.getCurrentCommand(query, selection);
         if (command) {
             let matchedString = command[0];
-
-            // Remove prefix of any length (single whitespace or unicode emoji)
-            const prefixMatch = MATCH_PREFIX_REGEX.exec(matchedString);
-            if (prefixMatch) {
-                matchedString = matchedString.slice(prefixMatch[0].length);
-                range.start += prefixMatch[0].length;
-            }
             completions = this.matcher.match(matchedString);
 
             // Do second match with shouldMatchWordsOnly in order to match against 'name'
             completions = completions.concat(this.nameMatcher.match(matchedString));
 
+            console.log("pre-sorted completions", completions);
+
             const sorters = [];
-            // First, sort by score (Infinity if matchedString not in shortname)
+            // make sure that emoticons come first
+            sorters.push((c) => score(matchedString, c.aliases_ascii));
+
+            // then sort by score (Infinity if matchedString not in shortname)
             sorters.push((c) => score(matchedString, c.shortname));
             // If the matchedString is not empty, sort by length of shortname. Example:
             //  matchedString = ":bookmark"
@@ -144,6 +123,8 @@ export default class EmojiProvider extends AutocompleteProvider {
                     range,
                 };
             }).slice(0, LIMIT);
+
+            console.log("mapped completions", completions);
         }
         return completions;
     }

src/components/views/rooms/MessageComposerInput.js

@@ -61,7 +61,7 @@ import ReplyThread from "../elements/ReplyThread";
 import {ContentHelpers} from 'matrix-js-sdk';
 import AccessibleButton from '../elements/AccessibleButton';
 
-const REGEX_EMOJI_WHITESPACE = new RegExp('(?:^|\\s)(' + EMOTICON_REGEX + ')\\s$');
+const REGEX_EMOTICON_WHITESPACE = new RegExp('(?:^|\\s)(' + EMOTICON_REGEX.source + ')\\s$');
 
 const TYPING_USER_TIMEOUT = 10000; const TYPING_SERVER_TIMEOUT = 30000;
 
@@ -532,14 +532,14 @@ export default class MessageComposerInput extends React.Component {
             // Automatic replacement of plaintext emoji to Unicode emoji
             if (SettingsStore.getValue('MessageComposerInput.autoReplaceEmoji')) {
                 // The first matched group includes just the matched plaintext emoji
-                const emojiMatch = REGEX_EMOJI_WHITESPACE.exec(text.slice(0, currentStartOffset));
+                const emoticonMatch = REGEX_EMOTICON_WHITESPACE.exec(text.slice(0, currentStartOffset));
-                if (emojiMatch) {
+                if (emoticonMatch) {
-                    const unicodeEmoji = EMOJIBASE.find(e => e.emoticon && e.emoticon.contains(emoijMatch[1]));
+                    const unicodeEmoji = EMOJIBASE.find(e => e.emoticon && e.emoticon.contains(emoticonMatch[1]));
 
                     const range = Range.create({
                         anchor: {
                             key: editorState.startText.key,
-                            offset: currentStartOffset - emojiMatch[1].length - 1,
+                            offset: currentStartOffset - emoticonMatch[1].length - 1,
                         },
                         focus: {
                             key: editorState.startText.key,