Consume @matrix-org/emojibase-bindings (#11433)

package.json

@@ -61,6 +61,7 @@
     "dependencies": {
         "@babel/runtime": "^7.12.5",
         "@matrix-org/analytics-events": "^0.6.0",
+        "@matrix-org/emojibase-bindings": "^1.1.2",
         "@matrix-org/matrix-wysiwyg": "^2.4.1",
         "@matrix-org/react-sdk-module-api": "^2.1.0",
         "@matrix-org/spec": "^1.7.0",
@@ -76,8 +77,6 @@
         "counterpart": "^0.18.6",
         "diff-dom": "^4.2.2",
         "diff-match-patch": "^1.0.5",
-        "emojibase": "15.0.0",
-        "emojibase-data": "15.0.0",
         "emojibase-regex": "15.0.0",
         "escape-html": "^1.0.3",
         "file-saver": "^2.0.5",

src/HtmlUtils.tsx

@@ -29,6 +29,7 @@ import { Optional } from "matrix-events-sdk";
 import _Linkify from "linkify-react";
 import escapeHtml from "escape-html";
 import GraphemeSplitter from "graphemer";
+import { getEmojiFromUnicode } from "@matrix-org/emojibase-bindings";
 
 import {
     _linkifyElement,
@@ -39,7 +40,6 @@ import {
 import { IExtendedSanitizeOptions } from "./@types/sanitize-html";
 import SettingsStore from "./settings/SettingsStore";
 import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks";
-import { getEmojiFromUnicode } from "./emoji";
 import { mediaFromMxc } from "./customisations/Media";
 import { stripHTMLReply, stripPlainReply } from "./utils/Reply";
 import { PERMITTED_URL_SCHEMES } from "./utils/UrlUtils";

src/autocomplete/EmojiProvider.tsx

@@ -22,6 +22,7 @@ import React from "react";
 import { uniq, sortBy, uniqBy, ListIteratee } from "lodash";
 import EMOTICON_REGEX from "emojibase-regex/emoticon";
 import { Room } from "matrix-js-sdk/src/matrix";
+import { EMOJI, Emoji, getEmojiFromUnicode } from "@matrix-org/emojibase-bindings";
 
 import { _t } from "../languageHandler";
 import AutocompleteProvider from "./AutocompleteProvider";
@@ -29,7 +30,6 @@ import QueryMatcher from "./QueryMatcher";
 import { PillCompletion } from "./Components";
 import { ICompletion, ISelectionRange } from "./Autocompleter";
 import SettingsStore from "../settings/SettingsStore";
-import { EMOJI, IEmoji, getEmojiFromUnicode } from "../emoji";
 import { TimelineRenderingType } from "../contexts/RoomContext";
 import * as recent from "../emojipicker/recent";
 import { filterBoolean } from "../utils/arrays";
@@ -41,7 +41,7 @@ const LIMIT = 20;
 const EMOJI_REGEX = new RegExp("(" + EMOTICON_REGEX.source + "|(?:^|\\s):[+-\\w]*:?)$", "g");
 
 interface ISortedEmoji {
-    emoji: IEmoji;
+    emoji: Emoji;
     _orderBy: number;
 }
 
@@ -79,7 +79,7 @@ function colonsTrimmed(str: string): string {
 export default class EmojiProvider extends AutocompleteProvider {
     public matcher: QueryMatcher<ISortedEmoji>;
     public nameMatcher: QueryMatcher<ISortedEmoji>;
-    private readonly recentlyUsed: IEmoji[];
+    private readonly recentlyUsed: Emoji[];
 
     public constructor(room: Room, renderingType?: TimelineRenderingType) {
         super({ commandRegex: EMOJI_REGEX, renderingType });

src/components/views/emojipicker/Category.tsx

@@ -16,10 +16,10 @@ limitations under the License.
 */
 
 import React, { RefObject } from "react";
+import { DATA_BY_CATEGORY, Emoji as IEmoji } from "@matrix-org/emojibase-bindings";
 
 import { CATEGORY_HEADER_HEIGHT, EMOJI_HEIGHT, EMOJIS_PER_ROW } from "./EmojiPicker";
 import LazyRenderList from "../elements/LazyRenderList";
-import { DATA_BY_CATEGORY, IEmoji } from "../../../emoji";
 import Emoji from "./Emoji";
 import { ButtonEvent } from "../elements/AccessibleButton";
 

src/components/views/emojipicker/Emoji.tsx

@@ -16,8 +16,8 @@ limitations under the License.
 */
 
 import React from "react";
+import { Emoji as IEmoji } from "@matrix-org/emojibase-bindings";
 
-import { IEmoji } from "../../../emoji";
 import { ButtonEvent } from "../elements/AccessibleButton";
 import { RovingAccessibleButton } from "../../../accessibility/RovingTabIndex";
 

src/components/views/emojipicker/EmojiPicker.tsx

@@ -16,10 +16,10 @@ limitations under the License.
 */
 
 import React, { Dispatch } from "react";
+import { DATA_BY_CATEGORY, getEmojiFromUnicode, Emoji as IEmoji } from "@matrix-org/emojibase-bindings";
 
 import { _t } from "../../../languageHandler";
 import * as recent from "../../../emojipicker/recent";
-import { DATA_BY_CATEGORY, getEmojiFromUnicode, IEmoji } from "../../../emoji";
 import AutoHideScrollbar from "../../structures/AutoHideScrollbar";
 import Header from "./Header";
 import Search from "./Search";

src/components/views/emojipicker/Preview.tsx

@@ -16,11 +16,10 @@ limitations under the License.
 */
 
 import React from "react";
-
+import { Emoji } from "@matrix-org/emojibase-bindings";
-import { IEmoji } from "../../../emoji";
 
 interface IProps {
-    emoji: IEmoji;
+    emoji: Emoji;
 }
 
 class Preview extends React.PureComponent<IProps> {

src/components/views/emojipicker/QuickReactions.tsx

@@ -16,9 +16,9 @@ limitations under the License.
 */
 
 import React from "react";
+import { getEmojiFromUnicode, Emoji as IEmoji } from "@matrix-org/emojibase-bindings";
 
 import { _t } from "../../../languageHandler";
-import { getEmojiFromUnicode, IEmoji } from "../../../emoji";
 import Emoji from "./Emoji";
 import { ButtonEvent } from "../elements/AccessibleButton";
 import Toolbar from "../../../accessibility/Toolbar";

src/components/views/rooms/BasicMessageComposer.tsx

@@ -19,6 +19,7 @@ import React, { createRef, ClipboardEvent, SyntheticEvent } from "react";
 import { Room, MatrixEvent } from "matrix-js-sdk/src/matrix";
 import EMOTICON_REGEX from "emojibase-regex/emoticon";
 import { logger } from "matrix-js-sdk/src/logger";
+import { EMOTICON_TO_EMOJI } from "@matrix-org/emojibase-bindings";
 
 import EditorModel from "../../../editor/model";
 import HistoryManager from "../../../editor/history";
@@ -36,7 +37,6 @@ import { parseEvent, parsePlainTextMessage } from "../../../editor/deserialize";
 import { renderModel } from "../../../editor/render";
 import SettingsStore from "../../../settings/SettingsStore";
 import { IS_MAC, Key } from "../../../Keyboard";
-import { EMOTICON_TO_EMOJI } from "../../../emoji";
 import { CommandCategories, CommandMap, parseCommandString } from "../../../SlashCommands";
 import Range from "../../../editor/range";
 import MessageComposerFormatBar, { Formatting } from "./MessageComposerFormatBar";

src/emoji.ts

@@ -1,120 +0,0 @@
-/*
-Copyright 2019 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import EMOJIBASE from "emojibase-data/en/compact.json";
-import SHORTCODES from "emojibase-data/en/shortcodes/iamcal.json";
-import { CompactEmoji } from "emojibase";
-
-export interface IEmoji extends Omit<CompactEmoji, "shortcodes"> {
-    // We generate a shortcode based on the label if none exist in the dataset
-    shortcodes: string[];
-}
-
-// The unicode is stored without the variant selector
-const UNICODE_TO_EMOJI = new Map<string, IEmoji>(); // not exported as gets for it are handled by getEmojiFromUnicode
-export const EMOTICON_TO_EMOJI = new Map<string, IEmoji>();
-
-export const getEmojiFromUnicode = (unicode: string): IEmoji | undefined =>
-    UNICODE_TO_EMOJI.get(stripVariation(unicode));
-
-const isRegionalIndicator = (x: string): boolean => {
-    // First verify that the string is a single character. We use Array.from
-    // to make sure we count by characters, not UTF-8 code units.
-    return (
-        Array.from(x).length === 1 &&
-        // Next verify that the character is within the code point range for
-        // regional indicators.
-        // http://unicode.org/charts/PDF/Unicode-6.0/U60-1F100.pdf
-        x >= "\u{1f1e6}" &&
-        x <= "\u{1f1ff}"
-    );
-};
-
-const EMOJIBASE_GROUP_ID_TO_CATEGORY = [
-    "people", // smileys
-    "people", // actually people
-    "control", // modifiers and such, not displayed in picker
-    "nature",
-    "foods",
-    "places",
-    "activity",
-    "objects",
-    "symbols",
-    "flags",
-];
-
-export const DATA_BY_CATEGORY: Record<string, IEmoji[]> = {
-    people: [],
-    nature: [],
-    foods: [],
-    places: [],
-    activity: [],
-    objects: [],
-    symbols: [],
-    flags: [],
-};
-
-// Store various mappings from unicode/emoticon/shortcode to the Emoji objects
-export const EMOJI: IEmoji[] = EMOJIBASE.map((emojiData) => {
-    // If there's ever a gap in shortcode coverage, we fudge it by
-    // filling it in with the emoji's CLDR annotation
-    const shortcodeData = SHORTCODES[emojiData.hexcode] ?? [emojiData.label.toLowerCase().replace(/\W+/g, "_")];
-
-    const emoji: IEmoji = {
-        ...emojiData,
-        // Homogenize shortcodes by ensuring that everything is an array
-        shortcodes: typeof shortcodeData === "string" ? [shortcodeData] : shortcodeData,
-    };
-
-    // We manually include regional indicators in the symbols group, since
-    // Emojibase intentionally leaves them uncategorized
-    const categoryId =
-        EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group!] ?? (isRegionalIndicator(emoji.unicode) ? "symbols" : null);
-
-    if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
-        DATA_BY_CATEGORY[categoryId].push(emoji);
-    }
-
-    // Add mapping from unicode to Emoji object
-    // The 'unicode' field that we use in emojibase has either
-    // VS15 or VS16 appended to any characters that can take
-    // variation selectors. Which one it appends depends
-    // on whether emojibase considers their type to be 'text' or
-    // 'emoji'. We therefore strip any variation chars from strings
-    // both when building the map and when looking up.
-    UNICODE_TO_EMOJI.set(stripVariation(emoji.unicode), emoji);
-
-    if (emoji.emoticon) {
-        // Add mapping from emoticon to Emoji object
-        Array.isArray(emoji.emoticon)
-            ? emoji.emoticon.forEach((x) => EMOTICON_TO_EMOJI.set(x, emoji))
-            : EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji);
-    }
-
-    return emoji;
-});
-
-/**
- * Strips variation selectors from the end of given string
- * NB. Skin tone modifiers are not variation selectors:
- * this function does not touch them. (Should it?)
- *
- * @param {string} str string to strip
- * @returns {string} stripped string
- */
-function stripVariation(str: string): string {
-    return str.replace(/[\uFE00-\uFE0F]$/, "");
-}

yarn.lock

@@ -1853,6 +1853,14 @@
   resolved "https://registry.yarnpkg.com/@matrix-org/analytics-events/-/analytics-events-0.6.0.tgz#6552882f94d026f13da25d49e2a208287521c275"
   integrity sha512-bTvNpp8LkC/2sItHABd1vGHdB8iclAcdlIYrL0Cn6qT+aohpdjb1wZ0dhUcx3NK5Q98IduI43RVH33V4Li/X0A==
 
+"@matrix-org/emojibase-bindings@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@matrix-org/emojibase-bindings/-/emojibase-bindings-1.1.2.tgz#3cbbed06258418895910b8778a3d9c885f0c48c5"
+  integrity sha512-6FLR4nzyeQPZl2FBsdPpbAvvDF7TuAZgEbNeFkID47/bzTovFS4MUXIHOMzwMy/PWehlVziuKMOe1AxD9PauKw==
+  dependencies:
+    emojibase "^15.0.0"
+    emojibase-data "^15.0.0"
+
 "@matrix-org/matrix-sdk-crypto-wasm@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-1.2.0.tgz#115cd21cb2bba3c8166cf09e7d61da0902aa8973"
@@ -4481,7 +4489,7 @@ emoji-regex@^9.2.2:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72"
   integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==
 
-emojibase-data@15.0.0:
+emojibase-data@^15.0.0:
   version "15.0.0"
   resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-15.0.0.tgz#d1f5467f3080688b9605103d0abdcd54bbc76419"
   integrity sha512-hqrLNhEeBejKOQp5ArJcofkzV3qZBcp8czXj8nIKUGpBVc50NddNGwir4yAYxn3oNgrSj/lYdB9XxIVAKTkong==
@@ -4491,7 +4499,7 @@ emojibase-regex@15.0.0:
   resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-15.0.0.tgz#b4d1c6328500aaea4a794b11fe61f97df20af4ee"
   integrity sha512-b5y58xrmZhH551zIa3ZOHl1mRI5KecA+5sAyWZCQEaj1maufZJJoENVwDqigzJoAkG604DuRqfdpy4E5rzSUsg==
 
-emojibase@15.0.0:
+emojibase@^15.0.0:
   version "15.0.0"
   resolved "https://registry.yarnpkg.com/emojibase/-/emojibase-15.0.0.tgz#f41b7773ec9a8a332373c18628ff4471255bd769"
   integrity sha512-bvSIs98sHaVnyKPmW+obRjo49MFx0g+rhfSz6mTePAagEZSlDPosq0b6AcSJa5gt48z3VP2ooXclyBs8vIkpGA==