From d4571aef6892eb73e41b56527a1fdeac9248aabe Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 21 Aug 2023 20:06:40 +0100 Subject: [PATCH] Consume @matrix-org/emojibase-bindings (#11433) --- package.json | 3 +- src/HtmlUtils.tsx | 2 +- src/autocomplete/EmojiProvider.tsx | 6 +- src/components/views/emojipicker/Category.tsx | 2 +- src/components/views/emojipicker/Emoji.tsx | 2 +- .../views/emojipicker/EmojiPicker.tsx | 2 +- src/components/views/emojipicker/Preview.tsx | 5 +- .../views/emojipicker/QuickReactions.tsx | 2 +- .../views/rooms/BasicMessageComposer.tsx | 2 +- src/emoji.ts | 120 ------------------ yarn.lock | 12 +- 11 files changed, 22 insertions(+), 136 deletions(-) delete mode 100644 src/emoji.ts diff --git a/package.json b/package.json index b5bdbeb5f0..3ce7b6809d 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "dependencies": { "@babel/runtime": "^7.12.5", "@matrix-org/analytics-events": "^0.6.0", + "@matrix-org/emojibase-bindings": "^1.1.2", "@matrix-org/matrix-wysiwyg": "^2.4.1", "@matrix-org/react-sdk-module-api": "^2.1.0", "@matrix-org/spec": "^1.7.0", @@ -76,8 +77,6 @@ "counterpart": "^0.18.6", "diff-dom": "^4.2.2", "diff-match-patch": "^1.0.5", - "emojibase": "15.0.0", - "emojibase-data": "15.0.0", "emojibase-regex": "15.0.0", "escape-html": "^1.0.3", "file-saver": "^2.0.5", diff --git a/src/HtmlUtils.tsx b/src/HtmlUtils.tsx index 17fb1e6ad0..78dcd4a139 100644 --- a/src/HtmlUtils.tsx +++ b/src/HtmlUtils.tsx @@ -29,6 +29,7 @@ import { Optional } from "matrix-events-sdk"; import _Linkify from "linkify-react"; import escapeHtml from "escape-html"; import GraphemeSplitter from "graphemer"; +import { getEmojiFromUnicode } from "@matrix-org/emojibase-bindings"; import { _linkifyElement, @@ -39,7 +40,6 @@ import { import { IExtendedSanitizeOptions } from "./@types/sanitize-html"; import SettingsStore from "./settings/SettingsStore"; import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks"; -import { getEmojiFromUnicode } from "./emoji"; import { mediaFromMxc } from "./customisations/Media"; import { stripHTMLReply, stripPlainReply } from "./utils/Reply"; import { PERMITTED_URL_SCHEMES } from "./utils/UrlUtils"; diff --git a/src/autocomplete/EmojiProvider.tsx b/src/autocomplete/EmojiProvider.tsx index fbf119dfcc..55359efb88 100644 --- a/src/autocomplete/EmojiProvider.tsx +++ b/src/autocomplete/EmojiProvider.tsx @@ -22,6 +22,7 @@ import React from "react"; import { uniq, sortBy, uniqBy, ListIteratee } from "lodash"; import EMOTICON_REGEX from "emojibase-regex/emoticon"; import { Room } from "matrix-js-sdk/src/matrix"; +import { EMOJI, Emoji, getEmojiFromUnicode } from "@matrix-org/emojibase-bindings"; import { _t } from "../languageHandler"; import AutocompleteProvider from "./AutocompleteProvider"; @@ -29,7 +30,6 @@ import QueryMatcher from "./QueryMatcher"; import { PillCompletion } from "./Components"; import { ICompletion, ISelectionRange } from "./Autocompleter"; import SettingsStore from "../settings/SettingsStore"; -import { EMOJI, IEmoji, getEmojiFromUnicode } from "../emoji"; import { TimelineRenderingType } from "../contexts/RoomContext"; import * as recent from "../emojipicker/recent"; import { filterBoolean } from "../utils/arrays"; @@ -41,7 +41,7 @@ const LIMIT = 20; const EMOJI_REGEX = new RegExp("(" + EMOTICON_REGEX.source + "|(?:^|\\s):[+-\\w]*:?)$", "g"); interface ISortedEmoji { - emoji: IEmoji; + emoji: Emoji; _orderBy: number; } @@ -79,7 +79,7 @@ function colonsTrimmed(str: string): string { export default class EmojiProvider extends AutocompleteProvider { public matcher: QueryMatcher; public nameMatcher: QueryMatcher; - private readonly recentlyUsed: IEmoji[]; + private readonly recentlyUsed: Emoji[]; public constructor(room: Room, renderingType?: TimelineRenderingType) { super({ commandRegex: EMOJI_REGEX, renderingType }); diff --git a/src/components/views/emojipicker/Category.tsx b/src/components/views/emojipicker/Category.tsx index cf662feea3..ce5e5c81c3 100644 --- a/src/components/views/emojipicker/Category.tsx +++ b/src/components/views/emojipicker/Category.tsx @@ -16,10 +16,10 @@ limitations under the License. */ import React, { RefObject } from "react"; +import { DATA_BY_CATEGORY, Emoji as IEmoji } from "@matrix-org/emojibase-bindings"; import { CATEGORY_HEADER_HEIGHT, EMOJI_HEIGHT, EMOJIS_PER_ROW } from "./EmojiPicker"; import LazyRenderList from "../elements/LazyRenderList"; -import { DATA_BY_CATEGORY, IEmoji } from "../../../emoji"; import Emoji from "./Emoji"; import { ButtonEvent } from "../elements/AccessibleButton"; diff --git a/src/components/views/emojipicker/Emoji.tsx b/src/components/views/emojipicker/Emoji.tsx index 6279887303..b210b62859 100644 --- a/src/components/views/emojipicker/Emoji.tsx +++ b/src/components/views/emojipicker/Emoji.tsx @@ -16,8 +16,8 @@ limitations under the License. */ import React from "react"; +import { Emoji as IEmoji } from "@matrix-org/emojibase-bindings"; -import { IEmoji } from "../../../emoji"; import { ButtonEvent } from "../elements/AccessibleButton"; import { RovingAccessibleButton } from "../../../accessibility/RovingTabIndex"; diff --git a/src/components/views/emojipicker/EmojiPicker.tsx b/src/components/views/emojipicker/EmojiPicker.tsx index edb5e427a3..51b3b77796 100644 --- a/src/components/views/emojipicker/EmojiPicker.tsx +++ b/src/components/views/emojipicker/EmojiPicker.tsx @@ -16,10 +16,10 @@ limitations under the License. */ import React, { Dispatch } from "react"; +import { DATA_BY_CATEGORY, getEmojiFromUnicode, Emoji as IEmoji } from "@matrix-org/emojibase-bindings"; import { _t } from "../../../languageHandler"; import * as recent from "../../../emojipicker/recent"; -import { DATA_BY_CATEGORY, getEmojiFromUnicode, IEmoji } from "../../../emoji"; import AutoHideScrollbar from "../../structures/AutoHideScrollbar"; import Header from "./Header"; import Search from "./Search"; diff --git a/src/components/views/emojipicker/Preview.tsx b/src/components/views/emojipicker/Preview.tsx index a13958efe1..0803592927 100644 --- a/src/components/views/emojipicker/Preview.tsx +++ b/src/components/views/emojipicker/Preview.tsx @@ -16,11 +16,10 @@ limitations under the License. */ import React from "react"; - -import { IEmoji } from "../../../emoji"; +import { Emoji } from "@matrix-org/emojibase-bindings"; interface IProps { - emoji: IEmoji; + emoji: Emoji; } class Preview extends React.PureComponent { diff --git a/src/components/views/emojipicker/QuickReactions.tsx b/src/components/views/emojipicker/QuickReactions.tsx index a58c6b875f..accf598503 100644 --- a/src/components/views/emojipicker/QuickReactions.tsx +++ b/src/components/views/emojipicker/QuickReactions.tsx @@ -16,9 +16,9 @@ limitations under the License. */ import React from "react"; +import { getEmojiFromUnicode, Emoji as IEmoji } from "@matrix-org/emojibase-bindings"; import { _t } from "../../../languageHandler"; -import { getEmojiFromUnicode, IEmoji } from "../../../emoji"; import Emoji from "./Emoji"; import { ButtonEvent } from "../elements/AccessibleButton"; import Toolbar from "../../../accessibility/Toolbar"; diff --git a/src/components/views/rooms/BasicMessageComposer.tsx b/src/components/views/rooms/BasicMessageComposer.tsx index e14aa9dc29..c1db383dc1 100644 --- a/src/components/views/rooms/BasicMessageComposer.tsx +++ b/src/components/views/rooms/BasicMessageComposer.tsx @@ -19,6 +19,7 @@ import React, { createRef, ClipboardEvent, SyntheticEvent } from "react"; import { Room, MatrixEvent } from "matrix-js-sdk/src/matrix"; import EMOTICON_REGEX from "emojibase-regex/emoticon"; import { logger } from "matrix-js-sdk/src/logger"; +import { EMOTICON_TO_EMOJI } from "@matrix-org/emojibase-bindings"; import EditorModel from "../../../editor/model"; import HistoryManager from "../../../editor/history"; @@ -36,7 +37,6 @@ import { parseEvent, parsePlainTextMessage } from "../../../editor/deserialize"; import { renderModel } from "../../../editor/render"; import SettingsStore from "../../../settings/SettingsStore"; import { IS_MAC, Key } from "../../../Keyboard"; -import { EMOTICON_TO_EMOJI } from "../../../emoji"; import { CommandCategories, CommandMap, parseCommandString } from "../../../SlashCommands"; import Range from "../../../editor/range"; import MessageComposerFormatBar, { Formatting } from "./MessageComposerFormatBar"; diff --git a/src/emoji.ts b/src/emoji.ts deleted file mode 100644 index faebd299e5..0000000000 --- a/src/emoji.ts +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2019 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import EMOJIBASE from "emojibase-data/en/compact.json"; -import SHORTCODES from "emojibase-data/en/shortcodes/iamcal.json"; -import { CompactEmoji } from "emojibase"; - -export interface IEmoji extends Omit { - // We generate a shortcode based on the label if none exist in the dataset - shortcodes: string[]; -} - -// The unicode is stored without the variant selector -const UNICODE_TO_EMOJI = new Map(); // not exported as gets for it are handled by getEmojiFromUnicode -export const EMOTICON_TO_EMOJI = new Map(); - -export const getEmojiFromUnicode = (unicode: string): IEmoji | undefined => - UNICODE_TO_EMOJI.get(stripVariation(unicode)); - -const isRegionalIndicator = (x: string): boolean => { - // First verify that the string is a single character. We use Array.from - // to make sure we count by characters, not UTF-8 code units. - return ( - Array.from(x).length === 1 && - // Next verify that the character is within the code point range for - // regional indicators. - // http://unicode.org/charts/PDF/Unicode-6.0/U60-1F100.pdf - x >= "\u{1f1e6}" && - x <= "\u{1f1ff}" - ); -}; - -const EMOJIBASE_GROUP_ID_TO_CATEGORY = [ - "people", // smileys - "people", // actually people - "control", // modifiers and such, not displayed in picker - "nature", - "foods", - "places", - "activity", - "objects", - "symbols", - "flags", -]; - -export const DATA_BY_CATEGORY: Record = { - people: [], - nature: [], - foods: [], - places: [], - activity: [], - objects: [], - symbols: [], - flags: [], -}; - -// Store various mappings from unicode/emoticon/shortcode to the Emoji objects -export const EMOJI: IEmoji[] = EMOJIBASE.map((emojiData) => { - // If there's ever a gap in shortcode coverage, we fudge it by - // filling it in with the emoji's CLDR annotation - const shortcodeData = SHORTCODES[emojiData.hexcode] ?? [emojiData.label.toLowerCase().replace(/\W+/g, "_")]; - - const emoji: IEmoji = { - ...emojiData, - // Homogenize shortcodes by ensuring that everything is an array - shortcodes: typeof shortcodeData === "string" ? [shortcodeData] : shortcodeData, - }; - - // We manually include regional indicators in the symbols group, since - // Emojibase intentionally leaves them uncategorized - const categoryId = - EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group!] ?? (isRegionalIndicator(emoji.unicode) ? "symbols" : null); - - if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) { - DATA_BY_CATEGORY[categoryId].push(emoji); - } - - // Add mapping from unicode to Emoji object - // The 'unicode' field that we use in emojibase has either - // VS15 or VS16 appended to any characters that can take - // variation selectors. Which one it appends depends - // on whether emojibase considers their type to be 'text' or - // 'emoji'. We therefore strip any variation chars from strings - // both when building the map and when looking up. - UNICODE_TO_EMOJI.set(stripVariation(emoji.unicode), emoji); - - if (emoji.emoticon) { - // Add mapping from emoticon to Emoji object - Array.isArray(emoji.emoticon) - ? emoji.emoticon.forEach((x) => EMOTICON_TO_EMOJI.set(x, emoji)) - : EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji); - } - - return emoji; -}); - -/** - * Strips variation selectors from the end of given string - * NB. Skin tone modifiers are not variation selectors: - * this function does not touch them. (Should it?) - * - * @param {string} str string to strip - * @returns {string} stripped string - */ -function stripVariation(str: string): string { - return str.replace(/[\uFE00-\uFE0F]$/, ""); -} diff --git a/yarn.lock b/yarn.lock index 1cb735557a..7d6df6f3ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1853,6 +1853,14 @@ resolved "https://registry.yarnpkg.com/@matrix-org/analytics-events/-/analytics-events-0.6.0.tgz#6552882f94d026f13da25d49e2a208287521c275" integrity sha512-bTvNpp8LkC/2sItHABd1vGHdB8iclAcdlIYrL0Cn6qT+aohpdjb1wZ0dhUcx3NK5Q98IduI43RVH33V4Li/X0A== +"@matrix-org/emojibase-bindings@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@matrix-org/emojibase-bindings/-/emojibase-bindings-1.1.2.tgz#3cbbed06258418895910b8778a3d9c885f0c48c5" + integrity sha512-6FLR4nzyeQPZl2FBsdPpbAvvDF7TuAZgEbNeFkID47/bzTovFS4MUXIHOMzwMy/PWehlVziuKMOe1AxD9PauKw== + dependencies: + emojibase "^15.0.0" + emojibase-data "^15.0.0" + "@matrix-org/matrix-sdk-crypto-wasm@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-1.2.0.tgz#115cd21cb2bba3c8166cf09e7d61da0902aa8973" @@ -4481,7 +4489,7 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== -emojibase-data@15.0.0: +emojibase-data@^15.0.0: version "15.0.0" resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-15.0.0.tgz#d1f5467f3080688b9605103d0abdcd54bbc76419" integrity sha512-hqrLNhEeBejKOQp5ArJcofkzV3qZBcp8czXj8nIKUGpBVc50NddNGwir4yAYxn3oNgrSj/lYdB9XxIVAKTkong== @@ -4491,7 +4499,7 @@ emojibase-regex@15.0.0: resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-15.0.0.tgz#b4d1c6328500aaea4a794b11fe61f97df20af4ee" integrity sha512-b5y58xrmZhH551zIa3ZOHl1mRI5KecA+5sAyWZCQEaj1maufZJJoENVwDqigzJoAkG604DuRqfdpy4E5rzSUsg== -emojibase@15.0.0: +emojibase@^15.0.0: version "15.0.0" resolved "https://registry.yarnpkg.com/emojibase/-/emojibase-15.0.0.tgz#f41b7773ec9a8a332373c18628ff4471255bd769" integrity sha512-bvSIs98sHaVnyKPmW+obRjo49MFx0g+rhfSz6mTePAagEZSlDPosq0b6AcSJa5gt48z3VP2ooXclyBs8vIkpGA==