2021-07-26 18:10:27 +00:00
|
|
|
/*
|
2022-10-18 01:54:10 +00:00
|
|
|
Copyright 2021 - 2022 The Matrix.org Foundation C.I.C.
|
2021-07-26 18:10:27 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2021-06-22 05:21:16 +00:00
|
|
|
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
|
|
|
|
import { Room } from "matrix-js-sdk/src/models/room";
|
2021-10-22 22:23:32 +00:00
|
|
|
import { MatrixClient } from "matrix-js-sdk/src/client";
|
|
|
|
import { Direction } from "matrix-js-sdk/src/models/event-timeline";
|
|
|
|
import { saveAs } from "file-saver";
|
|
|
|
import { logger } from "matrix-js-sdk/src/logger";
|
2022-10-18 01:54:10 +00:00
|
|
|
import sanitizeFilename from "sanitize-filename";
|
2021-10-22 22:23:32 +00:00
|
|
|
|
2021-06-22 05:21:16 +00:00
|
|
|
import { MatrixClientPeg } from "../../MatrixClientPeg";
|
2021-12-15 16:21:51 +00:00
|
|
|
import { ExportType, IExportOptions } from "./exportUtils";
|
2021-06-22 05:21:16 +00:00
|
|
|
import { decryptFile } from "../DecryptFile";
|
|
|
|
import { mediaFromContent } from "../../customisations/Media";
|
2022-10-18 01:54:10 +00:00
|
|
|
import { formatFullDateNoDay, formatFullDateNoDayISO } from "../../DateUtils";
|
2021-10-01 09:22:37 +00:00
|
|
|
import { isVoiceMessage } from "../EventUtils";
|
|
|
|
import { IMediaEventContent } from "../../customisations/models/IMediaEventContent";
|
2021-08-13 18:14:07 +00:00
|
|
|
import { _t } from "../../languageHandler";
|
2021-09-22 16:47:23 +00:00
|
|
|
import SdkConfig from "../../SdkConfig";
|
2021-05-24 15:18:13 +00:00
|
|
|
|
2021-07-26 16:45:05 +00:00
|
|
|
type BlobFile = {
|
2021-06-29 04:49:57 +00:00
|
|
|
name: string;
|
2021-07-26 16:45:05 +00:00
|
|
|
blob: Blob;
|
2021-06-23 06:28:50 +00:00
|
|
|
};
|
|
|
|
|
2021-06-03 08:09:14 +00:00
|
|
|
export default abstract class Exporter {
|
2021-08-13 03:29:28 +00:00
|
|
|
protected files: BlobFile[] = [];
|
2021-06-24 12:49:12 +00:00
|
|
|
protected client: MatrixClient;
|
2021-08-13 03:29:28 +00:00
|
|
|
protected cancelled = false;
|
2021-06-25 05:46:59 +00:00
|
|
|
|
2021-06-11 06:34:05 +00:00
|
|
|
protected constructor(
|
|
|
|
protected room: Room,
|
2021-08-13 03:00:50 +00:00
|
|
|
protected exportType: ExportType,
|
2021-07-26 18:10:27 +00:00
|
|
|
protected exportOptions: IExportOptions,
|
2021-08-13 18:33:02 +00:00
|
|
|
protected setProgressText: React.Dispatch<React.SetStateAction<string>>,
|
2021-06-23 06:28:50 +00:00
|
|
|
) {
|
2022-12-12 11:24:14 +00:00
|
|
|
if (
|
|
|
|
exportOptions.maxSize < 1 * 1024 * 1024 || // Less than 1 MB
|
2022-01-31 11:54:14 +00:00
|
|
|
exportOptions.maxSize > 8000 * 1024 * 1024 || // More than 8 GB
|
2022-12-12 11:24:14 +00:00
|
|
|
exportOptions.numberOfMessages > 10 ** 8
|
2021-08-13 03:00:50 +00:00
|
|
|
) {
|
2021-08-03 09:06:21 +00:00
|
|
|
throw new Error("Invalid export options");
|
|
|
|
}
|
2021-06-24 12:49:12 +00:00
|
|
|
this.client = MatrixClientPeg.get();
|
2021-06-25 05:46:59 +00:00
|
|
|
window.addEventListener("beforeunload", this.onBeforeUnload);
|
|
|
|
}
|
|
|
|
|
2022-10-18 01:54:10 +00:00
|
|
|
public get destinationFileName(): string {
|
|
|
|
return this.makeFileNameNoExtension(SdkConfig.get().brand) + ".zip";
|
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected onBeforeUnload(e: BeforeUnloadEvent): string {
|
2021-06-25 05:46:59 +00:00
|
|
|
e.preventDefault();
|
2022-12-12 11:24:14 +00:00
|
|
|
return (e.returnValue = _t("Are you sure you want to exit during this export?"));
|
2021-06-23 06:28:50 +00:00
|
|
|
}
|
|
|
|
|
2021-07-02 04:53:25 +00:00
|
|
|
protected updateProgress(progress: string, log = true, show = true): void {
|
2021-10-15 14:26:54 +00:00
|
|
|
if (log) logger.log(progress);
|
2021-08-13 18:33:02 +00:00
|
|
|
if (show) this.setProgressText(progress);
|
2021-07-02 04:53:25 +00:00
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected addFile(filePath: string, blob: Blob): void {
|
2021-06-23 06:28:50 +00:00
|
|
|
const file = {
|
|
|
|
name: filePath,
|
2021-07-26 16:45:05 +00:00
|
|
|
blob,
|
2021-06-30 08:38:22 +00:00
|
|
|
};
|
2021-06-23 06:28:50 +00:00
|
|
|
this.files.push(file);
|
|
|
|
}
|
|
|
|
|
2022-10-18 01:54:10 +00:00
|
|
|
protected makeFileNameNoExtension(brand = "matrix"): string {
|
|
|
|
// First try to use the real name of the room, then a translated copy of a generic name,
|
|
|
|
// then finally hardcoded default to guarantee we'll have a name.
|
|
|
|
const safeRoomName = sanitizeFilename(this.room.name ?? _t("Unnamed Room")).trim() || "Unnamed Room";
|
2022-12-12 11:24:14 +00:00
|
|
|
const safeDate = formatFullDateNoDayISO(new Date()).replace(/:/g, "-"); // ISO format automatically removes a lot of stuff for us
|
2022-10-18 01:54:10 +00:00
|
|
|
const safeBrand = sanitizeFilename(brand);
|
|
|
|
return `${safeBrand} - ${safeRoomName} - Chat Export - ${safeDate}`;
|
|
|
|
}
|
|
|
|
|
2021-08-13 18:14:07 +00:00
|
|
|
protected async downloadZIP(): Promise<string | void> {
|
2022-10-18 01:54:10 +00:00
|
|
|
const filename = this.destinationFileName;
|
2023-01-03 06:53:33 +00:00
|
|
|
const filenameWithoutExt = filename.substring(0, filename.lastIndexOf(".")); // take off the extension
|
2022-12-12 11:24:14 +00:00
|
|
|
const { default: JSZip } = await import("jszip");
|
2021-06-25 06:22:26 +00:00
|
|
|
|
2021-07-26 16:45:05 +00:00
|
|
|
const zip = new JSZip();
|
2021-06-25 06:22:26 +00:00
|
|
|
// Create a writable stream to the directory
|
2021-12-15 16:21:51 +00:00
|
|
|
if (!this.cancelled) this.updateProgress(_t("Generating a ZIP"));
|
2021-06-27 15:25:54 +00:00
|
|
|
else return this.cleanUp();
|
2021-06-25 06:22:26 +00:00
|
|
|
|
2021-11-15 21:37:16 +00:00
|
|
|
for (const file of this.files) zip.file(filenameWithoutExt + "/" + file.name, file.blob);
|
2021-06-25 05:46:59 +00:00
|
|
|
|
2021-07-26 16:45:05 +00:00
|
|
|
const content = await zip.generateAsync({ type: "blob" });
|
2023-01-03 06:53:33 +00:00
|
|
|
saveAs(content, filenameWithoutExt + ".zip");
|
2021-06-25 05:46:59 +00:00
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected cleanUp(): string {
|
2021-10-15 14:26:54 +00:00
|
|
|
logger.log("Cleaning up...");
|
2021-06-27 15:25:54 +00:00
|
|
|
window.removeEventListener("beforeunload", this.onBeforeUnload);
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
public async cancelExport(): Promise<void> {
|
2021-10-15 14:26:54 +00:00
|
|
|
logger.log("Cancelling export...");
|
2021-06-27 15:25:54 +00:00
|
|
|
this.cancelled = true;
|
|
|
|
}
|
|
|
|
|
2023-01-12 13:25:14 +00:00
|
|
|
protected downloadPlainText(fileName: string, text: string): void {
|
2022-09-01 10:12:01 +00:00
|
|
|
const content = new Blob([text], { type: "text/plain" });
|
2021-08-09 07:06:06 +00:00
|
|
|
saveAs(content, fileName);
|
2021-06-23 06:28:50 +00:00
|
|
|
}
|
2021-06-03 07:51:56 +00:00
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected setEventMetadata(event: MatrixEvent): MatrixEvent {
|
2021-06-24 12:49:12 +00:00
|
|
|
const roomState = this.client.getRoom(this.room.roomId).currentState;
|
2022-12-12 11:24:14 +00:00
|
|
|
event.sender = roomState.getSentinelMember(event.getSender());
|
2021-06-07 06:04:03 +00:00
|
|
|
if (event.getType() === "m.room.member") {
|
2022-12-12 11:24:14 +00:00
|
|
|
event.target = roomState.getSentinelMember(event.getStateKey());
|
2021-06-04 09:38:17 +00:00
|
|
|
}
|
2021-06-07 06:04:03 +00:00
|
|
|
return event;
|
2021-06-04 09:38:17 +00:00
|
|
|
}
|
|
|
|
|
2021-08-03 09:06:21 +00:00
|
|
|
public getLimit(): number {
|
2021-06-11 06:34:05 +00:00
|
|
|
let limit: number;
|
|
|
|
switch (this.exportType) {
|
2021-08-13 03:00:50 +00:00
|
|
|
case ExportType.LastNMessages:
|
2021-06-11 06:34:05 +00:00
|
|
|
limit = this.exportOptions.numberOfMessages;
|
|
|
|
break;
|
2021-08-13 03:00:50 +00:00
|
|
|
case ExportType.Timeline:
|
2021-06-11 06:34:05 +00:00
|
|
|
limit = 40;
|
|
|
|
break;
|
|
|
|
default:
|
2022-12-12 11:24:14 +00:00
|
|
|
limit = 10 ** 8;
|
2021-06-11 06:34:05 +00:00
|
|
|
}
|
|
|
|
return limit;
|
|
|
|
}
|
|
|
|
|
2021-06-30 08:38:22 +00:00
|
|
|
protected async getRequiredEvents(): Promise<MatrixEvent[]> {
|
2021-06-24 12:49:12 +00:00
|
|
|
const eventMapper = this.client.getEventMapper();
|
2021-06-04 09:38:17 +00:00
|
|
|
|
2022-12-12 11:24:14 +00:00
|
|
|
let prevToken: string | null = null;
|
2021-06-11 06:34:05 +00:00
|
|
|
let limit = this.getLimit();
|
2021-07-19 07:30:37 +00:00
|
|
|
const events: MatrixEvent[] = [];
|
2021-06-07 06:04:03 +00:00
|
|
|
|
2021-06-04 09:38:17 +00:00
|
|
|
while (limit) {
|
2021-06-10 06:23:41 +00:00
|
|
|
const eventsPerCrawl = Math.min(limit, 1000);
|
2021-07-19 07:58:09 +00:00
|
|
|
const res = await this.client.createMessagesRequest(
|
|
|
|
this.room.roomId,
|
|
|
|
prevToken,
|
|
|
|
eventsPerCrawl,
|
|
|
|
Direction.Backward,
|
|
|
|
);
|
2021-06-04 09:38:17 +00:00
|
|
|
|
2021-06-27 15:25:54 +00:00
|
|
|
if (this.cancelled) {
|
|
|
|
this.cleanUp();
|
|
|
|
return [];
|
|
|
|
}
|
|
|
|
|
2021-06-04 09:38:17 +00:00
|
|
|
if (res.chunk.length === 0) break;
|
|
|
|
|
2021-06-10 06:23:41 +00:00
|
|
|
limit -= res.chunk.length;
|
2021-06-04 09:38:17 +00:00
|
|
|
|
|
|
|
const matrixEvents: MatrixEvent[] = res.chunk.map(eventMapper);
|
|
|
|
|
2021-06-11 06:34:05 +00:00
|
|
|
for (const mxEv of matrixEvents) {
|
2021-08-03 09:06:21 +00:00
|
|
|
// if (this.exportOptions.startDate && mxEv.getTs() < this.exportOptions.startDate) {
|
|
|
|
// // Once the last message received is older than the start date, we break out of both the loops
|
|
|
|
// limit = 0;
|
|
|
|
// break;
|
|
|
|
// }
|
2021-06-11 06:34:05 +00:00
|
|
|
events.push(mxEv);
|
|
|
|
}
|
2021-12-15 16:21:51 +00:00
|
|
|
|
|
|
|
if (this.exportType === ExportType.LastNMessages) {
|
2022-12-12 11:24:14 +00:00
|
|
|
this.updateProgress(
|
|
|
|
_t("Fetched %(count)s events out of %(total)s", {
|
|
|
|
count: events.length,
|
|
|
|
total: this.exportOptions.numberOfMessages,
|
|
|
|
}),
|
|
|
|
);
|
2021-12-15 16:21:51 +00:00
|
|
|
} else {
|
2022-12-12 11:24:14 +00:00
|
|
|
this.updateProgress(
|
|
|
|
_t("Fetched %(count)s events so far", {
|
|
|
|
count: events.length,
|
|
|
|
}),
|
|
|
|
);
|
2021-12-15 16:21:51 +00:00
|
|
|
}
|
|
|
|
|
2021-06-04 09:38:17 +00:00
|
|
|
prevToken = res.end;
|
|
|
|
}
|
2021-06-08 13:07:36 +00:00
|
|
|
// Reverse the events so that we preserve the order
|
2022-12-12 11:24:14 +00:00
|
|
|
for (let i = 0; i < Math.floor(events.length / 2); i++) {
|
2021-07-19 07:30:37 +00:00
|
|
|
[events[i], events[events.length - i - 1]] = [events[events.length - i - 1], events[i]];
|
|
|
|
}
|
2021-06-04 09:38:17 +00:00
|
|
|
|
|
|
|
const decryptionPromises = events
|
2022-12-12 11:24:14 +00:00
|
|
|
.filter((event) => event.isEncrypted())
|
|
|
|
.map((event) => {
|
2021-06-24 12:49:12 +00:00
|
|
|
return this.client.decryptEventIfNeeded(event, {
|
2021-06-07 06:04:03 +00:00
|
|
|
isRetry: true,
|
|
|
|
emit: false,
|
2021-06-04 09:38:17 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
|
2021-06-08 13:07:14 +00:00
|
|
|
// Wait for all the events to get decrypted.
|
2021-06-04 09:38:17 +00:00
|
|
|
await Promise.all(decryptionPromises);
|
|
|
|
|
2021-06-07 09:17:27 +00:00
|
|
|
for (let i = 0; i < events.length; i++) this.setEventMetadata(events[i]);
|
2021-06-04 09:38:17 +00:00
|
|
|
|
2021-06-07 06:04:03 +00:00
|
|
|
return events;
|
2021-06-04 09:38:17 +00:00
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected async getMediaBlob(event: MatrixEvent): Promise<Blob> {
|
2021-06-14 12:36:40 +00:00
|
|
|
let blob: Blob;
|
|
|
|
try {
|
|
|
|
const isEncrypted = event.isEncrypted();
|
2021-10-01 09:22:37 +00:00
|
|
|
const content: IMediaEventContent = event.getContent();
|
|
|
|
const shouldDecrypt = isEncrypted && content.hasOwnProperty("file") && event.getType() !== "m.sticker";
|
2021-06-14 12:36:40 +00:00
|
|
|
if (shouldDecrypt) {
|
|
|
|
blob = await decryptFile(content.file);
|
|
|
|
} else {
|
2021-10-01 09:22:37 +00:00
|
|
|
const media = mediaFromContent(content);
|
2021-06-14 12:36:40 +00:00
|
|
|
const image = await fetch(media.srcHttp);
|
|
|
|
blob = await image.blob();
|
|
|
|
}
|
|
|
|
} catch (err) {
|
2021-10-15 14:26:54 +00:00
|
|
|
logger.log("Error decrypting media");
|
2021-06-14 12:36:40 +00:00
|
|
|
}
|
|
|
|
return blob;
|
|
|
|
}
|
|
|
|
|
2021-08-03 09:06:21 +00:00
|
|
|
public splitFileName(file: string): string[] {
|
2022-12-12 11:24:14 +00:00
|
|
|
const lastDot = file.lastIndexOf(".");
|
2021-06-14 12:36:40 +00:00
|
|
|
if (lastDot === -1) return [file, ""];
|
|
|
|
const fileName = file.slice(0, lastDot);
|
|
|
|
const ext = file.slice(lastDot + 1);
|
2022-12-12 11:24:14 +00:00
|
|
|
return [fileName, "." + ext];
|
2021-06-14 12:36:40 +00:00
|
|
|
}
|
|
|
|
|
2021-08-03 09:06:21 +00:00
|
|
|
public getFilePath(event: MatrixEvent): string {
|
2021-06-14 12:36:40 +00:00
|
|
|
const mediaType = event.getContent().msgtype;
|
|
|
|
let fileDirectory: string;
|
|
|
|
switch (mediaType) {
|
|
|
|
case "m.image":
|
|
|
|
fileDirectory = "images";
|
|
|
|
break;
|
|
|
|
case "m.video":
|
|
|
|
fileDirectory = "videos";
|
|
|
|
break;
|
|
|
|
case "m.audio":
|
|
|
|
fileDirectory = "audio";
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
fileDirectory = event.getType() === "m.sticker" ? "stickers" : "files";
|
|
|
|
}
|
|
|
|
const fileDate = formatFullDateNoDay(new Date(event.getTs()));
|
2021-06-25 09:49:17 +00:00
|
|
|
let [fileName, fileExt] = this.splitFileName(event.getContent().body);
|
2021-10-01 09:22:37 +00:00
|
|
|
|
2021-06-25 09:49:17 +00:00
|
|
|
if (event.getType() === "m.sticker") fileExt = ".png";
|
2021-10-01 09:22:37 +00:00
|
|
|
if (isVoiceMessage(event)) fileExt = ".ogg";
|
|
|
|
|
2022-12-12 11:24:14 +00:00
|
|
|
return fileDirectory + "/" + fileName + "-" + fileDate + fileExt;
|
2021-06-14 12:36:40 +00:00
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected isReply(event: MatrixEvent): boolean {
|
2021-06-22 07:20:15 +00:00
|
|
|
const isEncrypted = event.isEncrypted();
|
|
|
|
// If encrypted, in_reply_to lies in event.event.content
|
2023-02-15 13:36:22 +00:00
|
|
|
const content = isEncrypted ? event.event.content! : event.getContent();
|
2021-06-22 07:20:15 +00:00
|
|
|
const relatesTo = content["m.relates_to"];
|
2021-06-22 05:19:14 +00:00
|
|
|
return !!(relatesTo && relatesTo["m.in_reply_to"]);
|
|
|
|
}
|
|
|
|
|
2021-06-29 06:10:26 +00:00
|
|
|
protected isAttachment(mxEv: MatrixEvent): boolean {
|
2021-06-14 12:36:40 +00:00
|
|
|
const attachmentTypes = ["m.sticker", "m.image", "m.file", "m.video", "m.audio"];
|
2023-02-15 13:36:22 +00:00
|
|
|
return mxEv.getType() === attachmentTypes[0] || attachmentTypes.includes(mxEv.getContent().msgtype!);
|
2021-06-14 12:36:40 +00:00
|
|
|
}
|
|
|
|
|
2022-12-16 12:29:59 +00:00
|
|
|
public abstract export(): Promise<void>;
|
2021-05-24 15:18:13 +00:00
|
|
|
}
|