refactor to share downloading code across all formats

src/components/views/rooms/RoomHeader.js

@@ -84,11 +84,11 @@ export default class RoomHeader extends React.Component {
     _exportConversationalHistory = async () => {
         await exportConversationalHistory(
             this.props.room,
-            exportFormats.JSON,
+            exportFormats.PLAIN_TEXT,
             exportTypes.START_DATE,
             {
                 startDate: parseInt(new Date("2021.05.20").getTime().toFixed(0)),
-                attachmentsIncluded: false,
+                attachmentsIncluded: true,
                 maxSize: 7 * 1024 * 1024, // 7 MB
             },
         );

src/utils/exportUtils/Exporter.ts

@@ -1,3 +1,4 @@
+import streamSaver from "streamsaver";
 import { MatrixEvent } from "matrix-js-sdk/src/models/event";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { MatrixClientPeg } from "../../MatrixClientPeg";
@@ -6,6 +7,9 @@ import { decryptFile } from "../DecryptFile";
 import { mediaFromContent } from "../../customisations/Media";
 import { formatFullDateNoDay } from "../../DateUtils";
 import { MatrixClient } from "matrix-js-sdk";
+import streamToZIP from "./StreamToZip";
+import * as ponyfill from "web-streams-polyfill/ponyfill"
+import "web-streams-polyfill/ponyfill"; // to support streams API for older browsers
 
 type FileStream = {
     name: string,
@@ -15,6 +19,9 @@ type FileStream = {
 export default abstract class Exporter {
     protected files: FileStream[];
     protected client: MatrixClient;
+    protected writer: WritableStreamDefaultWriter<any>;
+    protected fileStream: WritableStream<any>;
+
     protected constructor(
         protected room: Room,
         protected exportType: exportTypes,
@@ -22,9 +29,16 @@ export default abstract class Exporter {
     ) {
         this.files = [];
         this.client = MatrixClientPeg.get();
+        window.addEventListener("beforeunload", this.onBeforeUnload);
+        window.addEventListener("onunload", this.abortExport);
     }
 
-    protected addFile = (filePath: string, blob: Blob) => {
+    protected onBeforeUnload(e: BeforeUnloadEvent) {
+        e.preventDefault();
+        return e.returnValue = "Are you sure you want to exit during this export?";
+    }
+
+    protected addFile(filePath: string, blob: Blob) {
         const file = {
             name: filePath,
             stream: () => blob.stream(),
@@ -32,16 +46,53 @@ export default abstract class Exporter {
         this.files.push(file);
     }
 
-    protected pumpToFileStream = async (reader: ReadableStreamDefaultReader, writer: WritableStreamDefaultWriter) => {
+    protected async downloadZIP() {
+        const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`;
+        //Support for firefox browser
+        streamSaver.WritableStream = ponyfill.WritableStream
+        //Create a writable stream to the directory
+        this.fileStream = streamSaver.createWriteStream(filename);
+
+        this.writer = this.fileStream.getWriter();
+        const files = this.files;
+
+        console.info("Generating a ZIP...");
+        const readableZipStream = streamToZIP({
+            start(ctrl) {
+                for (const file of files) ctrl.enqueue(file);
+                ctrl.close();
+            },
+        });
+
+        console.info("Writing to the file system...")
+
+        const reader = readableZipStream.getReader()
+        await this.pumpToFileStream(reader);
+    }
+
+    protected async downloadPlainText(fileName: string, text: string): Promise<any> {
+        this.fileStream = streamSaver.createWriteStream(fileName);
+        this.writer = this.fileStream.getWriter()
+        const data = new TextEncoder().encode(text);
+        await this.writer.write(data);
+        await this.writer.close();
+    }
+
+    protected async abortExport(): Promise<void> {
+        if (this.fileStream) await this.fileStream.abort();
+        if (this.writer) await this.writer.abort();
+    }
+
+    protected async pumpToFileStream(reader: ReadableStreamDefaultReader) {
         const res = await reader.read();
-        if (res.done) await writer.close();
+        if (res.done) await this.writer.close();
         else {
-            await writer.write(res.value);
+            await this.writer.write(res.value);
-            await this.pumpToFileStream(reader, writer)
+            await this.pumpToFileStream(reader);
         }
     }
 
-    protected setEventMetadata = (event: MatrixEvent) => {
+    protected setEventMetadata(event: MatrixEvent) {
         const roomState = this.client.getRoom(this.room.roomId).currentState;
         event.sender = roomState.getSentinelMember(
             event.getSender(),
@@ -54,7 +105,7 @@ export default abstract class Exporter {
         return event;
     }
 
-    protected getLimit = () => {
+    protected getLimit() {
         let limit: number;
         switch (this.exportType) {
             case exportTypes.LAST_N_MESSAGES:
@@ -69,7 +120,7 @@ export default abstract class Exporter {
         return limit;
     }
 
-    protected getRequiredEvents = async () : Promise<MatrixEvent[]> => {
+    protected async getRequiredEvents():Promise<MatrixEvent[]> {
         const eventMapper = this.client.getEventMapper();
 
         let prevToken: string|null = null;

src/utils/exportUtils/HtmlExport.tsx

@@ -1,5 +1,4 @@
 import React from "react"
-import streamSaver from "streamsaver";
 import Exporter from "./Exporter";
 import { mediaFromMxc } from "../../customisations/Media";
 import { Room } from "matrix-js-sdk/src/models/room";
@@ -7,11 +6,10 @@ import { MatrixEvent } from "matrix-js-sdk/src/models/event";
 import { renderToStaticMarkup } from 'react-dom/server'
 import { Layout } from "../../settings/Layout";
 import { shouldFormContinuation } from "../../components/structures/MessagePanel";
-import { formatFullDateNoDay, formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils";
+import { formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils";
 import { RoomPermalinkCreator } from "../permalinks/Permalinks";
 import { _t } from "../../languageHandler";
 import { EventType } from "matrix-js-sdk/src/@types/event";
-import * as ponyfill from "web-streams-polyfill/ponyfill"
 import * as Avatar from "../../Avatar";
 import EventTile, { haveTileForEvent } from "../../components/views/rooms/EventTile";
 import DateSeparator from "../../components/views/messages/DateSeparator";
@@ -22,7 +20,6 @@ import exportIcons from "./exportIcons";
 import { exportTypes } from "./exportUtils";
 import { exportOptions } from "./exportUtils";
 import MatrixClientContext from "../../contexts/MatrixClientContext";
-import zip from "./StreamToZip";
 
 export default class HTMLExporter extends Exporter {
     protected avatars: Map<string, boolean>;
@@ -38,12 +35,6 @@ export default class HTMLExporter extends Exporter {
         this.mediaOmitText = !this.exportOptions.attachmentsIncluded
             ? _t("Media omitted")
             : _t("Media omitted - file size limit exceeded");
-        window.addEventListener("beforeunload", this.onBeforeUnload)
-    }
-
-    protected onBeforeUnload = (e: BeforeUnloadEvent) => {
-        e.preventDefault();
-        return e.returnValue = "Are you sure you want to exit during this export?";
     }
 
     protected async getRoomAvatar() {
@@ -267,7 +258,7 @@ export default class HTMLExporter extends Exporter {
         return eventTileMarkup;
     }
 
-    protected createModifiedEvent = (text: string, mxEv: MatrixEvent) => {
+    protected createModifiedEvent(text: string, mxEv: MatrixEvent) {
         const modifiedContent = {
             msgtype: "m.text",
             body: `*${text}*`,
@@ -351,34 +342,14 @@ export default class HTMLExporter extends Exporter {
             this.addFile(`icons/${iconName}`, new Blob([exportIcons[iconName]]));
         }
 
-        const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`;
-
         console.info("HTML creation successful!");
 
-        //Support for firefox browser
+        await this.downloadZIP();
-        streamSaver.WritableStream = ponyfill.WritableStream
-        //Create a writable stream to the directory
-        const fileStream = streamSaver.createWriteStream(filename);
-
-        const writer = fileStream.getWriter();
-        const files = this.files;
-
-        console.info("Generating a ZIP...");
-        const readableZipStream = zip({
-            start(ctrl) {
-                for (const file of files) ctrl.enqueue(file);
-                ctrl.close();
-            },
-        });
-
-        console.info("Writing to file system...")
-
-        const reader = readableZipStream.getReader()
-        await this.pumpToFileStream(reader, writer);
 
         const exportEnd = performance.now();
         console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
         window.removeEventListener("beforeunload", this.onBeforeUnload);
+        window.removeEventListener("onunload", this.abortExport);
     }
 }
 

src/utils/exportUtils/JSONExport.ts

@@ -1,13 +1,10 @@
-import streamSaver from "streamsaver";
 import Exporter from "./Exporter";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { MatrixEvent } from "matrix-js-sdk/src/models/event";
 import { formatFullDateNoDay, formatFullDateNoDayNoTime } from "../../DateUtils";
-import * as ponyfill from "web-streams-polyfill/ponyfill"
 import { haveTileForEvent } from "../../components/views/rooms/EventTile";
 import { exportTypes } from "./exportUtils";
 import { exportOptions } from "./exportUtils";
-import zip from "./StreamToZip";
 import { EventType } from "matrix-js-sdk/src/@types/event";
 
 
@@ -17,7 +14,6 @@ export default class JSONExporter extends Exporter {
     constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) {
         super(room, exportType, exportOptions);
         this.totalSize = 0;
-        window.addEventListener("beforeunload", this.onBeforeUnload)
     }
 
     protected wrapJSON(json: string): string {
@@ -39,15 +35,10 @@ ${json}
 }`
     }
 
-    protected indentEachLine(string: string) {
+    protected indentEachLine(JSONString: string, spaces: number) {
         const indent = ' ';
         const regex = /^(?!\s*$)/gm;
-        return string.replace(regex, indent.repeat(2));
+        return JSONString.replace(regex, indent.repeat(spaces));
-    }
-
-    protected onBeforeUnload = (e: BeforeUnloadEvent) => {
-        e.preventDefault();
-        return e.returnValue = "Are you sure you want to exit during this export?";
     }
 
     protected async getJSONString(mxEv: MatrixEvent) {
@@ -76,7 +67,7 @@ ${json}
             if (!haveTileForEvent(event)) continue;
             content += await this.getJSONString(event);
         }
-        return this.wrapJSON(this.indentEachLine(content.slice(0, -2)));
+        return this.wrapJSON(this.indentEachLine(content.slice(0, -2), 2));
     }
 
     public async export() {
@@ -91,34 +82,18 @@ ${json}
 
         const text = await this.createOutput(res);
 
-        console.info("Writing to the file system...");
+        if (this.files.length) {
-        streamSaver.WritableStream = ponyfill.WritableStream
+            this.addFile("export.json", new Blob([text]));
-
+            await this.downloadZIP();
-        const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`;
-        const files = this.files;
-        if (files.length) {
-            this.addFile("result.json", new Blob([text]));
-            const fileStream = streamSaver.createWriteStream(fileName.slice(0, -5) + ".zip");
-            const readableZipStream = zip({
-                start(ctrl) {
-                    for (const file of files) ctrl.enqueue(file);
-                    ctrl.close();
-                },
-            });
-            const writer = fileStream.getWriter()
-            const reader = readableZipStream.getReader()
-            await this.pumpToFileStream(reader, writer);
         } else {
-            const fileStream = streamSaver.createWriteStream(fileName);
+            const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`;
-            const writer = fileStream.getWriter()
+            await this.downloadPlainText(fileName, text);
-            const data = new TextEncoder().encode(text);
-            await writer.write(data);
-            await writer.close();
         }
 
         const exportEnd = performance.now();
         console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
         window.removeEventListener("beforeunload", this.onBeforeUnload);
+        window.removeEventListener("onunload", this.abortExport);
     }
 }
 

src/utils/exportUtils/PlainTextExport.ts

@@ -1,35 +1,24 @@
-import streamSaver from "streamsaver";
 import Exporter from "./Exporter";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { MatrixEvent } from "matrix-js-sdk/src/models/event";
 import { formatFullDateNoDay } from "../../DateUtils";
 import { _t } from "../../languageHandler";
-import * as ponyfill from "web-streams-polyfill/ponyfill"
 import { haveTileForEvent } from "../../components/views/rooms/EventTile";
 import { exportTypes } from "./exportUtils";
 import { exportOptions } from "./exportUtils";
 import { textForEvent } from "../../TextForEvent";
-import zip from "./StreamToZip";
 
 
 export default class PlainTextExporter extends Exporter {
     protected totalSize: number;
     protected mediaOmitText: string;
-    private readonly fileDir: string;
 
     constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) {
         super(room, exportType, exportOptions);
         this.totalSize = 0;
-        this.fileDir = `matrix-export-${formatFullDateNoDay(new Date())}`;
         this.mediaOmitText = !this.exportOptions.attachmentsIncluded
             ? _t("Media omitted")
             : _t("Media omitted - file size limit exceeded");
-        window.addEventListener("beforeunload", this.onBeforeUnload)
-    }
-
-    protected onBeforeUnload = (e: BeforeUnloadEvent) => {
-        e.preventDefault();
-        return e.returnValue = "Are you sure you want to exit during this export?";
     }
 
     protected textForReplyEvent = (ev : MatrixEvent) => {
@@ -90,12 +79,6 @@ export default class PlainTextExporter extends Exporter {
         return content;
     }
 
-    protected getFileName = () => {
-        if (this.exportOptions.attachmentsIncluded) {
-            return `${this.room.name}.txt`;
-        } else return `${this.fileDir}.txt`
-    }
-
     public async export() {
         console.info("Starting export process...");
         console.info("Fetching events...");
@@ -108,32 +91,17 @@ export default class PlainTextExporter extends Exporter {
 
         const text = await this.createOutput(res);
 
-        console.info("Writing to the file system...");
+        if (this.files.length) {
-        streamSaver.WritableStream = ponyfill.WritableStream
+            this.addFile("export.txt", new Blob([text]));
-
+            await this.downloadZIP();
-        const files = this.files;
-        if (files.length) {
-            this.addFile(this.getFileName(), new Blob([text]));
-            const fileStream = streamSaver.createWriteStream(`${this.fileDir}.zip`);
-            const readableZipStream = zip({
-                start(ctrl) {
-                    for (const file of files) ctrl.enqueue(file);
-                    ctrl.close();
-                },
-            });
-            const writer = fileStream.getWriter()
-            const reader = readableZipStream.getReader()
-            await this.pumpToFileStream(reader, writer);
         } else {
-            const fileStream = streamSaver.createWriteStream(`${this.fileDir}.txt`);
+            const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.txt`;
-            const writer = fileStream.getWriter()
+            await this.downloadPlainText(fileName, text);
-            const data = new TextEncoder().encode(text);
-            await writer.write(data);
-            await writer.close();
         }
 
         const exportEnd = performance.now();
         console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
+        window.removeEventListener("onunload", this.abortExport);
         window.removeEventListener("beforeunload", this.onBeforeUnload);
     }
 }

src/utils/exportUtils/StreamToZip.ts

@@ -101,7 +101,7 @@ const pump = (zipObj: ZipObj) => zipObj.reader ? zipObj.reader.read().then(chunk
     }
 }) : undefined;
 
-export default function ZIP(underlyingSource: UnderlyingSource) {
+export default function streamToZIP(underlyingSource: UnderlyingSource) {
     const files = Object.create(null);
     const filenames: string[] = [];
     const encoder = new TextEncoder();

src/utils/exportUtils/exportUtils.ts

@@ -6,7 +6,7 @@ import PlainTextExporter from "./PlainTextExport";
 export enum exportFormats {
     HTML = "HTML",
     JSON = "JSON",
-    LOGS = "LOGS",
+    PLAIN_TEXT = "PLAIN_TEXT",
 }
 
 export enum exportTypes {
@@ -36,7 +36,7 @@ const exportConversationalHistory = async (
         case exportFormats.JSON:
             await new JSONExporter(room, exportType, exportOptions).export();
             break;
-        case exportFormats.LOGS:
+        case exportFormats.PLAIN_TEXT:
             await new PlainTextExporter(room, exportType, exportOptions).export();
             break;
     }