PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

refactor to share downloading code across all formats

Browse source
This commit is contained in:
Jaiwanth 2021-06-25 11:16:59 +05:30
parent 91b8b2ac5a
commit 3718826e94
7 changed files with 83 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/components/views/rooms/RoomHeader.js
View file

@ -84,11 +84,11 @@ export default class RoomHeader extends React.Component {
_exportConversationalHistory = async () => {
await exportConversationalHistory(
this.props.room,
exportFormats.JSON,
exportFormats.PLAIN_TEXT,
exportTypes.START_DATE,
{
startDate: parseInt(new Date("2021.05.20").getTime().toFixed(0)),
attachmentsIncluded: false,
attachmentsIncluded: true,
maxSize: 7 * 1024 * 1024, // 7 MB
},
);

67
src/utils/exportUtils/Exporter.ts
View file

@ -1,3 +1,4 @@
import streamSaver from "streamsaver";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixClientPeg } from "../../MatrixClientPeg";
@ -6,6 +7,9 @@ import { decryptFile } from "../DecryptFile";
import { mediaFromContent } from "../../customisations/Media";
import { formatFullDateNoDay } from "../../DateUtils";
import { MatrixClient } from "matrix-js-sdk";
import streamToZIP from "./StreamToZip";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import "web-streams-polyfill/ponyfill"; // to support streams API for older browsers
type FileStream = {
name: string,
@ -15,6 +19,9 @@ type FileStream = {
export default abstract class Exporter {
protected files: FileStream[];
protected client: MatrixClient;
protected writer: WritableStreamDefaultWriter<any>;
protected fileStream: WritableStream<any>;
protected constructor(
protected room: Room,
protected exportType: exportTypes,
@ -22,9 +29,16 @@ export default abstract class Exporter {
) {
this.files = [];
this.client = MatrixClientPeg.get();
window.addEventListener("beforeunload", this.onBeforeUnload);
window.addEventListener("onunload", this.abortExport);
}
protected addFile = (filePath: string, blob: Blob) => {
protected onBeforeUnload(e: BeforeUnloadEvent) {
e.preventDefault();
return e.returnValue = "Are you sure you want to exit during this export?";
}
protected addFile(filePath: string, blob: Blob) {
const file = {
name: filePath,
stream: () => blob.stream(),
@ -32,16 +46,53 @@ export default abstract class Exporter {
this.files.push(file);
}
protected pumpToFileStream = async (reader: ReadableStreamDefaultReader, writer: WritableStreamDefaultWriter) => {
protected async downloadZIP() {
const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`;
//Support for firefox browser
streamSaver.WritableStream = ponyfill.WritableStream
//Create a writable stream to the directory
this.fileStream = streamSaver.createWriteStream(filename);
this.writer = this.fileStream.getWriter();
const files = this.files;
console.info("Generating a ZIP...");
const readableZipStream = streamToZIP({
start(ctrl) {
for (const file of files) ctrl.enqueue(file);
ctrl.close();
},
});
console.info("Writing to the file system...")
const reader = readableZipStream.getReader()
await this.pumpToFileStream(reader);
}
protected async downloadPlainText(fileName: string, text: string): Promise<any> {
this.fileStream = streamSaver.createWriteStream(fileName);
this.writer = this.fileStream.getWriter()
const data = new TextEncoder().encode(text);
await this.writer.write(data);
await this.writer.close();
}
protected async abortExport(): Promise<void> {
if (this.fileStream) await this.fileStream.abort();
if (this.writer) await this.writer.abort();
}
protected async pumpToFileStream(reader: ReadableStreamDefaultReader) {
const res = await reader.read();
if (res.done) await writer.close();
if (res.done) await this.writer.close();
else {
await writer.write(res.value);
await this.pumpToFileStream(reader, writer)
await this.writer.write(res.value);
await this.pumpToFileStream(reader);
}
}
protected setEventMetadata = (event: MatrixEvent) => {
protected setEventMetadata(event: MatrixEvent) {
const roomState = this.client.getRoom(this.room.roomId).currentState;
event.sender = roomState.getSentinelMember(
event.getSender(),
@ -54,7 +105,7 @@ export default abstract class Exporter {
return event;
}
protected getLimit = () => {
protected getLimit() {
let limit: number;
switch (this.exportType) {
case exportTypes.LAST_N_MESSAGES:
@ -69,7 +120,7 @@ export default abstract class Exporter {
return limit;
}
protected getRequiredEvents = async () : Promise<MatrixEvent[]> => {
protected async getRequiredEvents():Promise<MatrixEvent[]> {
const eventMapper = this.client.getEventMapper();
let prevToken: string|null = null;

37
src/utils/exportUtils/HtmlExport.tsx
View file

@ -1,5 +1,4 @@
import React from "react"
import streamSaver from "streamsaver";
import Exporter from "./Exporter";
import { mediaFromMxc } from "../../customisations/Media";
import { Room } from "matrix-js-sdk/src/models/room";
@ -7,11 +6,10 @@ import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { renderToStaticMarkup } from 'react-dom/server'
import { Layout } from "../../settings/Layout";
import { shouldFormContinuation } from "../../components/structures/MessagePanel";
import { formatFullDateNoDay, formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils";
import { formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils";
import { RoomPermalinkCreator } from "../permalinks/Permalinks";
import { _t } from "../../languageHandler";
import { EventType } from "matrix-js-sdk/src/@types/event";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import * as Avatar from "../../Avatar";
import EventTile, { haveTileForEvent } from "../../components/views/rooms/EventTile";
import DateSeparator from "../../components/views/messages/DateSeparator";
@ -22,7 +20,6 @@ import exportIcons from "./exportIcons";
import { exportTypes } from "./exportUtils";
import { exportOptions } from "./exportUtils";
import MatrixClientContext from "../../contexts/MatrixClientContext";
import zip from "./StreamToZip";
export default class HTMLExporter extends Exporter {
protected avatars: Map<string, boolean>;
@ -38,12 +35,6 @@ export default class HTMLExporter extends Exporter {
this.mediaOmitText = !this.exportOptions.attachmentsIncluded
? _t("Media omitted")
: _t("Media omitted - file size limit exceeded");
window.addEventListener("beforeunload", this.onBeforeUnload)
}
protected onBeforeUnload = (e: BeforeUnloadEvent) => {
e.preventDefault();
return e.returnValue = "Are you sure you want to exit during this export?";
}
protected async getRoomAvatar() {
@ -267,7 +258,7 @@ export default class HTMLExporter extends Exporter {
return eventTileMarkup;
}
protected createModifiedEvent = (text: string, mxEv: MatrixEvent) => {
protected createModifiedEvent(text: string, mxEv: MatrixEvent) {
const modifiedContent = {
msgtype: "m.text",
body: `*${text}*`,
@ -351,34 +342,14 @@ export default class HTMLExporter extends Exporter {
this.addFile(`icons/${iconName}`, new Blob([exportIcons[iconName]]));
}
const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`;
console.info("HTML creation successful!");
//Support for firefox browser
streamSaver.WritableStream = ponyfill.WritableStream
//Create a writable stream to the directory
const fileStream = streamSaver.createWriteStream(filename);
const writer = fileStream.getWriter();
const files = this.files;
console.info("Generating a ZIP...");
const readableZipStream = zip({
start(ctrl) {
for (const file of files) ctrl.enqueue(file);
ctrl.close();
},
});
console.info("Writing to file system...")
const reader = readableZipStream.getReader()
await this.pumpToFileStream(reader, writer);
await this.downloadZIP();
const exportEnd = performance.now();
console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
window.removeEventListener("beforeunload", this.onBeforeUnload);
window.removeEventListener("onunload", this.abortExport);
}
}

43
src/utils/exportUtils/JSONExport.ts
View file

@ -1,13 +1,10 @@
import streamSaver from "streamsaver";
import Exporter from "./Exporter";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { formatFullDateNoDay, formatFullDateNoDayNoTime } from "../../DateUtils";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import { haveTileForEvent } from "../../components/views/rooms/EventTile";
import { exportTypes } from "./exportUtils";
import { exportOptions } from "./exportUtils";
import zip from "./StreamToZip";
import { EventType } from "matrix-js-sdk/src/@types/event";
@ -17,7 +14,6 @@ export default class JSONExporter extends Exporter {
constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) {
super(room, exportType, exportOptions);
this.totalSize = 0;
window.addEventListener("beforeunload", this.onBeforeUnload)
}
protected wrapJSON(json: string): string {
@ -39,15 +35,10 @@ ${json}
}`
}
protected indentEachLine(string: string) {
protected indentEachLine(JSONString: string, spaces: number) {
const indent = ' ';
const regex = /^(?!\s*$)/gm;
return string.replace(regex, indent.repeat(2));
}
protected onBeforeUnload = (e: BeforeUnloadEvent) => {
e.preventDefault();
return e.returnValue = "Are you sure you want to exit during this export?";
return JSONString.replace(regex, indent.repeat(spaces));
}
protected async getJSONString(mxEv: MatrixEvent) {
@ -76,7 +67,7 @@ ${json}
if (!haveTileForEvent(event)) continue;
content += await this.getJSONString(event);
}
return this.wrapJSON(this.indentEachLine(content.slice(0, -2)));
return this.wrapJSON(this.indentEachLine(content.slice(0, -2), 2));
}
public async export() {
@ -91,34 +82,18 @@ ${json}
const text = await this.createOutput(res);
console.info("Writing to the file system...");
streamSaver.WritableStream = ponyfill.WritableStream
const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`;
const files = this.files;
if (files.length) {
this.addFile("result.json", new Blob([text]));
const fileStream = streamSaver.createWriteStream(fileName.slice(0, -5) + ".zip");
const readableZipStream = zip({
start(ctrl) {
for (const file of files) ctrl.enqueue(file);
ctrl.close();
},
});
const writer = fileStream.getWriter()
const reader = readableZipStream.getReader()
await this.pumpToFileStream(reader, writer);
if (this.files.length) {
this.addFile("export.json", new Blob([text]));
await this.downloadZIP();
} else {
const fileStream = streamSaver.createWriteStream(fileName);
const writer = fileStream.getWriter()
const data = new TextEncoder().encode(text);
await writer.write(data);
await writer.close();
const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`;
await this.downloadPlainText(fileName, text);
}
const exportEnd = performance.now();
console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
window.removeEventListener("beforeunload", this.onBeforeUnload);
window.removeEventListener("onunload", this.abortExport);
}
}

44
src/utils/exportUtils/PlainTextExport.ts
View file

@ -1,35 +1,24 @@
import streamSaver from "streamsaver";
import Exporter from "./Exporter";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { formatFullDateNoDay } from "../../DateUtils";
import { _t } from "../../languageHandler";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import { haveTileForEvent } from "../../components/views/rooms/EventTile";
import { exportTypes } from "./exportUtils";
import { exportOptions } from "./exportUtils";
import { textForEvent } from "../../TextForEvent";
import zip from "./StreamToZip";
export default class PlainTextExporter extends Exporter {
protected totalSize: number;
protected mediaOmitText: string;
private readonly fileDir: string;
constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) {
super(room, exportType, exportOptions);
this.totalSize = 0;
this.fileDir = `matrix-export-${formatFullDateNoDay(new Date())}`;
this.mediaOmitText = !this.exportOptions.attachmentsIncluded
? _t("Media omitted")
: _t("Media omitted - file size limit exceeded");
window.addEventListener("beforeunload", this.onBeforeUnload)
}
protected onBeforeUnload = (e: BeforeUnloadEvent) => {
e.preventDefault();
return e.returnValue = "Are you sure you want to exit during this export?";
}
protected textForReplyEvent = (ev : MatrixEvent) => {
@ -90,12 +79,6 @@ export default class PlainTextExporter extends Exporter {
return content;
}
protected getFileName = () => {
if (this.exportOptions.attachmentsIncluded) {
return `${this.room.name}.txt`;
} else return `${this.fileDir}.txt`
}
public async export() {
console.info("Starting export process...");
console.info("Fetching events...");
@ -108,32 +91,17 @@ export default class PlainTextExporter extends Exporter {
const text = await this.createOutput(res);
console.info("Writing to the file system...");
streamSaver.WritableStream = ponyfill.WritableStream
const files = this.files;
if (files.length) {
this.addFile(this.getFileName(), new Blob([text]));
const fileStream = streamSaver.createWriteStream(`${this.fileDir}.zip`);
const readableZipStream = zip({
start(ctrl) {
for (const file of files) ctrl.enqueue(file);
ctrl.close();
},
});
const writer = fileStream.getWriter()
const reader = readableZipStream.getReader()
await this.pumpToFileStream(reader, writer);
if (this.files.length) {
this.addFile("export.txt", new Blob([text]));
await this.downloadZIP();
} else {
const fileStream = streamSaver.createWriteStream(`${this.fileDir}.txt`);
const writer = fileStream.getWriter()
const data = new TextEncoder().encode(text);
await writer.write(data);
await writer.close();
const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.txt`;
await this.downloadPlainText(fileName, text);
}
const exportEnd = performance.now();
console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
window.removeEventListener("onunload", this.abortExport);
window.removeEventListener("beforeunload", this.onBeforeUnload);
}
}

2
src/utils/exportUtils/StreamToZip.ts
View file

@ -101,7 +101,7 @@ const pump = (zipObj: ZipObj) => zipObj.reader ? zipObj.reader.read().then(chunk
}
}) : undefined;
export default function ZIP(underlyingSource: UnderlyingSource) {
export default function streamToZIP(underlyingSource: UnderlyingSource) {
const files = Object.create(null);
const filenames: string[] = [];
const encoder = new TextEncoder();

4
src/utils/exportUtils/exportUtils.ts
View file

@ -6,7 +6,7 @@ import PlainTextExporter from "./PlainTextExport";
export enum exportFormats {
HTML = "HTML",
JSON = "JSON",
LOGS = "LOGS",
PLAIN_TEXT = "PLAIN_TEXT",
}
export enum exportTypes {
@ -36,7 +36,7 @@ const exportConversationalHistory = async (
case exportFormats.JSON:
await new JSONExporter(room, exportType, exportOptions).export();
break;
case exportFormats.LOGS:
case exportFormats.PLAIN_TEXT:
await new PlainTextExporter(room, exportType, exportOptions).export();
break;
}