From 41bc2b6481c85a476167fb7ccb2eaee1e1f8e950 Mon Sep 17 00:00:00 2001 From: Jaiwanth Date: Mon, 26 Jul 2021 22:15:05 +0530 Subject: [PATCH] Move away from streamsaver(for now) --- package.json | 4 +- src/utils/exportUtils/Exporter.ts | 63 ++----- src/utils/exportUtils/ZipStream.ts | 279 ----------------------------- yarn.lock | 49 +++-- 4 files changed, 51 insertions(+), 344 deletions(-) delete mode 100644 src/utils/exportUtils/ZipStream.ts diff --git a/package.json b/package.json index 15c310c2ce..30254ed78d 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "highlight.js": "^10.5.0", "html-entities": "^1.4.0", "is-ip": "^3.1.0", + "jszip": "^3.7.0", "katex": "^0.12.0", "linkifyjs": "^2.1.9", "lodash": "^4.17.20", @@ -99,10 +100,8 @@ "resize-observer-polyfill": "^1.5.1", "rfc4648": "^1.4.0", "sanitize-html": "^2.3.2", - "streamsaver": "^2.0.5", "tar-js": "^0.3.0", "url": "^0.11.0", - "web-streams-polyfill": "^3.0.3", "what-input": "^5.2.10", "zxcvbn": "^4.4.2" }, @@ -131,6 +130,7 @@ "@types/counterpart": "^0.18.1", "@types/css-font-loading-module": "^0.0.6", "@types/diff-match-patch": "^1.0.32", + "@types/file-saver": "^2.0.3", "@types/flux": "^3.1.9", "@types/jest": "^26.0.20", "@types/linkifyjs": "^2.1.3", diff --git a/src/utils/exportUtils/Exporter.ts b/src/utils/exportUtils/Exporter.ts index d43f937c7b..c840731d47 100644 --- a/src/utils/exportUtils/Exporter.ts +++ b/src/utils/exportUtils/Exporter.ts @@ -1,4 +1,3 @@ -import streamSaver from "streamsaver"; import { MatrixEvent } from "matrix-js-sdk/src/models/event"; import { Room } from "matrix-js-sdk/src/models/room"; import { MatrixClientPeg } from "../../MatrixClientPeg"; @@ -7,21 +6,18 @@ import { decryptFile } from "../DecryptFile"; import { mediaFromContent } from "../../customisations/Media"; import { formatFullDateNoDay } from "../../DateUtils"; import { Direction, MatrixClient } from "matrix-js-sdk"; -import streamToZIP from "./ZipStream"; -import * as ponyfill from "web-streams-polyfill/ponyfill"; -import "web-streams-polyfill/ponyfill"; // to support streams API for older browsers import { MutableRefObject } from "react"; +import JSZip from "jszip"; +import { saveAs } from "file-saver"; -type FileStream = { +type BlobFile = { name: string; - stream(): ReadableStream; + blob: Blob; }; export default abstract class Exporter { - protected files: FileStream[]; + protected files: BlobFile[]; protected client: MatrixClient; - protected writer: WritableStreamDefaultWriter; - protected fileStream: WritableStream; protected cancelled: boolean; protected constructor( @@ -34,7 +30,6 @@ export default abstract class Exporter { this.files = []; this.client = MatrixClientPeg.get(); window.addEventListener("beforeunload", this.onBeforeUnload); - window.addEventListener("onunload", this.abortWriter); } protected onBeforeUnload(e: BeforeUnloadEvent): string { @@ -50,7 +45,7 @@ export default abstract class Exporter { protected addFile(filePath: string, blob: Blob): void { const file = { name: filePath, - stream: () => blob.stream(), + blob, }; this.files.push(file); } @@ -58,67 +53,31 @@ export default abstract class Exporter { protected async downloadZIP(): Promise { const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`; - // Support for older browsers - streamSaver.WritableStream = ponyfill.WritableStream; - + const zip = new JSZip(); // Create a writable stream to the directory - this.fileStream = streamSaver.createWriteStream(filename); - if (!this.cancelled) this.updateProgress("Generating a ZIP"); else return this.cleanUp(); - this.writer = this.fileStream.getWriter(); - const files = this.files; + for (const file of this.files) zip.file(file.name, file.blob); - const readableZipStream = streamToZIP({ - start(ctrl) { - for (const file of files) ctrl.enqueue(file); - ctrl.close(); - }, - }); + const content = await zip.generateAsync({ type: "blob" }); - if (this.cancelled) return this.cleanUp(); - - this.updateProgress("Writing to the file system"); - - const reader = readableZipStream.getReader(); - await this.pumpToFileStream(reader); + await saveAs(content, filename); } protected cleanUp(): string { console.log("Cleaning up..."); window.removeEventListener("beforeunload", this.onBeforeUnload); - window.removeEventListener("onunload", this.abortWriter); return ""; } public async cancelExport(): Promise { console.log("Cancelling export..."); this.cancelled = true; - await this.abortWriter(); } protected async downloadPlainText(fileName: string, text: string): Promise { - this.fileStream = streamSaver.createWriteStream(fileName); - this.writer = this.fileStream.getWriter(); - const data = new TextEncoder().encode(text); - if (this.cancelled) return this.cleanUp(); - await this.writer.write(data); - await this.writer.close(); - } - - protected async abortWriter(): Promise { - await this.fileStream?.abort(); - await this.writer?.abort(); - } - - protected async pumpToFileStream(reader: ReadableStreamDefaultReader): Promise { - const res = await reader.read(); - if (res.done) await this.writer.close(); - else { - await this.writer.write(res.value); - await this.pumpToFileStream(reader); - } + await saveAs(new Blob[text], fileName); } protected setEventMetadata(event: MatrixEvent): MatrixEvent { diff --git a/src/utils/exportUtils/ZipStream.ts b/src/utils/exportUtils/ZipStream.ts deleted file mode 100644 index 2ee355f09c..0000000000 --- a/src/utils/exportUtils/ZipStream.ts +++ /dev/null @@ -1,279 +0,0 @@ -// Based on https://github.com/jimmywarting/StreamSaver.js/blob/master/examples/zip-stream.js - -/* global ReadableStream */ - -type TypedArray = - | Int8Array - | Uint8Array - | Int16Array - | Uint16Array - | Int32Array - | Uint32Array - | Uint8ClampedArray - | Float32Array - | Float64Array; - -/** - * 32-bit cyclic redundancy check, or CRC-32 - checksum - */ -class Crc32 { - crc: number; - table: any; - constructor() { - this.crc = -1; - this.table = (() => { - let i; - let j; - let t; - const table = []; - - for (i = 0; i < 256; i++) { - t = i; - for (j = 0; j < 8; j++) { - t = (t & 1) - ? (t >>> 1) ^ 0xEDB88320 - : t >>> 1; - } - table[i] = t; - } - return table; - })(); - } - - append(data: TypedArray) { - let crc = this.crc | 0; - const table = this.table; - for (let offset = 0, len = data.length | 0; offset < len; offset++) { - crc = (crc >>> 8) ^ table[(crc ^ data[offset]) & 0xFF]; - } - this.crc = crc; - } - - get() { - return ~this.crc; - } -} - -type DataHelper = { - array: Uint8Array; - view: DataView; -}; - -const getDataHelper = (byteLength: number): DataHelper => { - const uint8 = new Uint8Array(byteLength); - return { - array: uint8, - view: new DataView(uint8.buffer), - }; -}; - -type FileLike = File & { - directory: string; - comment: string; - stream(): ReadableStream; -}; - -type ZipObj = { - crc?: Crc32; - uncompressedLength: number; - compressedLength: number; - ctrl: ReadableStreamDefaultController; - writeFooter: Function; - writeHeader: Function; - reader?: ReadableStreamDefaultReader; - offset: number; - header?: DataHelper; - fileLike: FileLike; - level: number; - directory: boolean; -}; - -const pump = (zipObj: ZipObj) => zipObj.reader ? zipObj.reader.read().then(chunk => { - if (zipObj.crc) { - if (chunk.done) return zipObj.writeFooter(); - const outputData = chunk.value; - zipObj.crc.append(outputData); - zipObj.uncompressedLength += outputData.length; - zipObj.compressedLength += outputData.length; - zipObj.ctrl.enqueue(outputData); - } else { - throw new Error('Missing zipObj.crc'); - } -}) : undefined; - -export default function streamToZIP(underlyingSource: UnderlyingSource) { - const files = Object.create(null); - const filenames: string[] = []; - const encoder = new TextEncoder(); - let offset = 0; - let activeZipIndex = 0; - let ctrl: ReadableStreamDefaultController; - let activeZipObject: ZipObj; - let closed: boolean; - - function next() { - activeZipIndex++; - activeZipObject = files[filenames[activeZipIndex]]; - if (activeZipObject) processNextChunk(); - else if (closed) closeZip(); - } - - const zipWriter: ReadableStreamDefaultController = { - desiredSize: null, - - error(err) { - console.error(err); - }, - - enqueue(fileLike: FileLike) { - if (closed) { - throw new TypeError( - "Cannot enqueue a chunk into a readable stream that is closed or has been requested to be closed", - ); - } - - let name = fileLike.name.trim(); - const date = new Date(typeof fileLike.lastModified === 'undefined' ? Date.now() : fileLike.lastModified); - - if (fileLike.directory && !name.endsWith('/')) name += '/'; - // if file already exists, do not enqueue - if (files[name]) return; - - const nameBuf = encoder.encode(name); - filenames.push(name); - - const zipObject: ZipObj = files[name] = { - level: 0, - ctrl, - directory: !!fileLike.directory, - nameBuf, - comment: encoder.encode(fileLike.comment || ''), - compressedLength: 0, - uncompressedLength: 0, - offset, - - writeHeader() { - const header = getDataHelper(26); - const data = getDataHelper(30 + nameBuf.length); - - zipObject.offset = offset; - zipObject.header = header; - - if (zipObject.level !== 0 && !zipObject.directory) { - header.view.setUint16(4, 0x0800); - } - - header.view.setUint32(0, 0x14000808); - header.view.setUint16( - 6, - (((date.getHours() << 6) | date.getMinutes()) << 5) | (date.getSeconds() / 2), - true, - ); - header.view.setUint16( - 8, - ((((date.getFullYear() - 1980) << 4) | (date.getMonth() + 1)) << 5) | - date.getDate(), - true, - ); - header.view.setUint16(22, nameBuf.length, true); - data.view.setUint32(0, 0x504b0304); - data.array.set(header.array, 4); - data.array.set(nameBuf, 30); - offset += data.array.length; - ctrl.enqueue(data.array); - }, - - writeFooter() { - const footer = getDataHelper(16); - footer.view.setUint32(0, 0x504b0708); - - if (zipObject.crc && zipObject.header) { - zipObject.header.view.setUint32(10, zipObject.crc.get(), true); - zipObject.header.view.setUint32(14, zipObject.compressedLength, true); - zipObject.header.view.setUint32(18, zipObject.uncompressedLength, true); - footer.view.setUint32(4, zipObject.crc.get(), true); - footer.view.setUint32(8, zipObject.compressedLength, true); - footer.view.setUint32(12, zipObject.uncompressedLength, true); - } - - ctrl.enqueue(footer.array); - offset += zipObject.compressedLength + 16; - next(); - }, - fileLike, - }; - - if (!activeZipObject) { - activeZipObject = zipObject; - processNextChunk(); - } - }, - - close() { - if (closed) { - throw new TypeError( - "Cannot close a readable stream that has already been requested to be closed", - ); - } - if (!activeZipObject) closeZip(); - closed = true; - }, - }; - - function closeZip() { - let length = 0; - let index = 0; - let indexFilename: number; - let file: any; - - for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) { - file = files[filenames[indexFilename]]; - length += 46 + file.nameBuf.length + file.comment.length; - } - const data = getDataHelper(length + 22); - for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) { - file = files[filenames[indexFilename]]; - data.view.setUint32(index, 0x504b0102); - data.view.setUint16(index + 4, 0x1400); - data.array.set(file.header.array, index + 6); - data.view.setUint16(index + 32, file.comment.length, true); - if (file.directory) { - data.view.setUint8(index + 38, 0x10); - } - data.view.setUint32(index + 42, file.offset, true); - data.array.set(file.nameBuf, index + 46); - data.array.set(file.comment, index + 46 + file.nameBuf.length); - index += 46 + file.nameBuf.length + file.comment.length; - } - data.view.setUint32(index, 0x504b0506); - data.view.setUint16(index + 8, filenames.length, true); - data.view.setUint16(index + 10, filenames.length, true); - data.view.setUint32(index + 12, length, true); - data.view.setUint32(index + 16, offset, true); - ctrl.enqueue(data.array); - ctrl.close(); - } - - function processNextChunk() { - if (!activeZipObject) return; - if (activeZipObject.reader) return pump(activeZipObject); - if (activeZipObject.fileLike.stream) { - activeZipObject.crc = new Crc32(); - activeZipObject.reader = activeZipObject.fileLike.stream().getReader(); - activeZipObject.writeHeader(); - } else next(); - } - - return new ReadableStream({ - start: c => { - ctrl = c; - underlyingSource.start && Promise.resolve(underlyingSource.start(zipWriter)); - }, - pull() { - return processNextChunk() || ( - underlyingSource.pull && - Promise.resolve(underlyingSource.pull(zipWriter)) - ); - }, - }); -} diff --git a/yarn.lock b/yarn.lock index 8801940a35..dc3bc2ca7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1468,6 +1468,11 @@ resolved "https://registry.yarnpkg.com/@types/fbemitter/-/fbemitter-2.0.32.tgz#8ed204da0f54e9c8eaec31b1eec91e25132d082c" integrity sha1-jtIE2g9U6cjq7DGx7skeJRMtCCw= +"@types/file-saver@^2.0.3": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.3.tgz#b734c4f5a04d20615eaed3dc106e2ab321082009" + integrity sha512-MBIou8pd/41jkff7s97B47bc9+p0BszqqDJsO51yDm49uUxeKzrfuNl5fSLC6BpLEWKA8zlwyqALVmXrFwoBHQ== + "@types/flux@^3.1.9": version "3.1.10" resolved "https://registry.yarnpkg.com/@types/flux/-/flux-3.1.10.tgz#7c6306e86ecb434d00f38cb82f092640c7bd4098" @@ -4141,6 +4146,11 @@ ignore@^5.1.4, ignore@^5.1.8: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps= + immutable@^3.7.4: version "3.8.2" resolved "https://registry.yarnpkg.com/immutable/-/immutable-3.8.2.tgz#c2439951455bb39913daf281376f1530e104adf3" @@ -5235,6 +5245,16 @@ jsprim@^1.2.2: array-includes "^3.1.2" object.assign "^4.1.2" +jszip@^3.7.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.7.0.tgz#9b8b995a4e7c9024653ce743e902076a82fdf4e6" + integrity sha512-Y2OlFIzrDOPWUnpU0LORIcDn2xN7rC9yKffFM/7pGhQuhO+SUhfm2trkJ/S5amjFvem0Y+1EALz/MEPkvHXVNw== + dependencies: + lie "~3.3.0" + pako "~1.0.2" + readable-stream "~2.3.6" + set-immediate-shim "~1.0.1" + katex@^0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/katex/-/katex-0.12.0.tgz#2fb1c665dbd2b043edcf8a1f5c555f46beaa0cb9" @@ -5302,6 +5322,13 @@ levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" +lie@~3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + lines-and-columns@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" @@ -6015,6 +6042,11 @@ pako@^2.0.3: resolved "https://registry.yarnpkg.com/pako/-/pako-2.0.3.tgz#cdf475e31b678565251406de9e759196a0ea7a43" integrity sha512-WjR1hOeg+kki3ZIOjaf4b5WVcay1jaliKSYiEaB1XzwhMQZJxRdQRv0V31EKBYlxb4T7SK3hjfc/jxyU64BoSw== +pako@~1.0.2: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -6552,7 +6584,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -readable-stream@^2.0.2: +readable-stream@^2.0.2, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -6953,6 +6985,11 @@ set-blocking@^2.0.0: resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= +set-immediate-shim@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61" + integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E= + set-value@^2.0.0, set-value@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" @@ -7206,11 +7243,6 @@ static-extend@^0.1.1: define-property "^0.2.5" object-copy "^0.1.0" -streamsaver@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/streamsaver/-/streamsaver-2.0.5.tgz#3212f0e908fcece5b3a65591094475cf87850d00" - integrity sha512-KIWtBvi8A6FiFZGNSyuIZRZM6C8AvnWTiCx/TYa7so420vC5sQwcBKkdqInuGWoWMfeWy/P+/cRqMtWVf4RW9w== - string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -7916,11 +7948,6 @@ walker@^1.0.7, walker@~1.0.5: dependencies: makeerror "1.0.x" -web-streams-polyfill@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz#f49e487eedeca47a207c1aee41ee5578f884b42f" - integrity sha512-d2H/t0eqRNM4w2WvmTdoeIvzAUSpK7JmATB8Nr2lb7nQ9BTIJVjbQ/TRFVEh2gUH1HwclPdoPtfMoFfetXaZnA== - webcrypto-core@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/webcrypto-core/-/webcrypto-core-1.2.0.tgz#44fda3f9315ed6effe9a1e47466e0935327733b5"