feat: bulk page imports (#1219)

* refactor imports - WIP

* Add readstream

* WIP

* fix attachmentId render

* fix attachmentId render

* turndown video tag

* feat: add stream upload support and improve file handling

- Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility

* WIP

* notion formatter

* move embed parser to editor-ext package

* import embeds

* utility files

* cleanup

* Switch from happy-dom to cheerio
* Refine code

* WIP

* bug fixes and UI

* sync

* WIP

* sync

* keep import modal mounted

* Show modal during upload

* WIP

* WIP

apps/server/src/integrations/storage/drivers/local.driver.ts

@@ -3,8 +3,11 @@ import {
   LocalStorageConfig,
   StorageOption,
 } from '../interfaces';
-import { join } from 'path';
+import { join, dirname } from 'path';
 import * as fs from 'fs-extra';
+import { Readable } from 'stream';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
 
 export class LocalDriver implements StorageDriver {
   private readonly config: LocalStorageConfig;
@@ -25,6 +28,16 @@ export class LocalDriver implements StorageDriver {
     }
   }
 
+  async uploadStream(filePath: string, file: Readable): Promise<void> {
+    try {
+      const fullPath = this._fullPath(filePath);
+      await fs.mkdir(dirname(fullPath), { recursive: true });
+      await pipeline(file, createWriteStream(fullPath));
+    } catch (err) {
+      throw new Error(`Failed to upload file: ${(err as Error).message}`);
+    }
+  }
+
   async copy(fromFilePath: string, toFilePath: string): Promise<void> {
     try {
       if (await this.exists(fromFilePath)) {
@@ -43,6 +56,14 @@ export class LocalDriver implements StorageDriver {
     }
   }
 
+  async readStream(filePath: string): Promise<Readable> {
+    try {
+      return createReadStream(this._fullPath(filePath));
+    } catch (err) {
+      throw new Error(`Failed to read file: ${(err as Error).message}`);
+    }
+  }
+
   async exists(filePath: string): Promise<boolean> {
     try {
       return await fs.pathExists(this._fullPath(filePath));

apps/server/src/integrations/storage/drivers/s3.driver.ts

@@ -12,6 +12,7 @@ import { streamToBuffer } from '../storage.utils';
 import { Readable } from 'stream';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { getMimeType } from '../../../common/helpers';
+import { Upload } from '@aws-sdk/lib-storage';
 
 export class S3Driver implements StorageDriver {
   private readonly s3Client: S3Client;
@@ -40,6 +41,26 @@ export class S3Driver implements StorageDriver {
     }
   }
 
+  async uploadStream(filePath: string, file: Readable): Promise<void> {
+    try {
+      const contentType = getMimeType(filePath);
+
+      const upload = new Upload({
+        client: this.s3Client,
+        params: {
+          Bucket: this.config.bucket,
+          Key: filePath,
+          Body: file,
+          ContentType: contentType,
+        },
+      });
+
+      await upload.done();
+    } catch (err) {
+      throw new Error(`Failed to upload file: ${(err as Error).message}`);
+    }
+  }
+
   async copy(fromFilePath: string, toFilePath: string): Promise<void> {
     try {
       if (await this.exists(fromFilePath)) {
@@ -71,6 +92,21 @@ export class S3Driver implements StorageDriver {
     }
   }
 
+  async readStream(filePath: string): Promise<Readable> {
+    try {
+      const command = new GetObjectCommand({
+        Bucket: this.config.bucket,
+        Key: filePath,
+      });
+
+      const response = await this.s3Client.send(command);
+
+      return response.Body as Readable;
+    } catch (err) {
+      throw new Error(`Failed to read file from S3: ${(err as Error).message}`);
+    }
+  }
+
   async exists(filePath: string): Promise<boolean> {
     try {
       const command = new HeadObjectCommand({

apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts

@@ -1,10 +1,16 @@
+import { Readable } from 'stream';
+
 export interface StorageDriver {
   upload(filePath: string, file: Buffer): Promise<void>;
 
+  uploadStream(filePath: string, file: Readable): Promise<void>;
+
   copy(fromFilePath: string, toFilePath: string): Promise<void>;
 
   read(filePath: string): Promise<Buffer>;
 
+  readStream(filePath: string): Promise<Readable>;
+
   exists(filePath: string): Promise<boolean>;
 
   getUrl(filePath: string): string;

apps/server/src/integrations/storage/storage.service.ts

@@ -1,6 +1,7 @@
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { STORAGE_DRIVER_TOKEN } from './constants/storage.constants';
 import { StorageDriver } from './interfaces';
+import { Readable } from 'stream';
 
 @Injectable()
 export class StorageService {
@@ -14,6 +15,11 @@ export class StorageService {
     this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
   }
 
+  async uploadStream(filePath: string, fileContent: Readable) {
+    await this.storageDriver.uploadStream(filePath, fileContent);
+    this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
+  }
+
   async copy(fromFilePath: string, toFilePath: string) {
     await this.storageDriver.copy(fromFilePath, toFilePath);
     this.logger.debug(`File copied successfully. Path: ${toFilePath}`);
@@ -23,6 +29,10 @@ export class StorageService {
     return this.storageDriver.read(filePath);
   }
 
+  async readStream(filePath: string): Promise<Readable> {
+    return this.storageDriver.readStream(filePath);
+  }
+
   async exists(filePath: string): Promise<boolean> {
     return this.storageDriver.exists(filePath);
   }