feat: bulk page imports (#1219)

* refactor imports - WIP

* Add readstream

* WIP

* fix attachmentId render

* fix attachmentId render

* turndown video tag

* feat: add stream upload support and improve file handling

- Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility

* WIP

* notion formatter

* move embed parser to editor-ext package

* import embeds

* utility files

* cleanup

* Switch from happy-dom to cheerio
* Refine code

* WIP

* bug fixes and UI

* sync

* WIP

* sync

* keep import modal mounted

* Show modal during upload

* WIP

* WIP

apps/server/src/integrations/import/import.controller.ts

@@ -21,8 +21,9 @@ import {
 import { FileInterceptor } from '../../common/interceptors/file.interceptor';
 import * as bytes from 'bytes';
 import * as path from 'path';
-import { ImportService } from './import.service';
+import { ImportService } from './services/import.service';
 import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
+import { EnvironmentService } from '../environment/environment.service';
 
 @Controller()
 export class ImportController {
@@ -31,6 +32,7 @@ export class ImportController {
   constructor(
     private readonly importService: ImportService,
     private readonly spaceAbility: SpaceAbilityFactory,
+    private readonly environmentService: EnvironmentService,
   ) {}
 
   @UseInterceptors(FileInterceptor)
@@ -44,18 +46,18 @@ export class ImportController {
   ) {
     const validFileExtensions = ['.md', '.html'];
 
-    const maxFileSize = bytes('100mb');
+    const maxFileSize = bytes('10mb');
 
     let file = null;
     try {
       file = await req.file({
-        limits: { fileSize: maxFileSize, fields: 3, files: 1 },
+        limits: { fileSize: maxFileSize, fields: 4, files: 1 },
       });
     } catch (err: any) {
       this.logger.error(err.message);
       if (err?.statusCode === 413) {
         throw new BadRequestException(
-          `File too large. Exceeds the 100mb import limit`,
+          `File too large. Exceeds the 10mb import limit`,
         );
       }
     }
@@ -73,7 +75,7 @@ export class ImportController {
     const spaceId = file.fields?.spaceId?.value;
 
     if (!spaceId) {
-      throw new BadRequestException('spaceId or format not found');
+      throw new BadRequestException('spaceId is required');
     }
 
     const ability = await this.spaceAbility.createForUser(user, spaceId);
@@ -83,4 +85,69 @@ export class ImportController {
 
     return this.importService.importPage(file, user.id, spaceId, workspace.id);
   }
+
+  @UseInterceptors(FileInterceptor)
+  @UseGuards(JwtAuthGuard)
+  @HttpCode(HttpStatus.OK)
+  @Post('pages/import-zip')
+  async importZip(
+    @Req() req: any,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
+    const validFileExtensions = ['.zip'];
+
+    const maxFileSize = bytes(this.environmentService.getFileImportSizeLimit());
+
+    let file = null;
+    try {
+      file = await req.file({
+        limits: { fileSize: maxFileSize, fields: 3, files: 1 },
+      });
+    } catch (err: any) {
+      this.logger.error(err.message);
+      if (err?.statusCode === 413) {
+        throw new BadRequestException(
+          `File too large. Exceeds the ${this.environmentService.getFileImportSizeLimit()} import limit`,
+        );
+      }
+    }
+
+    if (!file) {
+      throw new BadRequestException('Failed to upload file');
+    }
+
+    if (
+      !validFileExtensions.includes(path.extname(file.filename).toLowerCase())
+    ) {
+      throw new BadRequestException('Invalid import file extension.');
+    }
+
+    const spaceId = file.fields?.spaceId?.value;
+    const source = file.fields?.source?.value;
+
+    const validZipSources = ['generic', 'notion', 'confluence'];
+    if (!validZipSources.includes(source)) {
+      throw new BadRequestException(
+        'Invalid import source. Import source must either be generic, notion or confluence.',
+      );
+    }
+
+    if (!spaceId) {
+      throw new BadRequestException('spaceId is required');
+    }
+
+    const ability = await this.spaceAbility.createForUser(user, spaceId);
+    if (ability.cannot(SpaceCaslAction.Edit, SpaceCaslSubject.Page)) {
+      throw new ForbiddenException();
+    }
+
+    return this.importService.importZip(
+      file,
+      source,
+      user.id,
+      spaceId,
+      workspace.id,
+    );
+  }
 }