diff --git a/.env.example b/.env.example index cf2dafc1d..e97dacccb 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ JWT_TOKEN_EXPIRES_IN=30d DATABASE_URL="postgresql://postgres:password@localhost:5432/docmost?schema=public" REDIS_URL=redis://127.0.0.1:6379 -# options: local | s3 +# options: local | s3 | azure STORAGE_DRIVER=local # S3 driver config @@ -21,6 +21,12 @@ AWS_S3_BUCKET= AWS_S3_ENDPOINT= AWS_S3_FORCE_PATH_STYLE= +# Azure Blob Storage driver config +STORAGE_DRIVER=azure +AZURE_STORAGE_ACCOUNT_NAME= +AZURE_STORAGE_ACCOUNT_KEY= +AZURE_STORAGE_CONTAINER= + # default: 50mb FILE_UPLOAD_SIZE_LIMIT= diff --git a/apps/server/package.json b/apps/server/package.json index 8500ace47..40e11e32f 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -36,6 +36,7 @@ "@aws-sdk/client-s3": "3.1050.0", "@aws-sdk/lib-storage": "3.1050.0", "@aws-sdk/s3-request-presigner": "3.1050.0", + "@azure/storage-blob": "12.31.0", "@clickhouse/client": "^1.18.2", "@docmost/pdf-inspector": "1.9.4", "@fastify/cookie": "^11.0.2", @@ -163,7 +164,21 @@ "rootDir": "src", "testRegex": ".*\\.spec\\.ts$", "transform": { - "happy-dom.+\\.js$": ["babel-jest", { "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] }], + "happy-dom.+\\.js$": [ + "babel-jest", + { + "presets": [ + [ + "@babel/preset-env", + { + "targets": { + "node": "current" + } + } + ] + ] + } + ], "^.+\\.(t|j)s$": "ts-jest" }, "transformIgnorePatterns": [ diff --git a/apps/server/src/integrations/environment/environment.service.ts b/apps/server/src/integrations/environment/environment.service.ts index 20824d355..5667bf5a6 100644 --- a/apps/server/src/integrations/environment/environment.service.ts +++ b/apps/server/src/integrations/environment/environment.service.ts @@ -122,6 +122,26 @@ export class EnvironmentService { return this.configService.get('AWS_S3_URL'); } + getAzureStorageAccountName(): string { + return this.configService.get('AZURE_STORAGE_ACCOUNT_NAME'); + } + + getAzureStorageContainer(): string { + return this.configService.get('AZURE_STORAGE_CONTAINER'); + } + + getAzureStorageAccountKey(): string { + return this.configService.get('AZURE_STORAGE_ACCOUNT_KEY'); + } + + getAzureStorageEndpoint(): string { + return this.configService.get('AZURE_STORAGE_ENDPOINT'); + } + + getAzureStorageUrl(): string { + return this.configService.get('AZURE_STORAGE_URL'); + } + getMailDriver(): string { return this.configService.get('MAIL_DRIVER', 'log'); } diff --git a/apps/server/src/integrations/environment/environment.validation.ts b/apps/server/src/integrations/environment/environment.validation.ts index 3a59b08cd..ef3c420cf 100644 --- a/apps/server/src/integrations/environment/environment.validation.ts +++ b/apps/server/src/integrations/environment/environment.validation.ts @@ -49,7 +49,7 @@ export class EnvironmentVariables { MAIL_DRIVER: string; @IsOptional() - @IsIn(['local', 's3']) + @IsIn(['local', 's3', 'azure']) STORAGE_DRIVER: string; @IsOptional() diff --git a/apps/server/src/integrations/storage/drivers/azure.driver.ts b/apps/server/src/integrations/storage/drivers/azure.driver.ts new file mode 100644 index 000000000..034c46032 --- /dev/null +++ b/apps/server/src/integrations/storage/drivers/azure.driver.ts @@ -0,0 +1,192 @@ +import { Readable } from 'stream'; +import { + AzureStorageConfig, + StorageDriver, + StorageOption, +} from '../interfaces'; +import { + BlobSASPermissions, + BlobServiceClient, + BlockBlobClient, + ContainerClient, + generateBlobSASQueryParameters, + SASProtocol, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { Logger } from '@nestjs/common'; +import { getMimeType } from '../../../common/helpers'; + +export class AzureDriver implements StorageDriver { + private readonly config: AzureStorageConfig; + private readonly blobServiceClient: BlobServiceClient; + private readonly containerClient: ContainerClient; + private readonly sharedKeyCredential: StorageSharedKeyCredential; + private readonly accountUrl: string; + + constructor(config: AzureStorageConfig) { + this.config = config; + + if (!config.accountName) { + throw new Error('AzureDriver: accountName is required'); + } + if (!config.container) { + throw new Error('AzureDriver: container is required'); + } + if (!config.accountKey) { + throw new Error('AzureDriver: accountKey is required'); + } + + this.accountUrl = + config.endpoint ?? + `https://${config.accountName}.blob.core.windows.net`; + + this.sharedKeyCredential = new StorageSharedKeyCredential( + config.accountName, + config.accountKey, + ); + + this.blobServiceClient = this.createBlobServiceClient(); + this.containerClient = this.blobServiceClient.getContainerClient( + config.container, + ); + } + + private blockBlob(filePath: string): BlockBlobClient { + return this.containerClient.getBlockBlobClient(filePath); + } + + async upload(filePath: string, file: Buffer | Readable): Promise { + const stream: Readable = Buffer.isBuffer(file) ? Readable.from(file) : file; + await this.uploadStream(filePath, stream); + } + + async uploadStream( + filePath: string, + file: Readable, + options?: { recreateClient?: boolean }, + ): Promise { + const clientToUse = options?.recreateClient + ? this.createBlobServiceClient() + .getContainerClient(this.config.container) + .getBlockBlobClient(filePath) + : this.blockBlob(filePath); + + try { + const contentType = getMimeType(filePath); + await clientToUse.uploadStream(file, undefined, undefined, { + blobHTTPHeaders: { blobContentType: contentType }, + }); + } catch (err) { + Logger.error(err); + throw new Error(`Failed to upload file: ${(err as Error).message}`); + } + } + + async copy(fromFilePath: string, toFilePath: string): Promise { + try { + if (!(await this.exists(fromFilePath))) { + return; + } + const sourceUrl = await this.getSignedUrl(fromFilePath, 60); + const dest = this.blockBlob(toFilePath); + await dest.syncCopyFromURL(sourceUrl); + } catch (err) { + throw new Error(`Failed to copy file: ${(err as Error).message}`); + } + } + + async read(filePath: string): Promise { + try { + return await this.blockBlob(filePath).downloadToBuffer(); + } catch (err) { + throw new Error( + `Failed to read file from Azure: ${(err as Error).message}`, + ); + } + } + + async readStream(filePath: string): Promise { + try { + const response = await this.blockBlob(filePath).download(); + return response.readableStreamBody as Readable; + } catch (err) { + throw new Error( + `Failed to read file from Azure: ${(err as Error).message}`, + ); + } + } + + async readRangeStream( + filePath: string, + range: { start: number; end: number }, + ): Promise { + try { + const count = range.end - range.start + 1; + const response = await this.blockBlob(filePath).download( + range.start, + count, + ); + return response.readableStreamBody as Readable; + } catch (err) { + throw new Error( + `Failed to read file from Azure: ${(err as Error).message}`, + ); + } + } + + async exists(filePath: string): Promise { + try { + return await this.blockBlob(filePath).exists(); + } catch (err) { + throw new Error( + `Failed to check existence in Azure: ${(err as Error).message}`, + ); + } + } + + getUrl(filePath: string): string { + const base = this.config.baseUrl ?? this.accountUrl; + return `${base}/${this.config.container}/${filePath}`; + } + + async getSignedUrl(filePath: string, expiresIn: number): Promise { + const expiresOn = new Date(Date.now() + expiresIn * 1000); + const sas = generateBlobSASQueryParameters( + { + containerName: this.config.container, + blobName: filePath, + permissions: BlobSASPermissions.parse('r'), + expiresOn, + protocol: SASProtocol.HttpsAndHttp, + }, + this.sharedKeyCredential, + ).toString(); + return `${this.accountUrl}/${this.config.container}/${filePath}?${sas}`; + } + + async delete(filePath: string): Promise { + try { + await this.blockBlob(filePath).delete(); + } catch (err) { + throw new Error( + `Error deleting file ${filePath} from Azure: ${(err as Error).message}`, + ); + } + } + + getDriver(): BlobServiceClient { + return this.blobServiceClient; + } + + getDriverName(): string { + return StorageOption.AZURE; + } + + getConfig(): Record { + return this.config; + } + + private createBlobServiceClient(): BlobServiceClient { + return new BlobServiceClient(this.accountUrl, this.sharedKeyCredential); + } +} diff --git a/apps/server/src/integrations/storage/drivers/index.ts b/apps/server/src/integrations/storage/drivers/index.ts index 02ab4b30c..5b7a326ea 100644 --- a/apps/server/src/integrations/storage/drivers/index.ts +++ b/apps/server/src/integrations/storage/drivers/index.ts @@ -1,2 +1,3 @@ export { LocalDriver } from './local.driver'; export { S3Driver } from './s3.driver'; +export { AzureDriver } from './azure.driver'; diff --git a/apps/server/src/integrations/storage/interfaces/storage.interface.ts b/apps/server/src/integrations/storage/interfaces/storage.interface.ts index 48c684919..ef37f43c0 100644 --- a/apps/server/src/integrations/storage/interfaces/storage.interface.ts +++ b/apps/server/src/integrations/storage/interfaces/storage.interface.ts @@ -3,11 +3,13 @@ import { S3ClientConfig } from '@aws-sdk/client-s3'; export enum StorageOption { LOCAL = 'local', S3 = 's3', + AZURE = 'azure', } export type StorageConfig = | { driver: StorageOption.LOCAL; config: LocalStorageConfig } - | { driver: StorageOption.S3; config: S3StorageConfig }; + | { driver: StorageOption.S3; config: S3StorageConfig } + | { driver: StorageOption.AZURE; config: AzureStorageConfig }; export interface LocalStorageConfig { storagePath: string; @@ -20,6 +22,14 @@ export interface S3StorageConfig baseUrl?: string; // Optional CDN URL for assets } +export interface AzureStorageConfig { + accountName: string; + container: string; + accountKey: string; + endpoint?: string; + baseUrl?: string; +} + export interface StorageOptions { disk: StorageConfig; } diff --git a/apps/server/src/integrations/storage/providers/storage.provider.ts b/apps/server/src/integrations/storage/providers/storage.provider.ts index 114896667..91967f8ec 100644 --- a/apps/server/src/integrations/storage/providers/storage.provider.ts +++ b/apps/server/src/integrations/storage/providers/storage.provider.ts @@ -4,13 +4,14 @@ import { } from '../constants/storage.constants'; import { EnvironmentService } from '../../environment/environment.service'; import { + AzureStorageConfig, LocalStorageConfig, S3StorageConfig, StorageConfig, StorageDriver, StorageOption, } from '../interfaces'; -import { LocalDriver, S3Driver } from '../drivers'; +import { AzureDriver, LocalDriver, S3Driver } from '../drivers'; import * as process from 'node:process'; import { LOCAL_STORAGE_PATH } from '../../../common/helpers'; import path from 'path'; @@ -21,6 +22,8 @@ function createStorageDriver(disk: StorageConfig): StorageDriver { return new LocalDriver(disk.config as LocalStorageConfig); case StorageOption.S3: return new S3Driver(disk.config as S3StorageConfig); + case StorageOption.AZURE: + return new AzureDriver(disk.config as AzureStorageConfig); default: throw new Error(`Unknown storage driver`); } @@ -70,6 +73,18 @@ export const storageDriverConfigProvider = { return s3Config; } + case StorageOption.AZURE: + return { + driver, + config: { + accountName: environmentService.getAzureStorageAccountName(), + container: environmentService.getAzureStorageContainer(), + accountKey: environmentService.getAzureStorageAccountKey(), + endpoint: environmentService.getAzureStorageEndpoint() || undefined, + baseUrl: environmentService.getAzureStorageUrl() || undefined, + }, + }; + default: throw new Error(`Unknown storage driver: ${driver}`); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad2770b45..adfe75314 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -501,6 +501,9 @@ importers: '@aws-sdk/s3-request-presigner': specifier: 3.1050.0 version: 3.1050.0 + '@azure/storage-blob': + specifier: 12.31.0 + version: 12.31.0 '@clickhouse/client': specifier: ^1.18.2 version: 1.18.2 @@ -1114,6 +1117,61 @@ packages: resolution: {integrity: sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==} engines: {node: '>=18.0.0'} + '@azure/abort-controller@2.1.2': + resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==} + engines: {node: '>=18.0.0'} + + '@azure/core-auth@1.10.1': + resolution: {integrity: sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==} + engines: {node: '>=20.0.0'} + + '@azure/core-client@1.10.1': + resolution: {integrity: sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==} + engines: {node: '>=20.0.0'} + + '@azure/core-http-compat@2.4.0': + resolution: {integrity: sha512-f1P96IB399YiN2ARYHP7EpZi3Bf3wH4SN2lGzrw7JVwm7bbsVYtf2iKSBwTywD2P62NOPZGHFSZi+6jjb75JuA==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@azure/core-client': ^1.10.0 + '@azure/core-rest-pipeline': ^1.22.0 + + '@azure/core-lro@2.7.2': + resolution: {integrity: sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==} + engines: {node: '>=18.0.0'} + + '@azure/core-paging@1.6.2': + resolution: {integrity: sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==} + engines: {node: '>=18.0.0'} + + '@azure/core-rest-pipeline@1.23.0': + resolution: {integrity: sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==} + engines: {node: '>=20.0.0'} + + '@azure/core-tracing@1.3.1': + resolution: {integrity: sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==} + engines: {node: '>=20.0.0'} + + '@azure/core-util@1.13.1': + resolution: {integrity: sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==} + engines: {node: '>=20.0.0'} + + '@azure/core-xml@1.5.1': + resolution: {integrity: sha512-xcNRHqCoSp4AunOALEae6A8f3qATb83gSrm31Iqb01OzblvC3/W/bfXozcq78EzIdzZzuH1bZ2NvRR0TdX709w==} + engines: {node: '>=20.0.0'} + + '@azure/logger@1.3.0': + resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==} + engines: {node: '>=20.0.0'} + + '@azure/storage-blob@12.31.0': + resolution: {integrity: sha512-DBgNv10aCSxopt92DkTDD0o9xScXeBqPKGmR50FPZQaEcH4JLQ+GEOGEDv19V5BMkB7kxr+m4h6il/cCDPvmHg==} + engines: {node: '>=20.0.0'} + + '@azure/storage-common@12.3.0': + resolution: {integrity: sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==} + engines: {node: '>=20.0.0'} + '@babel/code-frame@7.27.1': resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} engines: {node: '>=6.9.0'} @@ -4965,6 +5023,10 @@ packages: resolution: {integrity: sha512-YWnmJkXbofiz9KbnbbwuA2rpGkFPLbAIetcCNO6mJ8gdhdZ/v7WDXsoGFAJuM6ikUFKTlSQnjWnVO4ux+UzS6A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typespec/ts-http-runtime@0.3.5': + resolution: {integrity: sha512-yURCknZhvywvQItHMMmFSo+fq5arCUIyz/CVk7jD89MSai7dkaX8ufjCWp3NttLojoTVbcE72ri+be/TnEbMHw==} + engines: {node: '>=20.0.0'} + '@ucast/core@1.10.2': resolution: {integrity: sha512-ons5CwXZ/51wrUPfoduC+cO7AS1/wRb0ybpQJ9RrssossDxVy4t49QxWoWgfBDvVKsz9VXzBk9z0wqTdZ+Cq8g==} @@ -10854,6 +10916,119 @@ snapshots: '@aws/lambda-invoke-store@0.2.3': {} + '@azure/abort-controller@2.1.2': + dependencies: + tslib: 2.8.1 + + '@azure/core-auth@1.10.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.1 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-client@1.10.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-http-compat@2.4.0(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0)': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-client': 1.10.1 + '@azure/core-rest-pipeline': 1.23.0 + + '@azure/core-lro@2.7.2': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-paging@1.6.2': + dependencies: + tslib: 2.8.1 + + '@azure/core-rest-pipeline@1.23.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + '@typespec/ts-http-runtime': 0.3.5 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-tracing@1.3.1': + dependencies: + tslib: 2.8.1 + + '@azure/core-util@1.13.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@typespec/ts-http-runtime': 0.3.5 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-xml@1.5.1': + dependencies: + fast-xml-parser: 5.7.3 + tslib: 2.8.1 + + '@azure/logger@1.3.0': + dependencies: + '@typespec/ts-http-runtime': 0.3.5 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/storage-blob@12.31.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-client': 1.10.1 + '@azure/core-http-compat': 2.4.0(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0) + '@azure/core-lro': 2.7.2 + '@azure/core-paging': 1.6.2 + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/core-xml': 1.5.1 + '@azure/logger': 1.3.0 + '@azure/storage-common': 12.3.0(@azure/core-client@1.10.1) + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/storage-common@12.3.0(@azure/core-client@1.10.1)': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-http-compat': 2.4.0(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0) + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - '@azure/core-client' + - supports-color + '@babel/code-frame@7.27.1': dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -15106,6 +15281,14 @@ snapshots: '@typescript-eslint/types': 8.57.1 eslint-visitor-keys: 5.0.1 + '@typespec/ts-http-runtime@0.3.5': + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + '@ucast/core@1.10.2': {} '@ucast/js@3.0.4':