Compare commits

...

2 Commits

Author SHA1 Message Date
Philipinho 13a7f1372f fix: update pdf-inspector package 2026-05-21 13:44:11 +01:00
Philip Okugbe 4295ea09f6 feat(storage): add Azure Blob Storage driver (#2222) 2026-05-21 12:18:58 +01:00
9 changed files with 453 additions and 11 deletions
+7 -1
View File
@@ -10,7 +10,7 @@ JWT_TOKEN_EXPIRES_IN=30d
DATABASE_URL="postgresql://postgres:password@localhost:5432/docmost?schema=public"
REDIS_URL=redis://127.0.0.1:6379
# options: local | s3
# options: local | s3 | azure
STORAGE_DRIVER=local
# S3 driver config
@@ -21,6 +21,12 @@ AWS_S3_BUCKET=
AWS_S3_ENDPOINT=
AWS_S3_FORCE_PATH_STYLE=
# Azure Blob Storage driver config
STORAGE_DRIVER=azure
AZURE_STORAGE_ACCOUNT_NAME=
AZURE_STORAGE_ACCOUNT_KEY=
AZURE_STORAGE_CONTAINER=
# default: 50mb
FILE_UPLOAD_SIZE_LIMIT=
+17 -2
View File
@@ -36,8 +36,9 @@
"@aws-sdk/client-s3": "3.1050.0",
"@aws-sdk/lib-storage": "3.1050.0",
"@aws-sdk/s3-request-presigner": "3.1050.0",
"@azure/storage-blob": "12.31.0",
"@clickhouse/client": "^1.18.2",
"@docmost/pdf-inspector": "1.9.4",
"@docmost/pdf-inspector": "1.9.6",
"@fastify/cookie": "^11.0.2",
"@fastify/multipart": "^10.0.0",
"@fastify/static": "^9.1.3",
@@ -163,7 +164,21 @@
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"happy-dom.+\\.js$": ["babel-jest", { "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] }],
"happy-dom.+\\.js$": [
"babel-jest",
{
"presets": [
[
"@babel/preset-env",
{
"targets": {
"node": "current"
}
}
]
]
}
],
"^.+\\.(t|j)s$": "ts-jest"
},
"transformIgnorePatterns": [
@@ -122,6 +122,26 @@ export class EnvironmentService {
return this.configService.get<string>('AWS_S3_URL');
}
getAzureStorageAccountName(): string {
return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_NAME');
}
getAzureStorageContainer(): string {
return this.configService.get<string>('AZURE_STORAGE_CONTAINER');
}
getAzureStorageAccountKey(): string {
return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_KEY');
}
getAzureStorageEndpoint(): string {
return this.configService.get<string>('AZURE_STORAGE_ENDPOINT');
}
getAzureStorageUrl(): string {
return this.configService.get<string>('AZURE_STORAGE_URL');
}
getMailDriver(): string {
return this.configService.get<string>('MAIL_DRIVER', 'log');
}
@@ -49,7 +49,7 @@ export class EnvironmentVariables {
MAIL_DRIVER: string;
@IsOptional()
@IsIn(['local', 's3'])
@IsIn(['local', 's3', 'azure'])
STORAGE_DRIVER: string;
@IsOptional()
@@ -0,0 +1,192 @@
import { Readable } from 'stream';
import {
AzureStorageConfig,
StorageDriver,
StorageOption,
} from '../interfaces';
import {
BlobSASPermissions,
BlobServiceClient,
BlockBlobClient,
ContainerClient,
generateBlobSASQueryParameters,
SASProtocol,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { Logger } from '@nestjs/common';
import { getMimeType } from '../../../common/helpers';
export class AzureDriver implements StorageDriver {
private readonly config: AzureStorageConfig;
private readonly blobServiceClient: BlobServiceClient;
private readonly containerClient: ContainerClient;
private readonly sharedKeyCredential: StorageSharedKeyCredential;
private readonly accountUrl: string;
constructor(config: AzureStorageConfig) {
this.config = config;
if (!config.accountName) {
throw new Error('AzureDriver: accountName is required');
}
if (!config.container) {
throw new Error('AzureDriver: container is required');
}
if (!config.accountKey) {
throw new Error('AzureDriver: accountKey is required');
}
this.accountUrl =
config.endpoint ??
`https://${config.accountName}.blob.core.windows.net`;
this.sharedKeyCredential = new StorageSharedKeyCredential(
config.accountName,
config.accountKey,
);
this.blobServiceClient = this.createBlobServiceClient();
this.containerClient = this.blobServiceClient.getContainerClient(
config.container,
);
}
private blockBlob(filePath: string): BlockBlobClient {
return this.containerClient.getBlockBlobClient(filePath);
}
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
const stream: Readable = Buffer.isBuffer(file) ? Readable.from(file) : file;
await this.uploadStream(filePath, stream);
}
async uploadStream(
filePath: string,
file: Readable,
options?: { recreateClient?: boolean },
): Promise<void> {
const clientToUse = options?.recreateClient
? this.createBlobServiceClient()
.getContainerClient(this.config.container)
.getBlockBlobClient(filePath)
: this.blockBlob(filePath);
try {
const contentType = getMimeType(filePath);
await clientToUse.uploadStream(file, undefined, undefined, {
blobHTTPHeaders: { blobContentType: contentType },
});
} catch (err) {
Logger.error(err);
throw new Error(`Failed to upload file: ${(err as Error).message}`);
}
}
async copy(fromFilePath: string, toFilePath: string): Promise<void> {
try {
if (!(await this.exists(fromFilePath))) {
return;
}
const sourceUrl = await this.getSignedUrl(fromFilePath, 60);
const dest = this.blockBlob(toFilePath);
await dest.syncCopyFromURL(sourceUrl);
} catch (err) {
throw new Error(`Failed to copy file: ${(err as Error).message}`);
}
}
async read(filePath: string): Promise<Buffer> {
try {
return await this.blockBlob(filePath).downloadToBuffer();
} catch (err) {
throw new Error(
`Failed to read file from Azure: ${(err as Error).message}`,
);
}
}
async readStream(filePath: string): Promise<Readable> {
try {
const response = await this.blockBlob(filePath).download();
return response.readableStreamBody as Readable;
} catch (err) {
throw new Error(
`Failed to read file from Azure: ${(err as Error).message}`,
);
}
}
async readRangeStream(
filePath: string,
range: { start: number; end: number },
): Promise<Readable> {
try {
const count = range.end - range.start + 1;
const response = await this.blockBlob(filePath).download(
range.start,
count,
);
return response.readableStreamBody as Readable;
} catch (err) {
throw new Error(
`Failed to read file from Azure: ${(err as Error).message}`,
);
}
}
async exists(filePath: string): Promise<boolean> {
try {
return await this.blockBlob(filePath).exists();
} catch (err) {
throw new Error(
`Failed to check existence in Azure: ${(err as Error).message}`,
);
}
}
getUrl(filePath: string): string {
const base = this.config.baseUrl ?? this.accountUrl;
return `${base}/${this.config.container}/${filePath}`;
}
async getSignedUrl(filePath: string, expiresIn: number): Promise<string> {
const expiresOn = new Date(Date.now() + expiresIn * 1000);
const sas = generateBlobSASQueryParameters(
{
containerName: this.config.container,
blobName: filePath,
permissions: BlobSASPermissions.parse('r'),
expiresOn,
protocol: SASProtocol.HttpsAndHttp,
},
this.sharedKeyCredential,
).toString();
return `${this.accountUrl}/${this.config.container}/${filePath}?${sas}`;
}
async delete(filePath: string): Promise<void> {
try {
await this.blockBlob(filePath).delete();
} catch (err) {
throw new Error(
`Error deleting file ${filePath} from Azure: ${(err as Error).message}`,
);
}
}
getDriver(): BlobServiceClient {
return this.blobServiceClient;
}
getDriverName(): string {
return StorageOption.AZURE;
}
getConfig(): Record<string, any> {
return this.config;
}
private createBlobServiceClient(): BlobServiceClient {
return new BlobServiceClient(this.accountUrl, this.sharedKeyCredential);
}
}
@@ -1,2 +1,3 @@
export { LocalDriver } from './local.driver';
export { S3Driver } from './s3.driver';
export { AzureDriver } from './azure.driver';
@@ -3,11 +3,13 @@ import { S3ClientConfig } from '@aws-sdk/client-s3';
export enum StorageOption {
LOCAL = 'local',
S3 = 's3',
AZURE = 'azure',
}
export type StorageConfig =
| { driver: StorageOption.LOCAL; config: LocalStorageConfig }
| { driver: StorageOption.S3; config: S3StorageConfig };
| { driver: StorageOption.S3; config: S3StorageConfig }
| { driver: StorageOption.AZURE; config: AzureStorageConfig };
export interface LocalStorageConfig {
storagePath: string;
@@ -20,6 +22,14 @@ export interface S3StorageConfig
baseUrl?: string; // Optional CDN URL for assets
}
export interface AzureStorageConfig {
accountName: string;
container: string;
accountKey: string;
endpoint?: string;
baseUrl?: string;
}
export interface StorageOptions {
disk: StorageConfig;
}
@@ -4,13 +4,14 @@ import {
} from '../constants/storage.constants';
import { EnvironmentService } from '../../environment/environment.service';
import {
AzureStorageConfig,
LocalStorageConfig,
S3StorageConfig,
StorageConfig,
StorageDriver,
StorageOption,
} from '../interfaces';
import { LocalDriver, S3Driver } from '../drivers';
import { AzureDriver, LocalDriver, S3Driver } from '../drivers';
import * as process from 'node:process';
import { LOCAL_STORAGE_PATH } from '../../../common/helpers';
import path from 'path';
@@ -21,6 +22,8 @@ function createStorageDriver(disk: StorageConfig): StorageDriver {
return new LocalDriver(disk.config as LocalStorageConfig);
case StorageOption.S3:
return new S3Driver(disk.config as S3StorageConfig);
case StorageOption.AZURE:
return new AzureDriver(disk.config as AzureStorageConfig);
default:
throw new Error(`Unknown storage driver`);
}
@@ -70,6 +73,18 @@ export const storageDriverConfigProvider = {
return s3Config; }
case StorageOption.AZURE:
return {
driver,
config: {
accountName: environmentService.getAzureStorageAccountName(),
container: environmentService.getAzureStorageContainer(),
accountKey: environmentService.getAzureStorageAccountKey(),
endpoint: environmentService.getAzureStorageEndpoint() || undefined,
baseUrl: environmentService.getAzureStorageUrl() || undefined,
},
};
default:
throw new Error(`Unknown storage driver: ${driver}`);
}
+188 -5
View File
@@ -501,12 +501,15 @@ importers:
'@aws-sdk/s3-request-presigner':
specifier: 3.1050.0
version: 3.1050.0
'@azure/storage-blob':
specifier: 12.31.0
version: 12.31.0
'@clickhouse/client':
specifier: ^1.18.2
version: 1.18.2
'@docmost/pdf-inspector':
specifier: 1.9.4
version: 1.9.4
specifier: 1.9.6
version: 1.9.6
'@fastify/cookie':
specifier: ^11.0.2
version: 11.0.2
@@ -1114,6 +1117,61 @@ packages:
resolution: {integrity: sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==}
engines: {node: '>=18.0.0'}
'@azure/abort-controller@2.1.2':
resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==}
engines: {node: '>=18.0.0'}
'@azure/core-auth@1.10.1':
resolution: {integrity: sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==}
engines: {node: '>=20.0.0'}
'@azure/core-client@1.10.1':
resolution: {integrity: sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==}
engines: {node: '>=20.0.0'}
'@azure/core-http-compat@2.4.0':
resolution: {integrity: sha512-f1P96IB399YiN2ARYHP7EpZi3Bf3wH4SN2lGzrw7JVwm7bbsVYtf2iKSBwTywD2P62NOPZGHFSZi+6jjb75JuA==}
engines: {node: '>=20.0.0'}
peerDependencies:
'@azure/core-client': ^1.10.0
'@azure/core-rest-pipeline': ^1.22.0
'@azure/core-lro@2.7.2':
resolution: {integrity: sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==}
engines: {node: '>=18.0.0'}
'@azure/core-paging@1.6.2':
resolution: {integrity: sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==}
engines: {node: '>=18.0.0'}
'@azure/core-rest-pipeline@1.23.0':
resolution: {integrity: sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==}
engines: {node: '>=20.0.0'}
'@azure/core-tracing@1.3.1':
resolution: {integrity: sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==}
engines: {node: '>=20.0.0'}
'@azure/core-util@1.13.1':
resolution: {integrity: sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==}
engines: {node: '>=20.0.0'}
'@azure/core-xml@1.5.1':
resolution: {integrity: sha512-xcNRHqCoSp4AunOALEae6A8f3qATb83gSrm31Iqb01OzblvC3/W/bfXozcq78EzIdzZzuH1bZ2NvRR0TdX709w==}
engines: {node: '>=20.0.0'}
'@azure/logger@1.3.0':
resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==}
engines: {node: '>=20.0.0'}
'@azure/storage-blob@12.31.0':
resolution: {integrity: sha512-DBgNv10aCSxopt92DkTDD0o9xScXeBqPKGmR50FPZQaEcH4JLQ+GEOGEDv19V5BMkB7kxr+m4h6il/cCDPvmHg==}
engines: {node: '>=20.0.0'}
'@azure/storage-common@12.3.0':
resolution: {integrity: sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==}
engines: {node: '>=20.0.0'}
'@babel/code-frame@7.27.1':
resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==}
engines: {node: '>=6.9.0'}
@@ -1842,8 +1900,8 @@ packages:
resolution: {integrity: sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==}
engines: {node: '>=18'}
'@docmost/pdf-inspector@1.9.4':
resolution: {integrity: sha512-G5DNyDtLNxybTXWakqi7PuOEuSb/A2ZjDlv2WCkOkiHszPeILdrC+G0a4e4UP10yxvzuLfb23pJ5jy8fUSYZPw==}
'@docmost/pdf-inspector@1.9.6':
resolution: {integrity: sha512-8k8N8Mwu9xbpRC1jLcz4sFv88ev2oBnW56a/2WLbrOBkfXzyZV2Tml5PikUwEWT4cUXfYfk2dGnJpWQYgCESCQ==}
'@emnapi/core@1.8.1':
resolution: {integrity: sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==}
@@ -4965,6 +5023,10 @@ packages:
resolution: {integrity: sha512-YWnmJkXbofiz9KbnbbwuA2rpGkFPLbAIetcCNO6mJ8gdhdZ/v7WDXsoGFAJuM6ikUFKTlSQnjWnVO4ux+UzS6A==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
'@typespec/ts-http-runtime@0.3.5':
resolution: {integrity: sha512-yURCknZhvywvQItHMMmFSo+fq5arCUIyz/CVk7jD89MSai7dkaX8ufjCWp3NttLojoTVbcE72ri+be/TnEbMHw==}
engines: {node: '>=20.0.0'}
'@ucast/core@1.10.2':
resolution: {integrity: sha512-ons5CwXZ/51wrUPfoduC+cO7AS1/wRb0ybpQJ9RrssossDxVy4t49QxWoWgfBDvVKsz9VXzBk9z0wqTdZ+Cq8g==}
@@ -10854,6 +10916,119 @@ snapshots:
'@aws/lambda-invoke-store@0.2.3': {}
'@azure/abort-controller@2.1.2':
dependencies:
tslib: 2.8.1
'@azure/core-auth@1.10.1':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-util': 1.13.1
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/core-client@1.10.1':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-auth': 1.10.1
'@azure/core-rest-pipeline': 1.23.0
'@azure/core-tracing': 1.3.1
'@azure/core-util': 1.13.1
'@azure/logger': 1.3.0
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/core-http-compat@2.4.0(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0)':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-client': 1.10.1
'@azure/core-rest-pipeline': 1.23.0
'@azure/core-lro@2.7.2':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-util': 1.13.1
'@azure/logger': 1.3.0
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/core-paging@1.6.2':
dependencies:
tslib: 2.8.1
'@azure/core-rest-pipeline@1.23.0':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-auth': 1.10.1
'@azure/core-tracing': 1.3.1
'@azure/core-util': 1.13.1
'@azure/logger': 1.3.0
'@typespec/ts-http-runtime': 0.3.5
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/core-tracing@1.3.1':
dependencies:
tslib: 2.8.1
'@azure/core-util@1.13.1':
dependencies:
'@azure/abort-controller': 2.1.2
'@typespec/ts-http-runtime': 0.3.5
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/core-xml@1.5.1':
dependencies:
fast-xml-parser: 5.7.3
tslib: 2.8.1
'@azure/logger@1.3.0':
dependencies:
'@typespec/ts-http-runtime': 0.3.5
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/storage-blob@12.31.0':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-auth': 1.10.1
'@azure/core-client': 1.10.1
'@azure/core-http-compat': 2.4.0(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0)
'@azure/core-lro': 2.7.2
'@azure/core-paging': 1.6.2
'@azure/core-rest-pipeline': 1.23.0
'@azure/core-tracing': 1.3.1
'@azure/core-util': 1.13.1
'@azure/core-xml': 1.5.1
'@azure/logger': 1.3.0
'@azure/storage-common': 12.3.0(@azure/core-client@1.10.1)
events: 3.3.0
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@azure/storage-common@12.3.0(@azure/core-client@1.10.1)':
dependencies:
'@azure/abort-controller': 2.1.2
'@azure/core-auth': 1.10.1
'@azure/core-http-compat': 2.4.0(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0)
'@azure/core-rest-pipeline': 1.23.0
'@azure/core-tracing': 1.3.1
'@azure/core-util': 1.13.1
'@azure/logger': 1.3.0
events: 3.3.0
tslib: 2.8.1
transitivePeerDependencies:
- '@azure/core-client'
- supports-color
'@babel/code-frame@7.27.1':
dependencies:
'@babel/helper-validator-identifier': 7.28.5
@@ -11723,7 +11898,7 @@ snapshots:
'@csstools/css-tokenizer@3.0.3': {}
'@docmost/pdf-inspector@1.9.4': {}
'@docmost/pdf-inspector@1.9.6': {}
'@emnapi/core@1.8.1':
dependencies:
@@ -15106,6 +15281,14 @@ snapshots:
'@typescript-eslint/types': 8.57.1
eslint-visitor-keys: 5.0.1
'@typespec/ts-http-runtime@0.3.5':
dependencies:
http-proxy-agent: 7.0.2
https-proxy-agent: 7.0.6
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
'@ucast/core@1.10.2': {}
'@ucast/js@3.0.4':