Compare commits

...

5 Commits

Author SHA1 Message Date
Philipinho 13a7f1372f fix: update pdf-inspector package 2026-05-21 13:44:11 +01:00
Philip Okugbe 4295ea09f6 feat(storage): add Azure Blob Storage driver (#2222) 2026-05-21 12:18:58 +01:00
Philipinho ed0501a864 fix passing wrong object 2026-05-20 19:09:22 +01:00
Philipinho aa0c37bd68 sync 2026-05-20 18:41:23 +01:00
Philip Okugbe a5858bc470 fix: update packages (#2221) 2026-05-20 18:30:15 +01:00
13 changed files with 732 additions and 697 deletions
+7 -1
View File
@@ -10,7 +10,7 @@ JWT_TOKEN_EXPIRES_IN=30d
DATABASE_URL="postgresql://postgres:password@localhost:5432/docmost?schema=public"
REDIS_URL=redis://127.0.0.1:6379
# options: local | s3
# options: local | s3 | azure
STORAGE_DRIVER=local
# S3 driver config
@@ -21,6 +21,12 @@ AWS_S3_BUCKET=
AWS_S3_ENDPOINT=
AWS_S3_FORCE_PATH_STYLE=
# Azure Blob Storage driver config
STORAGE_DRIVER=azure
AZURE_STORAGE_ACCOUNT_NAME=
AZURE_STORAGE_ACCOUNT_KEY=
AZURE_STORAGE_CONTAINER=
# default: 50mb
FILE_UPLOAD_SIZE_LIMIT=
+23 -8
View File
@@ -33,18 +33,19 @@
"@ai-sdk/google": "^3.0.52",
"@ai-sdk/openai": "^3.0.47",
"@ai-sdk/openai-compatible": "^2.0.37",
"@aws-sdk/client-s3": "3.1040.0",
"@aws-sdk/lib-storage": "3.1040.0",
"@aws-sdk/s3-request-presigner": "3.1040.0",
"@aws-sdk/client-s3": "3.1050.0",
"@aws-sdk/lib-storage": "3.1050.0",
"@aws-sdk/s3-request-presigner": "3.1050.0",
"@azure/storage-blob": "12.31.0",
"@clickhouse/client": "^1.18.2",
"@docmost/pdf-inspector": "1.9.4",
"@docmost/pdf-inspector": "1.9.6",
"@fastify/cookie": "^11.0.2",
"@fastify/multipart": "^10.0.0",
"@fastify/static": "^9.1.3",
"@keyv/redis": "^5.1.6",
"@langchain/core": "1.1.46",
"@langchain/textsplitters": "1.0.1",
"@modelcontextprotocol/sdk": "^1.29.0",
"@modelcontextprotocol/sdk": "1.29.0",
"@nest-lab/throttler-storage-redis": "^1.2.0",
"@nestjs-labs/nestjs-ioredis": "^11.0.4",
"@nestjs/bullmq": "^11.0.4",
@@ -68,7 +69,7 @@
"ai-sdk-ollama": "^3.8.1",
"bcrypt": "^6.0.0",
"bowser": "^2.14.1",
"bullmq": "^5.76.0",
"bullmq": "^5.76.10",
"cache-manager": "^7.2.8",
"cheerio": "^1.2.0",
"class-transformer": "^0.5.1",
@@ -118,7 +119,7 @@
"tseep": "^1.3.1",
"typesense": "^3.0.5",
"undici": "7.24.0",
"ws": "^8.20.0",
"ws": "^8.20.1",
"yauzl": "^3.2.1",
"zod": "^4.3.6"
},
@@ -163,7 +164,21 @@
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"happy-dom.+\\.js$": ["babel-jest", { "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] }],
"happy-dom.+\\.js$": [
"babel-jest",
{
"presets": [
[
"@babel/preset-env",
{
"targets": {
"node": "current"
}
}
]
]
}
],
"^.+\\.(t|j)s$": "ts-jest"
},
"transformIgnorePatterns": [
@@ -481,7 +481,7 @@ export class PageService {
);
await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
pageId: pageIdsToMove,
pageIds: pageIdsToMove,
workspaceId: rootPage.workspaceId,
});
}
@@ -122,6 +122,26 @@ export class EnvironmentService {
return this.configService.get<string>('AWS_S3_URL');
}
getAzureStorageAccountName(): string {
return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_NAME');
}
getAzureStorageContainer(): string {
return this.configService.get<string>('AZURE_STORAGE_CONTAINER');
}
getAzureStorageAccountKey(): string {
return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_KEY');
}
getAzureStorageEndpoint(): string {
return this.configService.get<string>('AZURE_STORAGE_ENDPOINT');
}
getAzureStorageUrl(): string {
return this.configService.get<string>('AZURE_STORAGE_URL');
}
getMailDriver(): string {
return this.configService.get<string>('MAIL_DRIVER', 'log');
}
@@ -49,7 +49,7 @@ export class EnvironmentVariables {
MAIL_DRIVER: string;
@IsOptional()
@IsIn(['local', 's3'])
@IsIn(['local', 's3', 'azure'])
STORAGE_DRIVER: string;
@IsOptional()
@@ -66,8 +66,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
@OnWorkerEvent('failed')
async onFailed(job: Job) {
const fileTaskId = job.data?.fileTaskId;
this.logger.error(
`Error processing ${job.name} job. File Task ID: ${job.data?.fileTaskId}. Reason: ${job.failedReason}`,
fileTaskId
? `Error processing ${job.name} job. File Task ID: ${fileTaskId}. Reason: ${job.failedReason}`
: `Error processing ${job.name} job. Reason: ${job.failedReason}`,
);
if (job.name === QueueJob.IMPORT_TASK) {
@@ -79,8 +82,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
@OnWorkerEvent('completed')
async onCompleted(job: Job) {
const fileTaskId = job.data?.fileTaskId;
this.logger.log(
`Completed ${job.name} job for File task ID ${job.data?.fileTaskId}`,
fileTaskId
? `Completed ${job.name} job for File task ID ${fileTaskId}`
: `Completed ${job.name} job`,
);
if (job.name === QueueJob.IMPORT_TASK) {
@@ -0,0 +1,192 @@
import { Readable } from 'stream';
import {
AzureStorageConfig,
StorageDriver,
StorageOption,
} from '../interfaces';
import {
BlobSASPermissions,
BlobServiceClient,
BlockBlobClient,
ContainerClient,
generateBlobSASQueryParameters,
SASProtocol,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { Logger } from '@nestjs/common';
import { getMimeType } from '../../../common/helpers';
export class AzureDriver implements StorageDriver {
private readonly config: AzureStorageConfig;
private readonly blobServiceClient: BlobServiceClient;
private readonly containerClient: ContainerClient;
private readonly sharedKeyCredential: StorageSharedKeyCredential;
private readonly accountUrl: string;
constructor(config: AzureStorageConfig) {
this.config = config;
if (!config.accountName) {
throw new Error('AzureDriver: accountName is required');
}
if (!config.container) {
throw new Error('AzureDriver: container is required');
}
if (!config.accountKey) {
throw new Error('AzureDriver: accountKey is required');
}
this.accountUrl =
config.endpoint ??
`https://${config.accountName}.blob.core.windows.net`;
this.sharedKeyCredential = new StorageSharedKeyCredential(
config.accountName,
config.accountKey,
);
this.blobServiceClient = this.createBlobServiceClient();
this.containerClient = this.blobServiceClient.getContainerClient(
config.container,
);
}
private blockBlob(filePath: string): BlockBlobClient {
return this.containerClient.getBlockBlobClient(filePath);
}
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
const stream: Readable = Buffer.isBuffer(file) ? Readable.from(file) : file;
await this.uploadStream(filePath, stream);
}
async uploadStream(
filePath: string,
file: Readable,
options?: { recreateClient?: boolean },
): Promise<void> {
const clientToUse = options?.recreateClient
? this.createBlobServiceClient()
.getContainerClient(this.config.container)
.getBlockBlobClient(filePath)
: this.blockBlob(filePath);
try {
const contentType = getMimeType(filePath);
await clientToUse.uploadStream(file, undefined, undefined, {
blobHTTPHeaders: { blobContentType: contentType },
});
} catch (err) {
Logger.error(err);
throw new Error(`Failed to upload file: ${(err as Error).message}`);
}
}
async copy(fromFilePath: string, toFilePath: string): Promise<void> {
try {
if (!(await this.exists(fromFilePath))) {
return;
}
const sourceUrl = await this.getSignedUrl(fromFilePath, 60);
const dest = this.blockBlob(toFilePath);
await dest.syncCopyFromURL(sourceUrl);
} catch (err) {
throw new Error(`Failed to copy file: ${(err as Error).message}`);
}
}
async read(filePath: string): Promise<Buffer> {
try {
return await this.blockBlob(filePath).downloadToBuffer();
} catch (err) {
throw new Error(
`Failed to read file from Azure: ${(err as Error).message}`,
);
}
}
async readStream(filePath: string): Promise<Readable> {
try {
const response = await this.blockBlob(filePath).download();
return response.readableStreamBody as Readable;
} catch (err) {
throw new Error(
`Failed to read file from Azure: ${(err as Error).message}`,
);
}
}
async readRangeStream(
filePath: string,
range: { start: number; end: number },
): Promise<Readable> {
try {
const count = range.end - range.start + 1;
const response = await this.blockBlob(filePath).download(
range.start,
count,
);
return response.readableStreamBody as Readable;
} catch (err) {
throw new Error(
`Failed to read file from Azure: ${(err as Error).message}`,
);
}
}
async exists(filePath: string): Promise<boolean> {
try {
return await this.blockBlob(filePath).exists();
} catch (err) {
throw new Error(
`Failed to check existence in Azure: ${(err as Error).message}`,
);
}
}
getUrl(filePath: string): string {
const base = this.config.baseUrl ?? this.accountUrl;
return `${base}/${this.config.container}/${filePath}`;
}
async getSignedUrl(filePath: string, expiresIn: number): Promise<string> {
const expiresOn = new Date(Date.now() + expiresIn * 1000);
const sas = generateBlobSASQueryParameters(
{
containerName: this.config.container,
blobName: filePath,
permissions: BlobSASPermissions.parse('r'),
expiresOn,
protocol: SASProtocol.HttpsAndHttp,
},
this.sharedKeyCredential,
).toString();
return `${this.accountUrl}/${this.config.container}/${filePath}?${sas}`;
}
async delete(filePath: string): Promise<void> {
try {
await this.blockBlob(filePath).delete();
} catch (err) {
throw new Error(
`Error deleting file ${filePath} from Azure: ${(err as Error).message}`,
);
}
}
getDriver(): BlobServiceClient {
return this.blobServiceClient;
}
getDriverName(): string {
return StorageOption.AZURE;
}
getConfig(): Record<string, any> {
return this.config;
}
private createBlobServiceClient(): BlobServiceClient {
return new BlobServiceClient(this.accountUrl, this.sharedKeyCredential);
}
}
@@ -1,2 +1,3 @@
export { LocalDriver } from './local.driver';
export { S3Driver } from './s3.driver';
export { AzureDriver } from './azure.driver';
@@ -3,11 +3,13 @@ import { S3ClientConfig } from '@aws-sdk/client-s3';
export enum StorageOption {
LOCAL = 'local',
S3 = 's3',
AZURE = 'azure',
}
export type StorageConfig =
| { driver: StorageOption.LOCAL; config: LocalStorageConfig }
| { driver: StorageOption.S3; config: S3StorageConfig };
| { driver: StorageOption.S3; config: S3StorageConfig }
| { driver: StorageOption.AZURE; config: AzureStorageConfig };
export interface LocalStorageConfig {
storagePath: string;
@@ -20,6 +22,14 @@ export interface S3StorageConfig
baseUrl?: string; // Optional CDN URL for assets
}
export interface AzureStorageConfig {
accountName: string;
container: string;
accountKey: string;
endpoint?: string;
baseUrl?: string;
}
export interface StorageOptions {
disk: StorageConfig;
}
@@ -4,13 +4,14 @@ import {
} from '../constants/storage.constants';
import { EnvironmentService } from '../../environment/environment.service';
import {
AzureStorageConfig,
LocalStorageConfig,
S3StorageConfig,
StorageConfig,
StorageDriver,
StorageOption,
} from '../interfaces';
import { LocalDriver, S3Driver } from '../drivers';
import { AzureDriver, LocalDriver, S3Driver } from '../drivers';
import * as process from 'node:process';
import { LOCAL_STORAGE_PATH } from '../../../common/helpers';
import path from 'path';
@@ -21,6 +22,8 @@ function createStorageDriver(disk: StorageConfig): StorageDriver {
return new LocalDriver(disk.config as LocalStorageConfig);
case StorageOption.S3:
return new S3Driver(disk.config as S3StorageConfig);
case StorageOption.AZURE:
return new AzureDriver(disk.config as AzureStorageConfig);
default:
throw new Error(`Unknown storage driver`);
}
@@ -70,6 +73,18 @@ export const storageDriverConfigProvider = {
return s3Config; }
case StorageOption.AZURE:
return {
driver,
config: {
accountName: environmentService.getAzureStorageAccountName(),
container: environmentService.getAzureStorageContainer(),
accountKey: environmentService.getAzureStorageAccountKey(),
endpoint: environmentService.getAzureStorageEndpoint() || undefined,
baseUrl: environmentService.getAzureStorageUrl() || undefined,
},
};
default:
throw new Error(`Unknown storage driver: ${driver}`);
}
+3 -3
View File
@@ -101,7 +101,7 @@
"prosemirror-changeset": "2.4.0",
"y-prosemirror": "1.3.7",
"glob": "13.0.6",
"ws": "8.20.0",
"ws": "8.20.1",
"dompurify": "3.4.1",
"tmp": "0.2.5",
"hono": "4.12.18",
@@ -127,13 +127,13 @@
"yaml@>=1.0.0 <1.10.3": "1.10.3",
"yaml@>=2.0.0 <2.8.3": "2.8.3",
"path-to-regexp@^8": "8.4.0",
"brace-expansion@^5": "5.0.5",
"brace-expansion@^5": "5.0.6",
"@xmldom/xmldom": "0.8.13",
"handlebars": "4.7.9",
"axios": "1.16.0",
"langsmith": "0.7.0",
"follow-redirects": "1.16.0",
"protobufjs": "7.5.6",
"protobufjs": "7.5.8",
"ip-address": "10.1.1"
},
"neverBuiltDependencies": []
+448 -678
View File
File diff suppressed because it is too large Load Diff