mirror of
https://github.com/docmost/docmost.git
synced 2026-05-22 01:32:55 +08:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 13a7f1372f | |||
| 4295ea09f6 | |||
| ed0501a864 | |||
| aa0c37bd68 | |||
| a5858bc470 |
+7
-1
@@ -10,7 +10,7 @@ JWT_TOKEN_EXPIRES_IN=30d
|
||||
DATABASE_URL="postgresql://postgres:password@localhost:5432/docmost?schema=public"
|
||||
REDIS_URL=redis://127.0.0.1:6379
|
||||
|
||||
# options: local | s3
|
||||
# options: local | s3 | azure
|
||||
STORAGE_DRIVER=local
|
||||
|
||||
# S3 driver config
|
||||
@@ -21,6 +21,12 @@ AWS_S3_BUCKET=
|
||||
AWS_S3_ENDPOINT=
|
||||
AWS_S3_FORCE_PATH_STYLE=
|
||||
|
||||
# Azure Blob Storage driver config
|
||||
STORAGE_DRIVER=azure
|
||||
AZURE_STORAGE_ACCOUNT_NAME=
|
||||
AZURE_STORAGE_ACCOUNT_KEY=
|
||||
AZURE_STORAGE_CONTAINER=
|
||||
|
||||
# default: 50mb
|
||||
FILE_UPLOAD_SIZE_LIMIT=
|
||||
|
||||
|
||||
@@ -33,18 +33,19 @@
|
||||
"@ai-sdk/google": "^3.0.52",
|
||||
"@ai-sdk/openai": "^3.0.47",
|
||||
"@ai-sdk/openai-compatible": "^2.0.37",
|
||||
"@aws-sdk/client-s3": "3.1040.0",
|
||||
"@aws-sdk/lib-storage": "3.1040.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.1040.0",
|
||||
"@aws-sdk/client-s3": "3.1050.0",
|
||||
"@aws-sdk/lib-storage": "3.1050.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.1050.0",
|
||||
"@azure/storage-blob": "12.31.0",
|
||||
"@clickhouse/client": "^1.18.2",
|
||||
"@docmost/pdf-inspector": "1.9.4",
|
||||
"@docmost/pdf-inspector": "1.9.6",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/multipart": "^10.0.0",
|
||||
"@fastify/static": "^9.1.3",
|
||||
"@keyv/redis": "^5.1.6",
|
||||
"@langchain/core": "1.1.46",
|
||||
"@langchain/textsplitters": "1.0.1",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"@nest-lab/throttler-storage-redis": "^1.2.0",
|
||||
"@nestjs-labs/nestjs-ioredis": "^11.0.4",
|
||||
"@nestjs/bullmq": "^11.0.4",
|
||||
@@ -68,7 +69,7 @@
|
||||
"ai-sdk-ollama": "^3.8.1",
|
||||
"bcrypt": "^6.0.0",
|
||||
"bowser": "^2.14.1",
|
||||
"bullmq": "^5.76.0",
|
||||
"bullmq": "^5.76.10",
|
||||
"cache-manager": "^7.2.8",
|
||||
"cheerio": "^1.2.0",
|
||||
"class-transformer": "^0.5.1",
|
||||
@@ -118,7 +119,7 @@
|
||||
"tseep": "^1.3.1",
|
||||
"typesense": "^3.0.5",
|
||||
"undici": "7.24.0",
|
||||
"ws": "^8.20.0",
|
||||
"ws": "^8.20.1",
|
||||
"yauzl": "^3.2.1",
|
||||
"zod": "^4.3.6"
|
||||
},
|
||||
@@ -163,7 +164,21 @@
|
||||
"rootDir": "src",
|
||||
"testRegex": ".*\\.spec\\.ts$",
|
||||
"transform": {
|
||||
"happy-dom.+\\.js$": ["babel-jest", { "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] }],
|
||||
"happy-dom.+\\.js$": [
|
||||
"babel-jest",
|
||||
{
|
||||
"presets": [
|
||||
[
|
||||
"@babel/preset-env",
|
||||
{
|
||||
"targets": {
|
||||
"node": "current"
|
||||
}
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"^.+\\.(t|j)s$": "ts-jest"
|
||||
},
|
||||
"transformIgnorePatterns": [
|
||||
|
||||
@@ -481,7 +481,7 @@ export class PageService {
|
||||
);
|
||||
|
||||
await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
|
||||
pageId: pageIdsToMove,
|
||||
pageIds: pageIdsToMove,
|
||||
workspaceId: rootPage.workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
+1
-1
Submodule apps/server/src/ee updated: b30e92f6a0...9e5f64d95d
@@ -122,6 +122,26 @@ export class EnvironmentService {
|
||||
return this.configService.get<string>('AWS_S3_URL');
|
||||
}
|
||||
|
||||
getAzureStorageAccountName(): string {
|
||||
return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_NAME');
|
||||
}
|
||||
|
||||
getAzureStorageContainer(): string {
|
||||
return this.configService.get<string>('AZURE_STORAGE_CONTAINER');
|
||||
}
|
||||
|
||||
getAzureStorageAccountKey(): string {
|
||||
return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_KEY');
|
||||
}
|
||||
|
||||
getAzureStorageEndpoint(): string {
|
||||
return this.configService.get<string>('AZURE_STORAGE_ENDPOINT');
|
||||
}
|
||||
|
||||
getAzureStorageUrl(): string {
|
||||
return this.configService.get<string>('AZURE_STORAGE_URL');
|
||||
}
|
||||
|
||||
getMailDriver(): string {
|
||||
return this.configService.get<string>('MAIL_DRIVER', 'log');
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ export class EnvironmentVariables {
|
||||
MAIL_DRIVER: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(['local', 's3'])
|
||||
@IsIn(['local', 's3', 'azure'])
|
||||
STORAGE_DRIVER: string;
|
||||
|
||||
@IsOptional()
|
||||
|
||||
@@ -66,8 +66,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
|
||||
@OnWorkerEvent('failed')
|
||||
async onFailed(job: Job) {
|
||||
const fileTaskId = job.data?.fileTaskId;
|
||||
this.logger.error(
|
||||
`Error processing ${job.name} job. File Task ID: ${job.data?.fileTaskId}. Reason: ${job.failedReason}`,
|
||||
fileTaskId
|
||||
? `Error processing ${job.name} job. File Task ID: ${fileTaskId}. Reason: ${job.failedReason}`
|
||||
: `Error processing ${job.name} job. Reason: ${job.failedReason}`,
|
||||
);
|
||||
|
||||
if (job.name === QueueJob.IMPORT_TASK) {
|
||||
@@ -79,8 +82,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
|
||||
@OnWorkerEvent('completed')
|
||||
async onCompleted(job: Job) {
|
||||
const fileTaskId = job.data?.fileTaskId;
|
||||
this.logger.log(
|
||||
`Completed ${job.name} job for File task ID ${job.data?.fileTaskId}`,
|
||||
fileTaskId
|
||||
? `Completed ${job.name} job for File task ID ${fileTaskId}`
|
||||
: `Completed ${job.name} job`,
|
||||
);
|
||||
|
||||
if (job.name === QueueJob.IMPORT_TASK) {
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
AzureStorageConfig,
|
||||
StorageDriver,
|
||||
StorageOption,
|
||||
} from '../interfaces';
|
||||
import {
|
||||
BlobSASPermissions,
|
||||
BlobServiceClient,
|
||||
BlockBlobClient,
|
||||
ContainerClient,
|
||||
generateBlobSASQueryParameters,
|
||||
SASProtocol,
|
||||
StorageSharedKeyCredential,
|
||||
} from '@azure/storage-blob';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { getMimeType } from '../../../common/helpers';
|
||||
|
||||
export class AzureDriver implements StorageDriver {
|
||||
private readonly config: AzureStorageConfig;
|
||||
private readonly blobServiceClient: BlobServiceClient;
|
||||
private readonly containerClient: ContainerClient;
|
||||
private readonly sharedKeyCredential: StorageSharedKeyCredential;
|
||||
private readonly accountUrl: string;
|
||||
|
||||
constructor(config: AzureStorageConfig) {
|
||||
this.config = config;
|
||||
|
||||
if (!config.accountName) {
|
||||
throw new Error('AzureDriver: accountName is required');
|
||||
}
|
||||
if (!config.container) {
|
||||
throw new Error('AzureDriver: container is required');
|
||||
}
|
||||
if (!config.accountKey) {
|
||||
throw new Error('AzureDriver: accountKey is required');
|
||||
}
|
||||
|
||||
this.accountUrl =
|
||||
config.endpoint ??
|
||||
`https://${config.accountName}.blob.core.windows.net`;
|
||||
|
||||
this.sharedKeyCredential = new StorageSharedKeyCredential(
|
||||
config.accountName,
|
||||
config.accountKey,
|
||||
);
|
||||
|
||||
this.blobServiceClient = this.createBlobServiceClient();
|
||||
this.containerClient = this.blobServiceClient.getContainerClient(
|
||||
config.container,
|
||||
);
|
||||
}
|
||||
|
||||
private blockBlob(filePath: string): BlockBlobClient {
|
||||
return this.containerClient.getBlockBlobClient(filePath);
|
||||
}
|
||||
|
||||
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
|
||||
const stream: Readable = Buffer.isBuffer(file) ? Readable.from(file) : file;
|
||||
await this.uploadStream(filePath, stream);
|
||||
}
|
||||
|
||||
async uploadStream(
|
||||
filePath: string,
|
||||
file: Readable,
|
||||
options?: { recreateClient?: boolean },
|
||||
): Promise<void> {
|
||||
const clientToUse = options?.recreateClient
|
||||
? this.createBlobServiceClient()
|
||||
.getContainerClient(this.config.container)
|
||||
.getBlockBlobClient(filePath)
|
||||
: this.blockBlob(filePath);
|
||||
|
||||
try {
|
||||
const contentType = getMimeType(filePath);
|
||||
await clientToUse.uploadStream(file, undefined, undefined, {
|
||||
blobHTTPHeaders: { blobContentType: contentType },
|
||||
});
|
||||
} catch (err) {
|
||||
Logger.error(err);
|
||||
throw new Error(`Failed to upload file: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async copy(fromFilePath: string, toFilePath: string): Promise<void> {
|
||||
try {
|
||||
if (!(await this.exists(fromFilePath))) {
|
||||
return;
|
||||
}
|
||||
const sourceUrl = await this.getSignedUrl(fromFilePath, 60);
|
||||
const dest = this.blockBlob(toFilePath);
|
||||
await dest.syncCopyFromURL(sourceUrl);
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to copy file: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async read(filePath: string): Promise<Buffer> {
|
||||
try {
|
||||
return await this.blockBlob(filePath).downloadToBuffer();
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`Failed to read file from Azure: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async readStream(filePath: string): Promise<Readable> {
|
||||
try {
|
||||
const response = await this.blockBlob(filePath).download();
|
||||
return response.readableStreamBody as Readable;
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`Failed to read file from Azure: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async readRangeStream(
|
||||
filePath: string,
|
||||
range: { start: number; end: number },
|
||||
): Promise<Readable> {
|
||||
try {
|
||||
const count = range.end - range.start + 1;
|
||||
const response = await this.blockBlob(filePath).download(
|
||||
range.start,
|
||||
count,
|
||||
);
|
||||
return response.readableStreamBody as Readable;
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`Failed to read file from Azure: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async exists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
return await this.blockBlob(filePath).exists();
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`Failed to check existence in Azure: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
getUrl(filePath: string): string {
|
||||
const base = this.config.baseUrl ?? this.accountUrl;
|
||||
return `${base}/${this.config.container}/${filePath}`;
|
||||
}
|
||||
|
||||
async getSignedUrl(filePath: string, expiresIn: number): Promise<string> {
|
||||
const expiresOn = new Date(Date.now() + expiresIn * 1000);
|
||||
const sas = generateBlobSASQueryParameters(
|
||||
{
|
||||
containerName: this.config.container,
|
||||
blobName: filePath,
|
||||
permissions: BlobSASPermissions.parse('r'),
|
||||
expiresOn,
|
||||
protocol: SASProtocol.HttpsAndHttp,
|
||||
},
|
||||
this.sharedKeyCredential,
|
||||
).toString();
|
||||
return `${this.accountUrl}/${this.config.container}/${filePath}?${sas}`;
|
||||
}
|
||||
|
||||
async delete(filePath: string): Promise<void> {
|
||||
try {
|
||||
await this.blockBlob(filePath).delete();
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`Error deleting file ${filePath} from Azure: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
getDriver(): BlobServiceClient {
|
||||
return this.blobServiceClient;
|
||||
}
|
||||
|
||||
getDriverName(): string {
|
||||
return StorageOption.AZURE;
|
||||
}
|
||||
|
||||
getConfig(): Record<string, any> {
|
||||
return this.config;
|
||||
}
|
||||
|
||||
private createBlobServiceClient(): BlobServiceClient {
|
||||
return new BlobServiceClient(this.accountUrl, this.sharedKeyCredential);
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
export { LocalDriver } from './local.driver';
|
||||
export { S3Driver } from './s3.driver';
|
||||
export { AzureDriver } from './azure.driver';
|
||||
|
||||
@@ -3,11 +3,13 @@ import { S3ClientConfig } from '@aws-sdk/client-s3';
|
||||
export enum StorageOption {
|
||||
LOCAL = 'local',
|
||||
S3 = 's3',
|
||||
AZURE = 'azure',
|
||||
}
|
||||
|
||||
export type StorageConfig =
|
||||
| { driver: StorageOption.LOCAL; config: LocalStorageConfig }
|
||||
| { driver: StorageOption.S3; config: S3StorageConfig };
|
||||
| { driver: StorageOption.S3; config: S3StorageConfig }
|
||||
| { driver: StorageOption.AZURE; config: AzureStorageConfig };
|
||||
|
||||
export interface LocalStorageConfig {
|
||||
storagePath: string;
|
||||
@@ -20,6 +22,14 @@ export interface S3StorageConfig
|
||||
baseUrl?: string; // Optional CDN URL for assets
|
||||
}
|
||||
|
||||
export interface AzureStorageConfig {
|
||||
accountName: string;
|
||||
container: string;
|
||||
accountKey: string;
|
||||
endpoint?: string;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
export interface StorageOptions {
|
||||
disk: StorageConfig;
|
||||
}
|
||||
|
||||
@@ -4,13 +4,14 @@ import {
|
||||
} from '../constants/storage.constants';
|
||||
import { EnvironmentService } from '../../environment/environment.service';
|
||||
import {
|
||||
AzureStorageConfig,
|
||||
LocalStorageConfig,
|
||||
S3StorageConfig,
|
||||
StorageConfig,
|
||||
StorageDriver,
|
||||
StorageOption,
|
||||
} from '../interfaces';
|
||||
import { LocalDriver, S3Driver } from '../drivers';
|
||||
import { AzureDriver, LocalDriver, S3Driver } from '../drivers';
|
||||
import * as process from 'node:process';
|
||||
import { LOCAL_STORAGE_PATH } from '../../../common/helpers';
|
||||
import path from 'path';
|
||||
@@ -21,6 +22,8 @@ function createStorageDriver(disk: StorageConfig): StorageDriver {
|
||||
return new LocalDriver(disk.config as LocalStorageConfig);
|
||||
case StorageOption.S3:
|
||||
return new S3Driver(disk.config as S3StorageConfig);
|
||||
case StorageOption.AZURE:
|
||||
return new AzureDriver(disk.config as AzureStorageConfig);
|
||||
default:
|
||||
throw new Error(`Unknown storage driver`);
|
||||
}
|
||||
@@ -70,6 +73,18 @@ export const storageDriverConfigProvider = {
|
||||
|
||||
return s3Config; }
|
||||
|
||||
case StorageOption.AZURE:
|
||||
return {
|
||||
driver,
|
||||
config: {
|
||||
accountName: environmentService.getAzureStorageAccountName(),
|
||||
container: environmentService.getAzureStorageContainer(),
|
||||
accountKey: environmentService.getAzureStorageAccountKey(),
|
||||
endpoint: environmentService.getAzureStorageEndpoint() || undefined,
|
||||
baseUrl: environmentService.getAzureStorageUrl() || undefined,
|
||||
},
|
||||
};
|
||||
|
||||
default:
|
||||
throw new Error(`Unknown storage driver: ${driver}`);
|
||||
}
|
||||
|
||||
+3
-3
@@ -101,7 +101,7 @@
|
||||
"prosemirror-changeset": "2.4.0",
|
||||
"y-prosemirror": "1.3.7",
|
||||
"glob": "13.0.6",
|
||||
"ws": "8.20.0",
|
||||
"ws": "8.20.1",
|
||||
"dompurify": "3.4.1",
|
||||
"tmp": "0.2.5",
|
||||
"hono": "4.12.18",
|
||||
@@ -127,13 +127,13 @@
|
||||
"yaml@>=1.0.0 <1.10.3": "1.10.3",
|
||||
"yaml@>=2.0.0 <2.8.3": "2.8.3",
|
||||
"path-to-regexp@^8": "8.4.0",
|
||||
"brace-expansion@^5": "5.0.5",
|
||||
"brace-expansion@^5": "5.0.6",
|
||||
"@xmldom/xmldom": "0.8.13",
|
||||
"handlebars": "4.7.9",
|
||||
"axios": "1.16.0",
|
||||
"langsmith": "0.7.0",
|
||||
"follow-redirects": "1.16.0",
|
||||
"protobufjs": "7.5.6",
|
||||
"protobufjs": "7.5.8",
|
||||
"ip-address": "10.1.1"
|
||||
},
|
||||
"neverBuiltDependencies": []
|
||||
|
||||
Generated
+448
-678
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user