sinvo/docmost
1
0
Fork 0
mirror of https://github.com/docmost/docmost.git synced 2026-05-15 05:04:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into perm-x

Browse Source
This commit is contained in:
Philipinho
2026-01-25 04:34:41 +00:00
parent 51928de956 1ca7d42203
commit 50fb8a1a52
114 changed files with 2981 additions and 2594 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/package.json
+2 -2
View File
@@ -76,6 +76,7 @@
"jsonwebtoken": "^9.0.3",
"kysely": "^0.28.2",
"kysely-migration-cli": "^0.4.2",
"kysely-postgres-js": "^3.0.0",
"ldapts": "^7.4.0",
"mammoth": "^1.11.0",
"mime-types": "^2.1.35",
@@ -89,9 +90,9 @@
"passport-google-oauth20": "^2.0.0",
"passport-jwt": "^4.0.1",
"pdfjs-dist": "^5.4.394",
"pg": "^8.16.3",
"pg-tsquery": "^8.4.2",
"pgvector": "^0.2.1",
"postgres": "^3.4.8",
"pino-http": "^11.0.0",
"pino-pretty": "^13.1.3",
"postmark": "^4.0.5",
@@ -121,7 +122,6 @@
"@types/nodemailer": "^6.4.17",
"@types/passport-google-oauth20": "^2.0.16",
"@types/passport-jwt": "^4.0.1",
"@types/pg": "^8.11.11",
"@types/supertest": "^6.0.2",
"@types/ws": "^8.5.14",
"@types/yauzl": "^2.10.3",
apps/server/src/collaboration/collaboration.gateway.ts
+2 -2
View File
@@ -26,7 +26,7 @@ export class CollaborationGateway {
) {
this.redisConfig = parseRedisUrl(this.environmentService.getRedisUrl());
this.hocuspocus = HocuspocusServer.configure({
this.hocuspocus = new Hocuspocus({
debounce: 10000,
maxDebounce: 45000,
unloadImmediately: false,
@@ -65,6 +65,6 @@ export class CollaborationGateway {
}
async destroy(): Promise<void> {
await this.hocuspocus.destroy();
//await this.hocuspocus.destroy();
}
}
apps/server/src/collaboration/collaboration.util.ts
+4 -4
View File
@@ -1,14 +1,12 @@
import { StarterKit } from '@tiptap/starter-kit';
import { TextAlign } from '@tiptap/extension-text-align';
import { TaskList } from '@tiptap/extension-task-list';
import { TaskItem } from '@tiptap/extension-task-item';
import { Underline } from '@tiptap/extension-underline';
import { Superscript } from '@tiptap/extension-superscript';
import SubScript from '@tiptap/extension-subscript';
import { Typography } from '@tiptap/extension-typography';
import { TextStyle } from '@tiptap/extension-text-style';
import { Color } from '@tiptap/extension-color';
import { Youtube } from '@tiptap/extension-youtube';
import { TaskList, TaskItem } from '@tiptap/extension-list';
import {
Heading,
Callout,
@@ -42,11 +40,14 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
// @tiptap/html library works best for generating prosemirror json state but not HTML
// see: https://github.com/ueberdosis/tiptap/issues/5352
// see:https://github.com/ueberdosis/tiptap/issues/4089
//import { generateJSON } from '@tiptap/html';
import { Node } from '@tiptap/pm/model';
export const tiptapExtensions = [
StarterKit.configure({
codeBlock: false,
link: false,
trailingNode: false,
heading: false,
}),
Heading,
@@ -59,7 +60,6 @@ export const tiptapExtensions = [
TaskItem.configure({
nested: true,
}),
Underline,
LinkExtension,
Superscript,
SubScript,
apps/server/src/collaboration/extensions/authentication.extension.ts
+2 -2
View File
@@ -83,7 +83,7 @@ export class AuthenticationExtension implements Extension {
}
if (!canEdit) {
data.connection.readOnly = true;
data.connectionConfig.readOnly = true;
this.logger.debug(
`User ${user.id} granted readonly access to restricted page: ${pageId}`,
);
@@ -91,7 +91,7 @@ export class AuthenticationExtension implements Extension {
} else {
// No restrictions - use space-level permissions
if (userSpaceRole === SpaceRole.READER) {
data.connection.readOnly = true;
data.connectionConfig.readOnly = true;
this.logger.debug(`User granted readonly access to page: ${pageId}`);
}
}
apps/server/src/common/helpers/utils.ts
+36
View File
@@ -2,6 +2,7 @@ import * as path from 'path';
import * as bcrypt from 'bcrypt';
import { sanitize } from 'sanitize-filename-ts';
import { FastifyRequest } from 'fastify';
import { Readable, Transform } from 'stream';
export const envPath = path.resolve(process.cwd(), '..', '..', '.env');
@@ -98,3 +99,38 @@ export function hasLicenseOrEE(opts: {
const { licenseKey, plan, isCloud } = opts;
return Boolean(licenseKey) || (isCloud && plan === 'business');
}
/**
* Normalizes a database URL for postgres.js compatibility.
* - Removes `sslmode=no-verify` (not supported by postgres.js), keeps other sslmode values
* - Removes `schema` parameter (has no effect via connection string)
* Note: If we don't strip them, the connection will fail
*/
export function normalizePostgresUrl(url: string): string {
const parsed = new URL(url);
const newParams = new URLSearchParams();
for (const [key, value] of parsed.searchParams) {
if (key === 'sslmode' && value === 'no-verify') continue;
if (key === 'schema') continue;
newParams.append(key, value);
}
parsed.search = newParams.toString();
return parsed.toString();
}
export function createByteCountingStream(source: Readable) {
let bytesRead = 0;
const stream = new Transform({
transform(chunk, encoding, callback) {
bytesRead += chunk.length;
callback(null, chunk);
},
});
source.pipe(stream);
source.on('error', (err) => stream.emit('error', err));
return { stream, getBytesRead: () => bytesRead };
}
apps/server/src/common/logger/pino.config.ts
+15 -8
View File
@@ -5,13 +5,14 @@ const CONTEXTS_TO_IGNORE = [
'InstanceLoader',
'RoutesResolver',
'RouterExplorer',
'LegacyRouteConverter',
'WebSocketsController',
];
export function createPinoConfig(): Params {
const isProduction = process.env.NODE_ENV === 'production';
const isDebugMode = process.env.DEBUG_MODE === 'true';
const logHttp = process.env.LOG_HTTP === 'true';
const isProduction = process.env.NODE_ENV?.toLowerCase() === 'production';
const isDebugMode = process.env.DEBUG_MODE?.toLowerCase() === 'true';
const logHttp = process.env.LOG_HTTP?.toLowerCase() === 'true';
const level = isProduction && !isDebugMode ? 'info' : 'debug';
@@ -32,14 +33,20 @@ export function createPinoConfig(): Params {
: undefined,
formatters: {
level: (label) => ({ level: label }),
log: (object: Record<string, unknown>) => {
},
hooks: {
logMethod(inputArgs, method) {
if (isProduction && !isDebugMode) {
const context = object['context'] as string | undefined;
if (context && CONTEXTS_TO_IGNORE.includes(context)) {
return { filtered: true };
for (const arg of inputArgs) {
if (typeof arg === 'object' && arg !== null && 'context' in arg) {
const context = (arg as Record<string, unknown>)['context'];
if (typeof context === 'string' && CONTEXTS_TO_IGNORE.includes(context)) {
return;
}
}
}
}
return object;
return method.apply(this, inputArgs);
},
},
serializers: {
apps/server/src/core/attachment/attachment.controller.ts
+8 -4
View File
@@ -175,7 +175,9 @@ export class AttachmentController {
await this.pageAccessService.validateCanView(page, user);
try {
const fileStream = await this.storageService.read(attachment.filePath);
const fileStream = await this.storageService.readStream(
attachment.filePath,
);
res.headers({
'Content-Type': attachment.mimeType,
'Cache-Control': 'private, max-age=3600',
@@ -235,7 +237,9 @@ export class AttachmentController {
}
try {
const fileStream = await this.storageService.read(attachment.filePath);
const fileStream = await this.storageService.readStream(
attachment.filePath,
);
res.headers({
'Content-Type': attachment.mimeType,
'Cache-Control': 'public, max-age=3600',
@@ -361,14 +365,14 @@ export class AttachmentController {
const filePath = `${getAttachmentFolderPath(attachmentType, workspace.id)}/${fileName}`;
try {
const fileStream = await this.storageService.read(filePath);
const fileStream = await this.storageService.readStream(filePath);
res.headers({
'Content-Type': getMimeType(filePath),
'Cache-Control': 'private, max-age=86400',
});
return res.send(fileStream);
} catch (err) {
// this.logger.error(err);
// this.logger.error(err);
throw new NotFoundException('File not found');
}
}
apps/server/src/core/attachment/attachment.utils.ts
+12 -3
View File
@@ -5,15 +5,17 @@ import { sanitizeFileName } from '../../common/helpers';
import * as sharp from 'sharp';
export interface PreparedFile {
buffer: Buffer;
buffer?: Buffer;
fileName: string;
fileSize: number;
fileExtension: string;
mimeType: string;
multiPartFile?: MultipartFile;
}
export async function prepareFile(
filePromise: Promise<MultipartFile>,
options: { skipBuffer?: boolean } = {},
): Promise<PreparedFile> {
const file = await filePromise;
@@ -22,10 +24,16 @@ export async function prepareFile(
}
try {
const buffer = await file.toBuffer();
let buffer: Buffer | undefined;
let fileSize = 0;
if (!options.skipBuffer) {
buffer = await file.toBuffer();
fileSize = buffer.length;
}
const sanitizedFilename = sanitizeFileName(file.filename);
const fileName = sanitizedFilename.slice(0, 255);
const fileSize = buffer.length;
const fileExtension = path.extname(file.filename).toLowerCase();
return {
@@ -34,6 +42,7 @@ export async function prepareFile(
fileSize,
fileExtension,
mimeType: file.mimetype,
multiPartFile: file,
};
} catch (error) {
throw error;
apps/server/src/core/attachment/services/attachment.service.ts
+19 -5
View File
@@ -4,6 +4,7 @@ import {
Logger,
NotFoundException,
} from '@nestjs/common';
import { Readable } from 'stream';
import { StorageService } from '../../../integrations/storage/storage.service';
import { MultipartFile } from '@fastify/multipart';
import {
@@ -26,6 +27,7 @@ import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
import { InjectQueue } from '@nestjs/bullmq';
import { QueueJob, QueueName } from '../../../integrations/queue/constants';
import { Queue } from 'bullmq';
import { createByteCountingStream } from '../../../common/helpers/utils';
@Injectable()
export class AttachmentService {
@@ -49,7 +51,9 @@ export class AttachmentService {
attachmentId?: string;
}) {
const { filePromise, pageId, spaceId, userId, workspaceId } = opts;
const preparedFile: PreparedFile = await prepareFile(filePromise);
const preparedFile: PreparedFile = await prepareFile(filePromise, {
skipBuffer: true,
});
let isUpdate = false;
let attachmentId = null;
@@ -81,7 +85,14 @@ export class AttachmentService {
const filePath = `${getAttachmentFolderPath(AttachmentType.File, workspaceId)}/${attachmentId}/${preparedFile.fileName}`;
await this.uploadToDrive(filePath, preparedFile.buffer);
const { stream, getBytesRead } = createByteCountingStream(
preparedFile.multiPartFile.file,
);
await this.uploadToDrive(filePath, stream);
// Update fileSize from the consumed stream
preparedFile.fileSize = getBytesRead();
let attachment: Attachment = null;
try {
@@ -142,7 +153,10 @@ export class AttachmentService {
const preparedFile: PreparedFile = await prepareFile(filePromise);
validateFileType(preparedFile.fileExtension, validImageExtensions);
const processedBuffer = await compressAndResizeIcon(preparedFile.buffer, type);
const processedBuffer = await compressAndResizeIcon(
preparedFile.buffer,
type,
);
preparedFile.buffer = processedBuffer;
preparedFile.fileSize = processedBuffer.length;
preparedFile.fileName = uuid4() + preparedFile.fileExtension;
@@ -232,9 +246,9 @@ export class AttachmentService {
}
}
async uploadToDrive(filePath: string, fileBuffer: any) {
async uploadToDrive(filePath: string, fileContent: Buffer | Readable) {
try {
await this.storageService.upload(filePath, fileBuffer);
await this.storageService.upload(filePath, fileContent);
} catch (err) {
this.logger.error('Error uploading file to drive:', err);
throw new BadRequestException('Error uploading file to drive');
apps/server/src/database/database.module.ts
+23 -20
View File
@@ -7,8 +7,7 @@ import {
} from '@nestjs/common';
import { InjectKysely, KyselyModule } from 'nestjs-kysely';
import { EnvironmentService } from '../integrations/environment/environment.service';
import { CamelCasePlugin, LogEvent, PostgresDialect, sql } from 'kysely';
import { Pool, types } from 'pg';
import { CamelCasePlugin, LogEvent, sql } from 'kysely';
import { GroupRepo } from '@docmost/db/repos/group/group.repo';
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
import { UserRepo } from '@docmost/db/repos/user/user.repo';
@@ -27,9 +26,9 @@ import { UserTokenRepo } from './repos/user-token/user-token.repo';
import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
import { ShareRepo } from '@docmost/db/repos/share/share.repo';
import { PageListener } from '@docmost/db/listeners/page.listener';
// https://github.com/brianc/node-postgres/issues/811
types.setTypeParser(types.builtins.INT8, (val) => Number(val));
import { PostgresJSDialect } from 'kysely-postgres-js';
import * as postgres from 'postgres';
import { normalizePostgresUrl } from '../common/helpers';
@Global()
@Module({
@@ -38,26 +37,30 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
imports: [],
inject: [EnvironmentService],
useFactory: (environmentService: EnvironmentService) => ({
dialect: new PostgresDialect({
pool: new Pool({
connectionString: environmentService.getDatabaseURL(),
max: environmentService.getDatabaseMaxPool(),
}).on('error', (err) => {
console.error('Database error:', err.message);
}),
dialect: new PostgresJSDialect({
postgres: postgres(
normalizePostgresUrl(environmentService.getDatabaseURL()),
{
max: environmentService.getDatabaseMaxPool(),
onnotice: () => {},
types: {
bigint: {
to: 20,
from: [20, 1700],
serialize: (value: number) => value.toString(),
parse: (value: string) => Number.parseInt(value),
},
},
},
),
}),
plugins: [new CamelCasePlugin()],
log: (event: LogEvent) => {
if (environmentService.getNodeEnv() !== 'development') return;
const logger = new Logger(DatabaseModule.name);
if (event.level) {
if (process.env.DEBUG_DB?.toLowerCase() === 'true') {
logger.debug(event.query.sql);
logger.debug('query time: ' + event.queryDurationMillis + ' ms');
//if (event.query.parameters.length > 0) {
// logger.debug('parameters: ' + event.query.parameters);
//}
}
if (process.env.DEBUG_DB?.toLowerCase() === 'true') {
logger.debug(event.query.sql);
logger.debug('query time: ' + event.queryDurationMillis + ' ms');
}
},
}),
apps/server/src/database/migrate.ts
+6 -12
View File
@@ -1,25 +1,19 @@
import * as path from 'path';
import { promises as fs } from 'fs';
import pg from 'pg';
import {
Kysely,
Migrator,
PostgresDialect,
FileMigrationProvider,
} from 'kysely';
import { Kysely, Migrator, FileMigrationProvider } from 'kysely';
import { run } from 'kysely-migration-cli';
import * as dotenv from 'dotenv';
import { envPath } from '../common/helpers/utils';
import { envPath, normalizePostgresUrl } from '../common/helpers';
import { PostgresJSDialect } from 'kysely-postgres-js';
import postgres from 'postgres';
dotenv.config({ path: envPath });
const migrationFolder = path.join(__dirname, './migrations');
const db = new Kysely<any>({
dialect: new PostgresDialect({
pool: new pg.Pool({
connectionString: process.env.DATABASE_URL,
}) as any,
dialect: new PostgresJSDialect({
postgres: postgres(normalizePostgresUrl(process.env.DATABASE_URL)),
}),
});
apps/server/src/ee
+1 -1
Submodule apps/server/src/ee updated: fce3e9e945...88e3d01f81
apps/server/src/integrations/export/export.controller.ts
+3 -3
View File
@@ -55,7 +55,7 @@ export class ExportController {
throw new ForbiddenException();
}
const zipFileBuffer = await this.exportService.exportPages(
const zipFileStream = await this.exportService.exportPages(
dto.pageId,
dto.format,
dto.includeAttachments,
@@ -71,7 +71,7 @@ export class ExportController {
'attachment; filename="' + encodeURIComponent(fileName) + '"',
});
res.send(zipFileBuffer);
res.send(zipFileStream);
}
@UseGuards(JwtAuthGuard)
@@ -102,6 +102,6 @@ export class ExportController {
'"',
});
res.send(exportFile.fileBuffer);
res.send(exportFile.fileStream);
}
}
apps/server/src/integrations/export/export.service.ts
+3 -3
View File
@@ -5,7 +5,6 @@ import {
NotFoundException,
} from '@nestjs/common';
import { jsonToHtml, jsonToNode } from '../../collaboration/collaboration.util';
import { turndown } from './turndown-utils';
import { ExportFormat } from './dto/export-dto';
import { Page } from '@docmost/db/types/entity.types';
import { InjectKysely } from 'nestjs-kysely';
@@ -32,6 +31,7 @@ import {
getAttachmentIds,
getProsemirrorContent,
} from '../../common/helpers/prosemirror/utils';
import { htmlToMarkdown } from '@docmost/editor-ext';
@Injectable()
export class ExportService {
@@ -85,7 +85,7 @@ export class ExportService {
/<colgroup[^>]*>[\s\S]*?<\/colgroup>/gim,
'',
);
return turndown(newPageHtml);
return htmlToMarkdown(newPageHtml);
}
return;
@@ -227,7 +227,7 @@ export class ExportService {
const fileName = `${space.name}-space-export.zip`;
return {
fileBuffer: zipFile,
fileStream: zipFile,
fileName,
};
}
apps/server/src/integrations/export/turndown-utils.ts
-165
View File
@@ -1,165 +0,0 @@
import * as TurndownService from '@joplin/turndown';
import * as TurndownPluginGfm from '@joplin/turndown-plugin-gfm';
import * as path from 'path';
export function turndown(html: string): string {
const turndownService = new TurndownService({
headingStyle: 'atx',
codeBlockStyle: 'fenced',
hr: '---',
bulletListMarker: '-',
});
const tables = TurndownPluginGfm.tables;
const strikethrough = TurndownPluginGfm.strikethrough;
const highlightedCodeBlock = TurndownPluginGfm.highlightedCodeBlock;
turndownService.use([
tables,
strikethrough,
highlightedCodeBlock,
taskList,
callout,
preserveDetail,
listParagraph,
mathInline,
mathBlock,
iframeEmbed,
video,
]);
return turndownService.turndown(html).replaceAll('<br>', ' ');
}
function listParagraph(turndownService: TurndownService) {
turndownService.addRule('paragraph', {
filter: ['p'],
replacement: (content: any, node: HTMLInputElement) => {
if (node.parentElement?.nodeName === 'LI') {
return content;
}
return `\n\n${content}\n\n`;
},
});
}
function callout(turndownService: TurndownService) {
turndownService.addRule('callout', {
filter: function (node: HTMLInputElement) {
return (
node.nodeName === 'DIV' && node.getAttribute('data-type') === 'callout'
);
},
replacement: function (content: any, node: HTMLInputElement) {
const calloutType = node.getAttribute('data-callout-type');
return `\n\n:::${calloutType}\n${content.trim()}\n:::\n\n`;
},
});
}
function taskList(turndownService: TurndownService) {
turndownService.addRule('taskListItem', {
filter: function (node: HTMLInputElement) {
return (
node.getAttribute('data-type') === 'taskItem' &&
node.parentNode.nodeName === 'UL'
);
},
replacement: function (content: any, node: HTMLInputElement) {
const checkbox = node.querySelector(
'input[type="checkbox"]',
) as HTMLInputElement;
const isChecked = checkbox.checked;
// Process content like regular list items
content = content
.replace(/^\n+/, '') // remove leading newlines
.replace(/\n+$/, '\n') // replace trailing newlines with just a single one
.replace(/\n/gm, '\n '); // indent nested content with 2 spaces
// Create the checkbox prefix
const prefix = `- ${isChecked ? '[x]' : '[ ]'} `;
return prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : '');
},
});
}
function preserveDetail(turndownService: TurndownService) {
turndownService.addRule('preserveDetail', {
filter: function (node: HTMLInputElement) {
return node.nodeName === 'DETAILS';
},
replacement: function (content: any, node: HTMLInputElement) {
const summary = node.querySelector(':scope > summary');
let detailSummary = '';
if (summary) {
detailSummary = `<summary>${turndownService.turndown(summary.innerHTML)}</summary>`;
}
const detailsContent = Array.from(node.childNodes)
.filter((child) => child.nodeName !== 'SUMMARY')
.map((child) =>
child.nodeType === 1
? turndownService.turndown((child as HTMLElement).outerHTML)
: child.textContent,
)
.join('');
return `\n<details>\n${detailSummary}\n\n${detailsContent}\n\n</details>\n`;
},
});
}
function mathInline(turndownService: TurndownService) {
turndownService.addRule('mathInline', {
filter: function (node: HTMLInputElement) {
return (
node.nodeName === 'SPAN' &&
node.getAttribute('data-type') === 'mathInline'
);
},
replacement: function (content: any, node: HTMLInputElement) {
return `$${content}$`;
},
});
}
function mathBlock(turndownService: TurndownService) {
turndownService.addRule('mathBlock', {
filter: function (node: HTMLInputElement) {
return (
node.nodeName === 'DIV' &&
node.getAttribute('data-type') === 'mathBlock'
);
},
replacement: function (content: any, node: HTMLInputElement) {
return `\n$$\n${content}\n$$\n`;
},
});
}
function iframeEmbed(turndownService: TurndownService) {
turndownService.addRule('iframeEmbed', {
filter: function (node: HTMLInputElement) {
return node.nodeName === 'IFRAME';
},
replacement: function (content: any, node: HTMLInputElement) {
const src = node.getAttribute('src');
return '[' + src + '](' + src + ')';
},
});
}
function video(turndownService: TurndownService) {
turndownService.addRule('video', {
filter: function (node: HTMLInputElement) {
return node.tagName === 'VIDEO';
},
replacement: function (content: any, node: HTMLInputElement) {
const src = node.getAttribute('src') || '';
const name = path.basename(src);
return '[' + name + '](' + src + ')';
},
});
}
apps/server/src/integrations/export/utils.ts
+15 -5
View File
@@ -1,4 +1,5 @@
import { jsonToNode } from 'src/collaboration/collaboration.util';
import { Logger } from '@nestjs/common';
import { ExportFormat } from './dto/export-dto';
import { Node } from '@tiptap/pm/model';
import { validate as isValidUUID } from 'uuid';
@@ -88,7 +89,7 @@ export function replaceInternalLinks(
// if link and text are same, use page title
if (markLink === node.text) {
//@ts-expect-error
node.text = getInternalLinkPageName(relativePath);
node.text = getInternalLinkPageName(relativePath, currentPagePath);
}
}
}
@@ -99,10 +100,19 @@ export function replaceInternalLinks(
return doc.toJSON();
}
export function getInternalLinkPageName(path: string): string {
return decodeURIComponent(
path?.split('/').pop().split('.').slice(0, -1).join('.'),
);
export function getInternalLinkPageName(path: string, currentFilePath?: string): string {
const name = path?.split('/').pop().split('.').slice(0, -1).join('.');
try {
return decodeURIComponent(name);
} catch (err) {
if (currentFilePath) {
Logger.warn(
`URI malformed in page ${currentFilePath}: ${name}. Falling back to raw name.`,
'ExportUtils',
);
}
return name;
}
}
export function extractPageSlugId(input: string): string {
apps/server/src/integrations/import/services/import.service.ts
+24 -10
View File
@@ -10,7 +10,11 @@ import {
} from '../../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { generateSlugId, sanitizeFileName } from '../../../common/helpers';
import {
generateSlugId,
sanitizeFileName,
createByteCountingStream,
} from '../../../common/helpers';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { TiptapTransformer } from '@hocuspocus/transformer';
import * as Y from 'yjs';
@@ -173,15 +177,24 @@ export class ImportService {
};
}
async getNewPagePosition(spaceId: string): Promise<string> {
const lastPage = await this.db
async getNewPagePosition(
spaceId: string,
parentPageId?: string,
): Promise<string> {
let query = this.db
.selectFrom('pages')
.select(['id', 'position'])
.where('spaceId', '=', spaceId)
.orderBy('position', (ob) => ob.collate('C').desc())
.limit(1)
.where('parentPageId', 'is', null)
.executeTakeFirst();
.limit(1);
if (parentPageId) {
query = query.where('parentPageId', '=', parentPageId);
} else {
query = query.where('parentPageId', 'is', null);
}
const lastPage = await query.executeTakeFirst();
if (lastPage) {
return generateJitteredKeyBetween(lastPage.position, null);
@@ -198,20 +211,21 @@ export class ImportService {
workspaceId: string,
) {
const file = await filePromise;
const fileBuffer = await file.toBuffer();
const fileExtension = path.extname(file.filename).toLowerCase();
const fileName = sanitizeFileName(
path.basename(file.filename, fileExtension),
);
const fileSize = fileBuffer.length;
const fileNameWithExt = fileName + fileExtension;
const fileTaskId = uuid7();
const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`;
// upload file
await this.storageService.upload(filePath, fileBuffer);
const { stream, getBytesRead } = createByteCountingStream(file.file);
await this.storageService.upload(filePath, stream);
const fileSize = getBytesRead();
const fileTask = await this.db
.insertInto('fileTasks')
apps/server/src/integrations/import/utils/import-formatter.ts
+12 -1
View File
@@ -1,4 +1,5 @@
import { getEmbedUrlAndProvider } from '@docmost/editor-ext';
import { Logger } from '@nestjs/common';
import * as path from 'path';
import { v7 } from 'uuid';
import { InsertableBacklink } from '@docmost/db/types/entity.types';
@@ -280,8 +281,18 @@ export async function rewriteInternalLinksToMentionHtml(
const $a = $(el);
const raw = $a.attr('href')!;
if (raw.startsWith('http') || raw.startsWith('/api/')) return;
let decodedRaw = raw;
try {
decodedRaw = decodeURIComponent(raw);
} catch (err) {
Logger.warn(
`URI malformed in page ${currentFilePath}: ${raw}. Falling back to raw path.`,
'ImportFormatter',
);
}
const resolved = normalize(
path.join(path.dirname(currentFilePath), decodeURIComponent(raw)),
path.join(path.dirname(currentFilePath), decodedRaw),
);
const meta = filePathToPageMetaMap.get(resolved);
if (!meta) return;
apps/server/src/integrations/import/utils/import.utils.ts
+8 -2
View File
@@ -1,3 +1,4 @@
import { Logger } from '@nestjs/common';
import { promises as fs } from 'fs';
import * as path from 'path';
@@ -30,8 +31,13 @@ export function resolveRelativeAttachmentPath(
pageDir: string,
attachmentCandidates: Map<string, string>,
): string | null {
const mainRel = decodeURIComponent(raw.replace(/^\.?\/+/, ''));
const fallback = path.normalize(path.join(pageDir, mainRel));
let mainRel = raw.replace(/^\.?\/+/, '');
try {
mainRel = decodeURIComponent(mainRel);
} catch (err) {
Logger.warn(`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, 'ImportUtils');
}
const fallback = path.normalize(path.join(pageDir, mainRel)).split(path.sep).join('/');
if (attachmentCandidates.has(mainRel)) {
return mainRel;
apps/server/src/integrations/storage/drivers/local.driver.ts
+9 -3
View File
@@ -20,9 +20,15 @@ export class LocalDriver implements StorageDriver {
return join(this.config.storagePath, filePath);
}
async upload(filePath: string, file: Buffer): Promise<void> {
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
try {
await fs.outputFile(this._fullPath(filePath), file);
const fullPath = this._fullPath(filePath);
if (file instanceof Buffer) {
await fs.outputFile(fullPath, file);
} else {
await fs.mkdir(dirname(fullPath), { recursive: true });
await pipeline(file, createWriteStream(fullPath));
}
} catch (err) {
throw new Error(`Failed to upload file: ${(err as Error).message}`);
}
@@ -42,7 +48,7 @@ export class LocalDriver implements StorageDriver {
try {
const fromFullPath = this._fullPath(fromFilePath);
const toFullPath = this._fullPath(toFilePath);
if (await this.exists(fromFilePath)) {
await fs.copy(fromFullPath, toFullPath);
}
apps/server/src/integrations/storage/drivers/s3.driver.ts
+10 -8
View File
@@ -23,19 +23,21 @@ export class S3Driver implements StorageDriver {
this.s3Client = new S3Client(config as any);
}
async upload(filePath: string, file: Buffer): Promise<void> {
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
try {
const contentType = getMimeType(filePath);
const command = new PutObjectCommand({
Bucket: this.config.bucket,
Key: filePath,
Body: file,
ContentType: contentType,
// ACL: "public-read",
const upload = new Upload({
client: this.s3Client,
params: {
Bucket: this.config.bucket,
Key: filePath,
Body: file,
ContentType: contentType,
},
});
await this.s3Client.send(command);
await upload.done();
} catch (err) {
throw new Error(`Failed to upload file: ${(err as Error).message}`);
}
apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import { Readable } from 'stream';
export interface StorageDriver {
upload(filePath: string, file: Buffer): Promise<void>;
upload(filePath: string, file: Buffer | Readable): Promise<void>;
uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise<void>;
apps/server/src/integrations/storage/storage.service.ts
+2 -2
View File
@@ -8,9 +8,9 @@ export class StorageService {
private readonly logger = new Logger(StorageService.name);
constructor(
@Inject(STORAGE_DRIVER_TOKEN) private storageDriver: StorageDriver,
) {}
) { }
async upload(filePath: string, fileContent: Buffer | any) {
async upload(filePath: string, fileContent: Buffer | Readable) {
await this.storageDriver.upload(filePath, fileContent);
this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
}