Merge branch 'main' into feat/oauth

This commit is contained in:
Philipinho
2026-08-26 02:29:34 +01:00
97 changed files with 2332 additions and 651 deletions
+11 -10
View File
@@ -40,9 +40,9 @@
"@clickhouse/client": "1.18.2",
"@docmost/base-formula": "workspace:*",
"@docmost/pdf-inspector": "1.9.6",
"@fastify/cookie": "11.0.2",
"@fastify/multipart": "10.0.0",
"@fastify/static": "10.1.2",
"@fastify/cookie": "11.1.2",
"@fastify/multipart": "10.1.1",
"@fastify/static": "10.1.3",
"@jmondi/oauth2-server": "^4.3.7",
"@keyv/redis": "5.1.6",
"@langchain/core": "1.1.46",
@@ -50,23 +50,24 @@
"@modelcontextprotocol/sdk": "1.30.0",
"@nest-lab/throttler-storage-redis": "1.2.0",
"@nestjs-labs/nestjs-ioredis": "11.0.4",
"@nestjs/bullmq": "11.0.4",
"@nestjs/bullmq": "11.0.5",
"@nestjs/cache-manager": "3.1.3",
"@nestjs/common": "11.1.28",
"@nestjs/common": "11.2.1",
"@nestjs/config": "4.0.4",
"@nestjs/core": "11.1.27",
"@nestjs/core": "11.2.1",
"@nestjs/event-emitter": "3.1.0",
"@nestjs/jwt": "11.0.2",
"@nestjs/mapped-types": "2.1.1",
"@nestjs/passport": "11.0.5",
"@nestjs/platform-fastify": "11.1.28",
"@nestjs/platform-socket.io": "11.1.28",
"@nestjs/platform-fastify": "11.2.1",
"@nestjs/platform-socket.io": "11.2.1",
"@nestjs/schedule": "6.1.3",
"@nestjs/terminus": "11.1.1",
"@nestjs/throttler": "6.5.0",
"@nestjs/websockets": "11.1.28",
"@nestjs/websockets": "11.2.1",
"@node-saml/passport-saml": "5.1.0",
"@socket.io/redis-adapter": "8.3.0",
"@turbopuffer/turbopuffer": "^2.8.0",
"ai": "6.0.134",
"ai-sdk-ollama": "3.8.1",
"bcrypt": "6.0.0",
@@ -120,7 +121,7 @@
"tmp-promise": "3.0.3",
"typesense": "3.0.5",
"undici": "7.29.0",
"ws": "8.21.0",
"ws": "8.21.3",
"yauzl": "3.4.0",
"zod": "4.3.6"
},
+13 -2
View File
@@ -22,7 +22,8 @@ import { TelemetryModule } from './integrations/telemetry/telemetry.module';
import { RedisModule } from '@nestjs-labs/nestjs-ioredis';
import { RedisConfigService } from './integrations/redis/redis-config.service';
import { CacheModule } from '@nestjs/cache-manager';
import KeyvRedis from '@keyv/redis';
import KeyvRedis, { defaultReconnectStrategy } from '@keyv/redis';
import { parseRedisUrl } from './common/helpers';
import { LoggerModule } from './common/logger/logger.module';
import { ClsModule } from 'nestjs-cls';
import { NoopAuditModule } from './integrations/audit/audit.module';
@@ -62,10 +63,20 @@ try {
isGlobal: true,
useFactory: async (environmentService: EnvironmentService) => {
const redisUrl = environmentService.getRedisUrl();
const { family, tls } = parseRedisUrl(redisUrl);
return {
ttl: 5 * 1000,
stores: [new KeyvRedis(redisUrl)],
stores: [
new KeyvRedis({
url: redisUrl,
socket: {
family,
reconnectStrategy: defaultReconnectStrategy,
...tls,
},
}),
],
};
},
inject: [EnvironmentService],
@@ -66,6 +66,7 @@ export class CollaborationGateway {
password: this.redisConfig.password,
db: this.redisConfig.db,
family: this.redisConfig.family,
tls: this.redisConfig.tls,
retryStrategy: createRetryStrategy(),
}),
serverId: `collab-${os?.hostname()}-${nanoid(10)}`,
@@ -57,6 +57,7 @@ import {
JSONContent,
} from '@tiptap/core';
import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
import { collapseBlankLines } from '../common/helpers';
// @tiptap/html library works best for generating prosemirror json state but not HTML
// see: https://github.com/ueberdosis/tiptap/issues/5352
// see:https://github.com/ueberdosis/tiptap/issues/4089
@@ -146,7 +147,7 @@ export function htmlToJson(html: string) {
}
export function jsonToText(tiptapJson: JSONContent) {
return generateText(tiptapJson, tiptapExtensions);
return collapseBlankLines(generateText(tiptapJson, tiptapExtensions));
}
export function jsonToNode(tiptapJson: JSONContent) {
+1
View File
@@ -24,6 +24,7 @@ export const Feature = {
DOCX_EXPORT: 'export:docx',
BASES: 'bases',
OAUTH: 'oauth',
AI_CONTROLS: 'ai:controls',
} as const;
export type FeatureKey = (typeof Feature)[keyof typeof Feature];
+1
View File
@@ -1,4 +1,5 @@
export * from './utils';
export * from './text.utils';
export * from './nanoid.utils';
export * from './file.helper';
export * from './constants';
@@ -0,0 +1,14 @@
import { collapseBlankLines } from './text.utils';
describe('collapseBlankLines', () => {
it.each([
['a\n\n\n\nb', 'a\n\nb'],
['a\n\nb', 'a\n\nb'],
['a\nb', 'a\nb'],
['\n\n\n\na\n\n\n', '\n\na\n\n'],
['no newlines', 'no newlines'],
['', ''],
])('collapses %j to %j', (input, expected) => {
expect(collapseBlankLines(input)).toBe(expected);
});
});
@@ -0,0 +1,3 @@
export function collapseBlankLines(text: string): string {
return text.replace(/\n{2,}/g, '\n\n');
}
+12 -4
View File
@@ -30,13 +30,14 @@ export type RedisConfig = {
db: number;
password?: string;
family?: number;
tls?: { rejectUnauthorized?: boolean };
};
export function parseRedisUrl(redisUrl: string): RedisConfig {
// format - redis[s]://[[username][:password]@][host][:port][/db-number][?family=4|6]
// format - redis[s]://[[username][:password]@][host][:port][/db-number][?family=4|6][&rejectUnauthorized=false]
const url = new URL(redisUrl);
const { hostname, port, password, pathname, searchParams } = url;
const portInt = parseInt(port, 10);
const { hostname, port, password, pathname, protocol, searchParams } = url;
const portInt = port ? parseInt(port, 10) : 6379;
let db: number = 0;
// extract db value if present
@@ -54,7 +55,14 @@ export function parseRedisUrl(redisUrl: string): RedisConfig {
family = parseInt(familyParam, 10);
}
return { host: hostname, port: portInt, password, db, family };
const tls =
protocol === 'rediss:'
? searchParams.get('rejectUnauthorized') === 'false'
? { rejectUnauthorized: false }
: {}
: undefined;
return { host: hostname, port: portInt, password: password || undefined, db, family, tls };
}
export function createRetryStrategy() {
@@ -116,8 +116,8 @@ export class AttachmentService {
});
}
// Only index PDFs and DOCX files
if (['.pdf', '.docx'].includes(attachment.fileExt.toLowerCase())) {
// Only index PDF, DOCX and TXT files
if (['.pdf', '.docx', '.txt'].includes(attachment.fileExt.toLowerCase())) {
await this.attachmentQueue.add(
QueueJob.ATTACHMENT_INDEX_CONTENT,
{
@@ -56,7 +56,7 @@ export class CommentController {
@AuthWorkspace() workspace: Workspace,
) {
const page = await this.pageRepo.findById(createCommentDto.pageId);
if (!page || page.deletedAt) {
if (!page || page.workspaceId !== workspace.id || page.deletedAt) {
throw new NotFoundException('Page not found');
}
@@ -92,9 +92,10 @@ export class CommentController {
@Body()
pagination: PaginationOptions,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
const page = await this.pageRepo.findById(input.pageId);
if (!page) {
if (!page || page.workspaceId !== workspace.id || page.deletedAt) {
throw new NotFoundException('Page not found');
}
@@ -105,14 +106,18 @@ export class CommentController {
@HttpCode(HttpStatus.OK)
@Post('info')
async findOne(@Body() input: CommentIdDto, @AuthUser() user: User) {
async findOne(
@Body() input: CommentIdDto,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
const comment = await this.commentRepo.findById(input.commentId);
if (!comment) {
throw new NotFoundException('Comment not found');
}
const page = await this.pageRepo.findById(comment.pageId);
if (!page) {
if (!page || page.workspaceId !== workspace.id || page.deletedAt) {
throw new NotFoundException('Page not found');
}
@@ -134,7 +139,7 @@ export class CommentController {
}
const page = await this.pageRepo.findById(comment.pageId);
if (!page) {
if (!page || page.workspaceId !== workspace.id || page.deletedAt) {
throw new NotFoundException('Page not found');
}
@@ -152,7 +157,7 @@ export class CommentController {
}
const page = await this.pageRepo.findById(comment.pageId);
if (!page) {
if (!page || page.workspaceId !== workspace.id || page.deletedAt) {
throw new NotFoundException('Page not found');
}
@@ -7,12 +7,15 @@ import { executeTx } from '@docmost/db/utils';
import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
import { normalizeLabelName } from './utils';
import { EventEmitter2 } from "@nestjs/event-emitter";
import { EventName } from "src/common/events/event.contants";
@Injectable()
export class LabelService {
constructor(
private readonly labelRepo: LabelRepo,
private readonly pagePermissionRepo: PagePermissionRepo,
private readonly eventEmitter: EventEmitter2,
@InjectKysely() private readonly db: KyselyDB,
) {}
@@ -34,6 +37,12 @@ export class LabelService {
attached.push(label);
}
});
this.eventEmitter.emit(EventName.PAGE_UPDATED, {
pageIds: [pageId],
workspaceId: workspaceId,
});
return attached;
}
@@ -64,6 +73,11 @@ export class LabelService {
await this.labelRepo.deleteLabel(labelId, workspaceId, trx);
}
});
this.eventEmitter.emit(EventName.PAGE_UPDATED, {
pageIds: [pageId],
workspaceId: workspaceId,
});
}
async getPageLabels(pageId: string, pagination: PaginationOptions) {
@@ -496,10 +496,21 @@ export class PageService {
},
);
await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
pageIds: pageIdsToMove,
workspaceId: rootPage.workspaceId,
});
await this.aiQueue.add(
QueueJob.PAGE_MOVED_TO_SPACE,
{
pageIds: pageIdsToMove,
spaceId,
workspaceId: rootPage.workspaceId,
},
{
attempts: 2,
backoff: {
type: 'fixed',
delay: 2 * 60 * 1000,
},
},
);
}
});
+12 -2
View File
@@ -1,4 +1,5 @@
import {
IsArray,
IsBoolean,
IsNotEmpty,
IsNumber,
@@ -8,9 +9,9 @@ import {
} from 'class-validator';
export class SearchDTO {
@IsNotEmpty()
@IsOptional()
@IsString()
query: string;
query?: string;
@IsOptional()
@IsUUID()
@@ -24,6 +25,15 @@ export class SearchDTO {
@IsUUID()
creatorId?: string;
@IsOptional()
@IsArray()
@IsUUID('all', { each: true })
labelIds?: string[];
@IsOptional()
@IsBoolean()
titleOnly?: boolean;
@IsOptional()
@IsNumber()
limit?: number;
+57 -16
View File
@@ -29,12 +29,36 @@ export class SearchService {
workspaceId: string;
},
): Promise<{ items: SearchResponseDto[] }> {
const { query } = searchParams;
const query = searchParams.query?.trim() ?? '';
const labelIds = [...new Set(searchParams.labelIds ?? [])];
// selected filters (labels, creator) are browsable without a query
const browseByFilters =
query.length < 1 &&
(labelIds.length > 0 || Boolean(searchParams.creatorId));
if (query.length < 1) {
if (query.length < 1 && !browseByFilters) {
return { items: [] };
}
const searchQuery = tsquery(query.trim() + '*');
const searchQuery = tsquery(query + '*');
const titleOnly = searchParams.titleOnly === true;
const titleQuery = query;
// escape LIKE wildcards; ranking keeps the raw query
const titleLikeQuery = query.replace(/[\\%_]/g, '\\$&');
const rankColumn = browseByFilters
? sql<number>`0`.as('rank')
: titleOnly
? sql<number>`word_similarity(lower(${titleQuery}), lower(pages.title))`.as(
'rank',
)
: sql<number>`ts_rank(tsv, to_tsquery('english', f_unaccent(${searchQuery})))`.as(
'rank',
);
const highlightColumn = browseByFilters || titleOnly
? sql<string>`''`.as('highlight')
: sql<string>`ts_headline('english', text_content, to_tsquery('english', f_unaccent(${searchQuery})),'MinWords=9, MaxWords=10, MaxFragments=3')`.as(
'highlight',
);
let queryResults = this.db
.selectFrom('pages')
@@ -47,23 +71,41 @@ export class SearchService {
'creatorId',
'createdAt',
'updatedAt',
sql<number>`ts_rank(tsv, to_tsquery('english', f_unaccent(${searchQuery})))`.as(
'rank',
),
sql<string>`ts_headline('english', text_content, to_tsquery('english', f_unaccent(${searchQuery})),'MinWords=9, MaxWords=10, MaxFragments=3')`.as(
'highlight',
),
rankColumn,
highlightColumn,
])
.where(
'tsv',
'@@',
sql<string>`to_tsquery('english', f_unaccent(${searchQuery}))`,
.$if(!browseByFilters && !titleOnly, (qb) =>
qb.where(
'tsv',
'@@',
sql<string>`to_tsquery('english', f_unaccent(${searchQuery}))`,
),
)
.$if(!browseByFilters && titleOnly, (qb) =>
qb.where((eb) =>
eb(
sql`lower(pages.title)`,
'like',
sql`lower(${`%${titleLikeQuery}%`})`,
),
),
)
.$if(Boolean(searchParams.creatorId), (qb) =>
qb.where('creatorId', '=', searchParams.creatorId),
)
.$if(labelIds?.length > 0, (qb) =>
qb.where(
'id',
'in',
this.db
.selectFrom('pageLabels')
.select('pageId')
.where('labelId', 'in', labelIds),
),
)
.where('deletedAt', 'is', null)
.orderBy('rank', 'desc')
.$if(browseByFilters, (qb) => qb.orderBy('updatedAt', 'desc'))
.$if(!browseByFilters, (qb) => qb.orderBy('rank', 'desc'))
.limit(searchParams.limit || 25)
.offset(searchParams.offset || 0);
@@ -71,8 +113,7 @@ export class SearchService {
queryResults = queryResults.select((eb) => this.pageRepo.withSpace(eb));
}
if (searchParams.spaceId) {
// search by spaceId
if (searchParams.spaceId && opts.userId) {
queryResults = queryResults.where('spaceId', '=', searchParams.spaceId);
} else if (opts.userId && !searchParams.spaceId) {
// only search spaces the user is a member of
+9 -4
View File
@@ -46,8 +46,9 @@ export class ShareService {
throw new NotFoundException('Share not found');
}
const isRestricted =
await this.pagePermissionRepo.hasRestrictedAncestor(share.pageId);
const isRestricted = await this.pagePermissionRepo.hasRestrictedAncestor(
share.pageId,
);
if (isRestricted) {
throw new NotFoundException('Share not found');
}
@@ -110,6 +111,9 @@ export class ShareService {
}
async getSharedPage(dto: ShareInfoDto, workspaceId: string) {
//TODO: we should resolve the page from the share id
if (!dto.pageId) throw new NotFoundException('Shared page not found');
const share = await this.getShareForPage(dto.pageId, workspaceId);
if (!share) {
@@ -126,8 +130,9 @@ export class ShareService {
}
// Block access to restricted pages
const isRestricted =
await this.pagePermissionRepo.hasRestrictedAncestor(page.id);
const isRestricted = await this.pagePermissionRepo.hasRestrictedAncestor(
page.id,
);
if (isRestricted) {
throw new NotFoundException('Shared page not found');
}
@@ -339,15 +339,25 @@ export class SpaceMemberService {
return;
}
if (spaceMember.role === SpaceRole.ADMIN) {
await this.validateLastAdmin(dto.spaceId);
}
await executeTx(this.db, async (trx) => {
await trx
.selectFrom('spaces')
.select('id')
.where('id', '=', dto.spaceId)
.forUpdate()
.executeTakeFirst();
await this.spaceMemberRepo.updateSpaceMember(
{ role: dto.role },
spaceMember.id,
dto.spaceId,
);
if (spaceMember.role === SpaceRole.ADMIN) {
await this.validateLastAdmin(dto.spaceId, trx);
}
await this.spaceMemberRepo.updateSpaceMember(
{ role: dto.role },
spaceMember.id,
dto.spaceId,
trx,
);
});
this.auditService.log({
event: AuditEvent.SPACE_MEMBER_ROLE_CHANGED,
@@ -368,10 +378,14 @@ export class SpaceMemberService {
});
}
async validateLastAdmin(spaceId: string): Promise<void> {
async validateLastAdmin(
spaceId: string,
trx?: KyselyTransaction,
): Promise<void> {
const spaceOwnerCount = await this.spaceMemberRepo.roleCountBySpaceId(
SpaceRole.ADMIN,
spaceId,
trx,
);
if (spaceOwnerCount === 1) {
throw new BadRequestException(
@@ -92,4 +92,12 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
@ValidateNested({ each: true })
@Type(() => TrustedOAuthClientDto)
trustedOauthClients?: TrustedOAuthClient[];
@IsOptional()
@IsBoolean()
aiChatReadOnly: boolean;
@IsOptional()
@IsBoolean()
aiChatWorkspaceKnowledgeOnly: boolean;
}
@@ -344,7 +344,9 @@ export class WorkspaceService {
typeof updateWorkspaceDto.allowMemberTemplates !== 'undefined' ||
typeof updateWorkspaceDto.isScimEnabled !== 'undefined' ||
typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined' ||
typeof updateWorkspaceDto.trustedOauthClients !== 'undefined'
typeof updateWorkspaceDto.trustedOauthClients !== 'undefined' ||
typeof updateWorkspaceDto.aiChatReadOnly !== 'undefined' ||
typeof updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly !== 'undefined'
) {
const ws = await this.db
.selectFrom('workspaces')
@@ -396,6 +398,21 @@ export class WorkspaceService {
}
}
if (
typeof updateWorkspaceDto.aiChatReadOnly !== 'undefined' ||
typeof updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly !== 'undefined'
) {
if (
!this.licenseCheckService.hasFeature(
ws.licenseKey,
Feature.AI_CONTROLS,
ws.plan,
)
) {
throw new ForbiddenException('This feature requires a valid license');
}
}
if (
typeof updateWorkspaceDto.disablePublicSharing !== 'undefined' ||
typeof updateWorkspaceDto.trashRetentionDays !== 'undefined' ||
@@ -418,7 +435,10 @@ export class WorkspaceService {
}
}
if (updateWorkspaceDto.aiSearch) {
if (
updateWorkspaceDto.aiSearch &&
this.environmentService.getAiVectorDriver() !== 'turbopuffer'
) {
const tableExists = await isPageEmbeddingsTableExists(this.db);
if (!tableExists) {
throw new BadRequestException(
@@ -535,6 +555,34 @@ export class WorkspaceService {
);
}
if (typeof updateWorkspaceDto.aiChatReadOnly !== 'undefined') {
const prev = settingsBefore?.ai?.chatReadOnly ?? false;
if (prev !== updateWorkspaceDto.aiChatReadOnly) {
before.aiChatReadOnly = prev;
after.aiChatReadOnly = updateWorkspaceDto.aiChatReadOnly;
}
await this.workspaceRepo.updateAiSettings(
workspaceId,
'chatReadOnly',
updateWorkspaceDto.aiChatReadOnly,
trx,
);
}
if (typeof updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly !== 'undefined') {
const prev = settingsBefore?.ai?.chatWorkspaceKnowledgeOnly ?? false;
if (prev !== updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly) {
before.aiChatWorkspaceKnowledgeOnly = prev;
after.aiChatWorkspaceKnowledgeOnly = updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly;
}
await this.workspaceRepo.updateAiSettings(
workspaceId,
'chatWorkspaceKnowledgeOnly',
updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly,
trx,
);
}
if (typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined') {
const prev = settingsBefore?.spaces?.allowPersonal ?? false;
if (prev !== updateWorkspaceDto.allowPersonalSpaces) {
@@ -572,6 +620,8 @@ export class WorkspaceService {
delete updateWorkspaceDto.aiChat;
delete updateWorkspaceDto.allowPersonalSpaces;
delete updateWorkspaceDto.defaultPageEditMode;
delete updateWorkspaceDto.aiChatReadOnly;
delete updateWorkspaceDto.aiChatWorkspaceKnowledgeOnly;
await this.workspaceRepo.updateWorkspace(
updateWorkspaceDto,
@@ -8,6 +8,7 @@ import { EnvironmentService } from '../../integrations/environment/environment.s
export class SpaceEvent {
spaceId: string;
workspaceId: string;
}
@Injectable()
@@ -22,12 +23,12 @@ export class SpaceListener {
@OnEvent(EventName.SPACE_DELETED)
async handleSpaceDeleted(event: SpaceEvent) {
const { spaceId } = event;
const { spaceId, workspaceId } = event;
if (this.isTypesense()) {
await this.searchQueue.add(QueueJob.SPACE_DELETED, { spaceId });
}
await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId });
await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId, workspaceId });
}
isTypesense(): boolean {
@@ -0,0 +1,17 @@
import { type Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await sql`CREATE INDEX IF NOT EXISTS pages_title_trgm_idx ON pages USING gin (lower(title) gin_trgm_ops)`.execute(
db,
);
// separators normalized to spaces so space-typed queries match How_to_export.pdf
await sql`CREATE INDEX IF NOT EXISTS attachments_file_name_trgm_idx ON attachments USING gin (lower(translate(file_name, '_.-', ' ')) gin_trgm_ops)`.execute(
db,
);
}
export async function down(db: Kysely<any>): Promise<void> {
await sql`DROP INDEX IF EXISTS attachments_file_name_trgm_idx`.execute(db);
await sql`DROP INDEX IF EXISTS pages_title_trgm_idx`.execute(db);
}
@@ -46,8 +46,10 @@ export class SpaceMemberRepo {
updatableSpaceMember: UpdatableSpaceMember,
spaceMemberId: string,
spaceId: string,
trx?: KyselyTransaction,
): Promise<void> {
await this.db
const db = dbOrTx(this.db, trx);
await db
.updateTable('spaceMembers')
.set(updatableSpaceMember)
.where('id', '=', spaceMemberId)
@@ -92,8 +94,13 @@ export class SpaceMemberRepo {
.execute();
}
async roleCountBySpaceId(role: string, spaceId: string): Promise<number> {
const { count } = await this.db
async roleCountBySpaceId(
role: string,
spaceId: string,
trx?: KyselyTransaction,
): Promise<number> {
const db = dbOrTx(this.db, trx);
const { count } = await db
.selectFrom('spaceMembers')
.select((eb) => eb.fn.count('role').as('count'))
.where('role', '=', role)
@@ -230,6 +230,7 @@ export class SpaceRepo {
this.eventEmitter.emit(EventName.SPACE_DELETED, {
spaceId,
workspaceId,
});
}
}
@@ -212,6 +212,24 @@ export class WorkspaceRepo {
.executeTakeFirst();
}
async updateAiEmbeddingFingerprint(
workspaceId: string,
fingerprint: { driver: string; model: string; dimensions: number },
trx?: KyselyTransaction,
) {
const db = dbOrTx(this.db, trx);
return db
.updateTable('workspaces')
.set({
settings: sql`COALESCE(settings, '{}'::jsonb)
|| jsonb_build_object('ai', COALESCE(settings->'ai', '{}'::jsonb)
|| jsonb_build_object('embedding', ${JSON.stringify(fingerprint)}::text::jsonb))`,
updatedAt: new Date(),
})
.where('id', '=', workspaceId)
.execute();
}
async updateSharingSettings(
workspaceId: string,
prefKey: string,
@@ -310,6 +310,31 @@ export class EnvironmentService {
return val === 'true';
}
getAiVectorDriver(): string {
return this.configService
.get<string>('AI_VECTOR_DRIVER', 'pgvector')
.toLowerCase();
}
getTurbopufferApiKey(): string {
return this.configService.get<string>('TURBOPUFFER_API_KEY');
}
getTurbopufferRegion(): string {
return this.configService.get<string>('TURBOPUFFER_REGION');
}
getTurbopufferBaseUrl(): string {
return this.configService.get<string>('TURBOPUFFER_BASE_URL');
}
getTurbopufferNamespacePrefix(): string {
return this.configService.get<string>(
'TURBOPUFFER_NAMESPACE_PREFIX',
'docmost',
);
}
getOpenAiApiKey(): string {
return this.configService.get<string>('OPENAI_API_KEY');
}
@@ -5,6 +5,7 @@ import {
IsOptional,
IsString,
IsUrl,
Matches,
MinLength,
ValidateIf,
validateSync,
@@ -108,6 +109,41 @@ export class EnvironmentVariables {
@IsString()
AI_DRIVER: string;
@IsOptional()
@ValidateIf((obj) => obj.AI_VECTOR_DRIVER)
@IsIn(['pgvector', 'turbopuffer'])
@IsString()
AI_VECTOR_DRIVER: string;
@ValidateIf((obj) => obj.AI_VECTOR_DRIVER === 'turbopuffer')
@IsNotEmpty()
@IsString()
TURBOPUFFER_API_KEY: string;
@ValidateIf(
(obj) =>
obj.AI_VECTOR_DRIVER === 'turbopuffer' && !obj.TURBOPUFFER_BASE_URL,
)
@IsNotEmpty({
message:
'TURBOPUFFER_REGION is required when AI_VECTOR_DRIVER is turbopuffer, unless TURBOPUFFER_BASE_URL is set',
})
@IsString()
TURBOPUFFER_REGION: string;
@IsOptional()
@ValidateIf((obj) => obj.TURBOPUFFER_BASE_URL != '' && obj.TURBOPUFFER_BASE_URL != null)
@IsUrl({ protocols: ['http', 'https'], require_tld: false })
TURBOPUFFER_BASE_URL: string;
@IsOptional()
@IsString()
@Matches(/^[A-Za-z0-9\-_.]{1,90}$/, {
message:
'TURBOPUFFER_NAMESPACE_PREFIX may only contain letters, digits, dot, dash, underscore (max 90 chars)',
})
TURBOPUFFER_NAMESPACE_PREFIX: string;
@IsOptional()
@IsString()
AI_EMBEDDING_MODEL: string;
@@ -5,6 +5,7 @@ import {
import { Injectable, Logger } from '@nestjs/common';
import { EnvironmentService } from '../environment/environment.service';
import { Redis } from 'ioredis';
import { parseRedisUrl } from '../../common/helpers';
@Injectable()
export class RedisHealthIndicator {
@@ -19,8 +20,10 @@ export class RedisHealthIndicator {
const indicator = this.healthIndicatorService.check(key);
try {
const redis = new Redis(this.environmentService.getRedisUrl(), {
const redisUrl = this.environmentService.getRedisUrl();
const redis = new Redis(redisUrl, {
maxRetriesPerRequest: 15,
tls: parseRedisUrl(redisUrl).tls,
});
await redis.ping();
@@ -61,6 +61,7 @@ export enum QueueJob {
WORKSPACE_DELETED = 'workspace-deleted',
WORKSPACE_CREATE_EMBEDDINGS = 'workspace-create-embeddings',
WORKSPACE_DELETE_EMBEDDINGS = 'workspace-delete-embeddings',
WORKSPACE_RESET_EMBEDDINGS = 'workspace-reset-embeddings',
GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',
@@ -18,6 +18,7 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
password: redisConfig.password,
db: redisConfig.db,
family: redisConfig.family,
tls: redisConfig.tls,
retryStrategy: createRetryStrategy(),
},
defaultJobOptions: {
@@ -19,6 +19,7 @@ export class RedisConfigService implements RedisOptionsFactory {
password: redisConfig.password,
db: redisConfig.db,
family: redisConfig.family,
tls: redisConfig.tls,
retryStrategy: createRetryStrategy(),
},
};
@@ -49,6 +49,10 @@ export class StaticModule implements OnModuleInit {
: undefined,
POSTHOG_HOST: this.environmentService.getPostHogHost(),
POSTHOG_KEY: this.environmentService.getPostHogKey(),
AI_VECTOR_DRIVER:
this.environmentService.getAiVectorDriver() === 'turbopuffer'
? 'turbopuffer'
: undefined,
};
const windowScriptContent = `<script>window.CONFIG=${JSON.stringify(configString)};</script>`;
@@ -67,6 +71,15 @@ export class StaticModule implements OnModuleInit {
await app.register(fastifyStatic, {
root: clientDistPath,
wildcard: false,
setHeaders: (reply: any, pathName: string) => {
// Vite content-hashes everything under /assets, so they can be cached forever
if (/[\\/]assets[\\/]/.test(pathName)) {
reply.header(
'Cache-Control',
'public, max-age=31536000, immutable',
);
}
},
});
app.get(RENDER_PATH, (req: any, res: any) => {
@@ -3,7 +3,7 @@ import { ThrottlerModule } from '@nestjs/throttler';
import { ThrottlerStorageRedisService } from '@nest-lab/throttler-storage-redis';
import { EnvironmentService } from '../environment/environment.service';
import { EnvironmentModule } from '../environment/environment.module';
import { parseRedisUrl } from '../../common/helpers';
import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
import {
AUTH_THROTTLER,
AI_CHAT_THROTTLER,
@@ -36,6 +36,8 @@ import Redis from 'ioredis';
password: redisConfig.password,
db: redisConfig.db,
family: redisConfig.family,
tls: redisConfig.tls,
retryStrategy: createRetryStrategy(),
keyPrefix: 'throttle:',
}),
),
@@ -17,6 +17,7 @@ export class WsRedisIoAdapter extends IoAdapter {
const options: RedisOptions = {
family: this.redisConfig.family,
tls: this.redisConfig.tls,
retryStrategy: createRetryStrategy(),
};