Compare commits

..
Author SHA1 Message Date
Salihu efc113de3e convert only bare links to embeds 2026-08-24 23:12:59 +01:00
Salihu cd9c166927 Merge pull request #2417 from docmost/fix/checkbox-filtering
fix(ee): checkbox filtering
2026-08-23 21:21:02 +01:00
Philip Okugbe 549cf7c005 fix: skip pgvector table check when enabling AI search on turbopuffer (#2416) 2026-08-22 13:20:35 +01:00
Philip Okugbe e14f499f3d fix: pass tls to redis in throttle module (#2415) 2026-08-22 03:46:00 +01:00
Salihu b814bd0f12 fix: checkbox filtering 2026-08-21 19:29:10 +01:00
Salihu b86abd3d40 Revert "fix: checkbox filtering"
This reverts commit 3b858746e3.
2026-08-21 19:23:48 +01:00
Salihu 3b858746e3 fix: checkbox filtering 2026-08-21 19:21:14 +01:00
Philipinho 66b424a3b8 fix: preserve hash in vimeo embed url 2026-08-20 22:25:45 +01:00
Philipinho ab43031375 fix race issue 2026-08-19 21:46:01 +01:00
Michael LohrandPhilipinho 8c2c49ea6d fix: handle empty password, missing port, and TLS in Redis URL parsing (#2021)
* fix: handle empty password, missing port, and TLS in Redis URL parsing
---------

Co-authored-by: Philipinho <16838612+Philipinho@users.noreply.github.com>
2026-08-19 21:05:44 +01:00
Philip Okugbe 8913d20aa0 chore: package updates (#2412) 2026-08-19 19:59:00 +01:00
Philip Okugbe 232beda471 feat(ee): turbopuffer ai vector search driver (#2402)
* feat(ai): add AI_VECTOR_DRIVER and turbopuffer configuration

* feat(ai): carry workspace and target space ids on vector lifecycle events

* feat(ai): add vector driver interface and turbopuffer request helpers

* feat(ai): add pgvector driver behind the vector driver interface

* refactor(ai): route vector reads and writes through the vector driver

* feat(ai): add turbopuffer vector driver

* feat(ai): rebuild turbopuffer namespaces when the embedding model changes

* feat(ai): warm the vector namespace cache on session start

* fix(ai): harden turbopuffer misconfiguration and reset failure paths

* fix(ai): collapse blank-line runs in extracted page text

* fix(ai): skip full re-embed when ai search is re-enabled within the delete grace window

* sync

* fix(ai): store real embedding dimensions instead of serialized vector length

* fix(ai): filter search hits by the page's current space at query time

* fix(ai): retry the page moved-to-space vector patch job

* sync

* feat(ai): pre-warm the vector namespace

* fix(ai): pass AI_VECTOR_DRIVER through the client build config
2026-08-17 21:18:16 +01:00
18 changed files with 289 additions and 324 deletions
+1 -1
View File
@@ -52,7 +52,7 @@
"mantine-form-zod-resolver": "1.3.0", "mantine-form-zod-resolver": "1.3.0",
"mermaid": "11.16.1", "mermaid": "11.16.1",
"mitt": "3.0.1", "mitt": "3.0.1",
"nanoid": "3.3.17", "nanoid": "3.3.18",
"posthog-js": "1.391.2", "posthog-js": "1.391.2",
"react": "19.2.7", "react": "19.2.7",
"react-clear-modal": "^2.0.18", "react-clear-modal": "^2.0.18",
+10 -10
View File
@@ -40,30 +40,30 @@
"@clickhouse/client": "1.18.2", "@clickhouse/client": "1.18.2",
"@docmost/base-formula": "workspace:*", "@docmost/base-formula": "workspace:*",
"@docmost/pdf-inspector": "1.9.6", "@docmost/pdf-inspector": "1.9.6",
"@fastify/cookie": "11.0.2", "@fastify/cookie": "11.1.2",
"@fastify/multipart": "10.0.0", "@fastify/multipart": "10.1.1",
"@fastify/static": "10.1.2", "@fastify/static": "10.1.3",
"@keyv/redis": "5.1.6", "@keyv/redis": "5.1.6",
"@langchain/core": "1.1.46", "@langchain/core": "1.1.46",
"@langchain/textsplitters": "1.0.1", "@langchain/textsplitters": "1.0.1",
"@modelcontextprotocol/sdk": "1.30.0", "@modelcontextprotocol/sdk": "1.30.0",
"@nest-lab/throttler-storage-redis": "1.2.0", "@nest-lab/throttler-storage-redis": "1.2.0",
"@nestjs-labs/nestjs-ioredis": "11.0.4", "@nestjs-labs/nestjs-ioredis": "11.0.4",
"@nestjs/bullmq": "11.0.4", "@nestjs/bullmq": "11.0.5",
"@nestjs/cache-manager": "3.1.3", "@nestjs/cache-manager": "3.1.3",
"@nestjs/common": "11.1.28", "@nestjs/common": "11.2.1",
"@nestjs/config": "4.0.4", "@nestjs/config": "4.0.4",
"@nestjs/core": "11.1.27", "@nestjs/core": "11.2.1",
"@nestjs/event-emitter": "3.1.0", "@nestjs/event-emitter": "3.1.0",
"@nestjs/jwt": "11.0.2", "@nestjs/jwt": "11.0.2",
"@nestjs/mapped-types": "2.1.1", "@nestjs/mapped-types": "2.1.1",
"@nestjs/passport": "11.0.5", "@nestjs/passport": "11.0.5",
"@nestjs/platform-fastify": "11.1.28", "@nestjs/platform-fastify": "11.2.1",
"@nestjs/platform-socket.io": "11.1.28", "@nestjs/platform-socket.io": "11.2.1",
"@nestjs/schedule": "6.1.3", "@nestjs/schedule": "6.1.3",
"@nestjs/terminus": "11.1.1", "@nestjs/terminus": "11.1.1",
"@nestjs/throttler": "6.5.0", "@nestjs/throttler": "6.5.0",
"@nestjs/websockets": "11.1.28", "@nestjs/websockets": "11.2.1",
"@node-saml/passport-saml": "5.1.0", "@node-saml/passport-saml": "5.1.0",
"@socket.io/redis-adapter": "8.3.0", "@socket.io/redis-adapter": "8.3.0",
"@turbopuffer/turbopuffer": "^2.8.0", "@turbopuffer/turbopuffer": "^2.8.0",
@@ -120,7 +120,7 @@
"tmp-promise": "3.0.3", "tmp-promise": "3.0.3",
"typesense": "3.0.5", "typesense": "3.0.5",
"undici": "7.29.0", "undici": "7.29.0",
"ws": "8.21.0", "ws": "8.21.3",
"yauzl": "3.4.0", "yauzl": "3.4.0",
"zod": "4.3.6" "zod": "4.3.6"
}, },
+13 -2
View File
@@ -22,7 +22,8 @@ import { TelemetryModule } from './integrations/telemetry/telemetry.module';
import { RedisModule } from '@nestjs-labs/nestjs-ioredis'; import { RedisModule } from '@nestjs-labs/nestjs-ioredis';
import { RedisConfigService } from './integrations/redis/redis-config.service'; import { RedisConfigService } from './integrations/redis/redis-config.service';
import { CacheModule } from '@nestjs/cache-manager'; import { CacheModule } from '@nestjs/cache-manager';
import KeyvRedis from '@keyv/redis'; import KeyvRedis, { defaultReconnectStrategy } from '@keyv/redis';
import { parseRedisUrl } from './common/helpers';
import { LoggerModule } from './common/logger/logger.module'; import { LoggerModule } from './common/logger/logger.module';
import { ClsModule } from 'nestjs-cls'; import { ClsModule } from 'nestjs-cls';
import { NoopAuditModule } from './integrations/audit/audit.module'; import { NoopAuditModule } from './integrations/audit/audit.module';
@@ -62,10 +63,20 @@ try {
isGlobal: true, isGlobal: true,
useFactory: async (environmentService: EnvironmentService) => { useFactory: async (environmentService: EnvironmentService) => {
const redisUrl = environmentService.getRedisUrl(); const redisUrl = environmentService.getRedisUrl();
const { family, tls } = parseRedisUrl(redisUrl);
return { return {
ttl: 5 * 1000, ttl: 5 * 1000,
stores: [new KeyvRedis(redisUrl)], stores: [
new KeyvRedis({
url: redisUrl,
socket: {
family,
reconnectStrategy: defaultReconnectStrategy,
...tls,
},
}),
],
}; };
}, },
inject: [EnvironmentService], inject: [EnvironmentService],
@@ -66,6 +66,7 @@ export class CollaborationGateway {
password: this.redisConfig.password, password: this.redisConfig.password,
db: this.redisConfig.db, db: this.redisConfig.db,
family: this.redisConfig.family, family: this.redisConfig.family,
tls: this.redisConfig.tls,
retryStrategy: createRetryStrategy(), retryStrategy: createRetryStrategy(),
}), }),
serverId: `collab-${os?.hostname()}-${nanoid(10)}`, serverId: `collab-${os?.hostname()}-${nanoid(10)}`,
+12 -4
View File
@@ -30,13 +30,14 @@ export type RedisConfig = {
db: number; db: number;
password?: string; password?: string;
family?: number; family?: number;
tls?: { rejectUnauthorized?: boolean };
}; };
export function parseRedisUrl(redisUrl: string): RedisConfig { export function parseRedisUrl(redisUrl: string): RedisConfig {
// format - redis[s]://[[username][:password]@][host][:port][/db-number][?family=4|6] // format - redis[s]://[[username][:password]@][host][:port][/db-number][?family=4|6][&rejectUnauthorized=false]
const url = new URL(redisUrl); const url = new URL(redisUrl);
const { hostname, port, password, pathname, searchParams } = url; const { hostname, port, password, pathname, protocol, searchParams } = url;
const portInt = parseInt(port, 10); const portInt = port ? parseInt(port, 10) : 6379;
let db: number = 0; let db: number = 0;
// extract db value if present // extract db value if present
@@ -54,7 +55,14 @@ export function parseRedisUrl(redisUrl: string): RedisConfig {
family = parseInt(familyParam, 10); family = parseInt(familyParam, 10);
} }
return { host: hostname, port: portInt, password, db, family }; const tls =
protocol === 'rediss:'
? searchParams.get('rejectUnauthorized') === 'false'
? { rejectUnauthorized: false }
: {}
: undefined;
return { host: hostname, port: portInt, password: password || undefined, db, family, tls };
} }
export function createRetryStrategy() { export function createRetryStrategy() {
@@ -339,15 +339,25 @@ export class SpaceMemberService {
return; return;
} }
await executeTx(this.db, async (trx) => {
await trx
.selectFrom('spaces')
.select('id')
.where('id', '=', dto.spaceId)
.forUpdate()
.executeTakeFirst();
if (spaceMember.role === SpaceRole.ADMIN) { if (spaceMember.role === SpaceRole.ADMIN) {
await this.validateLastAdmin(dto.spaceId); await this.validateLastAdmin(dto.spaceId, trx);
} }
await this.spaceMemberRepo.updateSpaceMember( await this.spaceMemberRepo.updateSpaceMember(
{ role: dto.role }, { role: dto.role },
spaceMember.id, spaceMember.id,
dto.spaceId, dto.spaceId,
trx,
); );
});
this.auditService.log({ this.auditService.log({
event: AuditEvent.SPACE_MEMBER_ROLE_CHANGED, event: AuditEvent.SPACE_MEMBER_ROLE_CHANGED,
@@ -368,10 +378,14 @@ export class SpaceMemberService {
}); });
} }
async validateLastAdmin(spaceId: string): Promise<void> { async validateLastAdmin(
spaceId: string,
trx?: KyselyTransaction,
): Promise<void> {
const spaceOwnerCount = await this.spaceMemberRepo.roleCountBySpaceId( const spaceOwnerCount = await this.spaceMemberRepo.roleCountBySpaceId(
SpaceRole.ADMIN, SpaceRole.ADMIN,
spaceId, spaceId,
trx,
); );
if (spaceOwnerCount === 1) { if (spaceOwnerCount === 1) {
throw new BadRequestException( throw new BadRequestException(
@@ -396,7 +396,10 @@ export class WorkspaceService {
} }
} }
if (updateWorkspaceDto.aiSearch) { if (
updateWorkspaceDto.aiSearch &&
this.environmentService.getAiVectorDriver() !== 'turbopuffer'
) {
const tableExists = await isPageEmbeddingsTableExists(this.db); const tableExists = await isPageEmbeddingsTableExists(this.db);
if (!tableExists) { if (!tableExists) {
throw new BadRequestException( throw new BadRequestException(
@@ -46,8 +46,10 @@ export class SpaceMemberRepo {
updatableSpaceMember: UpdatableSpaceMember, updatableSpaceMember: UpdatableSpaceMember,
spaceMemberId: string, spaceMemberId: string,
spaceId: string, spaceId: string,
trx?: KyselyTransaction,
): Promise<void> { ): Promise<void> {
await this.db const db = dbOrTx(this.db, trx);
await db
.updateTable('spaceMembers') .updateTable('spaceMembers')
.set(updatableSpaceMember) .set(updatableSpaceMember)
.where('id', '=', spaceMemberId) .where('id', '=', spaceMemberId)
@@ -92,8 +94,13 @@ export class SpaceMemberRepo {
.execute(); .execute();
} }
async roleCountBySpaceId(role: string, spaceId: string): Promise<number> { async roleCountBySpaceId(
const { count } = await this.db role: string,
spaceId: string,
trx?: KyselyTransaction,
): Promise<number> {
const db = dbOrTx(this.db, trx);
const { count } = await db
.selectFrom('spaceMembers') .selectFrom('spaceMembers')
.select((eb) => eb.fn.count('role').as('count')) .select((eb) => eb.fn.count('role').as('count'))
.where('role', '=', role) .where('role', '=', role)
@@ -5,6 +5,7 @@ import {
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { EnvironmentService } from '../environment/environment.service'; import { EnvironmentService } from '../environment/environment.service';
import { Redis } from 'ioredis'; import { Redis } from 'ioredis';
import { parseRedisUrl } from '../../common/helpers';
@Injectable() @Injectable()
export class RedisHealthIndicator { export class RedisHealthIndicator {
@@ -19,8 +20,10 @@ export class RedisHealthIndicator {
const indicator = this.healthIndicatorService.check(key); const indicator = this.healthIndicatorService.check(key);
try { try {
const redis = new Redis(this.environmentService.getRedisUrl(), { const redisUrl = this.environmentService.getRedisUrl();
const redis = new Redis(redisUrl, {
maxRetriesPerRequest: 15, maxRetriesPerRequest: 15,
tls: parseRedisUrl(redisUrl).tls,
}); });
await redis.ping(); await redis.ping();
@@ -97,6 +97,15 @@ export function xwikiFormatter($: CheerioAPI, $root: Cheerio<any>) {
} }
} }
function isBareLink($el: Cheerio<any>): boolean {
const href = $el.attr("href")?.trim();
const text = $el.text().trim();
if(!text || !href) return false
return text === href;
}
export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) { export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) {
normalizeTableColumnWidths($, $root); normalizeTableColumnWidths($, $root);
@@ -104,7 +113,9 @@ export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) {
const $el = $(el); const $el = $(el);
const url = $el.attr('href')!; const url = $el.attr('href')!;
const { provider } = getEmbedUrlAndProvider(url); const { provider } = getEmbedUrlAndProvider(url);
if (provider === 'iframe') return; if (provider === 'iframe' || !isBareLink($el)) {
return;
}
const embed = `<div data-type=\"embed\" data-src=\"${url}\" data-provider=\"${provider}\" data-align=\"center\" data-width=\"640\" data-height=\"480\"></div>`; const embed = `<div data-type=\"embed\" data-src=\"${url}\" data-provider=\"${provider}\" data-align=\"center\" data-width=\"640\" data-height=\"480\"></div>`;
$el.replaceWith(embed); $el.replaceWith(embed);
@@ -18,6 +18,7 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
password: redisConfig.password, password: redisConfig.password,
db: redisConfig.db, db: redisConfig.db,
family: redisConfig.family, family: redisConfig.family,
tls: redisConfig.tls,
retryStrategy: createRetryStrategy(), retryStrategy: createRetryStrategy(),
}, },
defaultJobOptions: { defaultJobOptions: {
@@ -19,6 +19,7 @@ export class RedisConfigService implements RedisOptionsFactory {
password: redisConfig.password, password: redisConfig.password,
db: redisConfig.db, db: redisConfig.db,
family: redisConfig.family, family: redisConfig.family,
tls: redisConfig.tls,
retryStrategy: createRetryStrategy(), retryStrategy: createRetryStrategy(),
}, },
}; };
@@ -3,7 +3,7 @@ import { ThrottlerModule } from '@nestjs/throttler';
import { ThrottlerStorageRedisService } from '@nest-lab/throttler-storage-redis'; import { ThrottlerStorageRedisService } from '@nest-lab/throttler-storage-redis';
import { EnvironmentService } from '../environment/environment.service'; import { EnvironmentService } from '../environment/environment.service';
import { EnvironmentModule } from '../environment/environment.module'; import { EnvironmentModule } from '../environment/environment.module';
import { parseRedisUrl } from '../../common/helpers'; import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
import { AUTH_THROTTLER, AI_CHAT_THROTTLER } from './throttler-names'; import { AUTH_THROTTLER, AI_CHAT_THROTTLER } from './throttler-names';
import Redis from 'ioredis'; import Redis from 'ioredis';
@@ -27,6 +27,8 @@ import Redis from 'ioredis';
password: redisConfig.password, password: redisConfig.password,
db: redisConfig.db, db: redisConfig.db,
family: redisConfig.family, family: redisConfig.family,
tls: redisConfig.tls,
retryStrategy: createRetryStrategy(),
keyPrefix: 'throttle:', keyPrefix: 'throttle:',
}), }),
), ),
@@ -17,6 +17,7 @@ export class WsRedisIoAdapter extends IoAdapter {
const options: RedisOptions = { const options: RedisOptions = {
family: this.redisConfig.family, family: this.redisConfig.family,
tls: this.redisConfig.tls,
retryStrategy: createRetryStrategy(), retryStrategy: createRetryStrategy(),
}; };
@@ -73,9 +73,13 @@ export const embedProviders: IEmbedProvider[] = [
id: "vimeo", id: "vimeo",
name: "Vimeo", name: "Vimeo",
regex: regex:
/^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/, /^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)(?:\/([\da-zA-Z]+))?/,
getEmbedUrl: (match) => { getEmbedUrl: (match, url: string) => {
return `https://player.vimeo.com/video/${match[4]}`; // preserve ?h= hash for unlisted videos
const hash =
match[5] ?? new URL(url, "https://vimeo.com").searchParams.get("h");
const base = `https://player.vimeo.com/video/${match[4]}`;
return hash ? `${base}?h=${hash}` : base;
}, },
}, },
{ {
+183 -285
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -6,12 +6,12 @@ patchedDependencies:
overrides: overrides:
prosemirror-changeset: 2.4.0 prosemirror-changeset: 2.4.0
glob: 13.0.6 glob: 13.0.6
ws: 8.21.0 ws: 8.21.3
dompurify: 3.4.13 dompurify: 3.4.13
mermaid: 11.16.1 mermaid: 11.16.1
undici: 7.29.0 undici: 7.29.0
tmp: 0.2.7 tmp: 0.2.7
nanoid@^3: 3.3.17 nanoid@^3: 3.3.18
lodash-es: 4.18.1 lodash-es: 4.18.1
express-rate-limit: 8.2.2 express-rate-limit: 8.2.2
flatted: 3.4.2 flatted: 3.4.2