mirror of
https://github.com/docmost/docmost.git
synced 2026-09-11 07:56:54 +08:00
feat(ee): SIEM (#2471)
This commit is contained in:
Philip Okugbe
@@ -161,7 +161,8 @@
|
||||
"moduleFileExtensions": [
|
||||
"js",
|
||||
"json",
|
||||
"ts"
|
||||
"ts",
|
||||
"tsx"
|
||||
],
|
||||
"rootDir": "src",
|
||||
"testRegex": ".*\\.spec\\.ts$",
|
||||
@@ -181,7 +182,7 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"^.+\\.(t|j)s$": "ts-jest"
|
||||
"^.+\\.(t|j)sx?$": "ts-jest"
|
||||
},
|
||||
"transformIgnorePatterns": [
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom)(@|/))"
|
||||
|
||||
@@ -28,6 +28,7 @@ import { LoggerModule } from './common/logger/logger.module';
|
||||
import { ClsModule } from 'nestjs-cls';
|
||||
import { NoopAuditModule } from './integrations/audit/audit.module';
|
||||
import { ThrottleModule } from './integrations/throttle/throttle.module';
|
||||
import { OutboundModule } from './integrations/outbound/outbound.module';
|
||||
import { EncryptionModule } from './integrations/encryption/encryption.module';
|
||||
|
||||
const enterpriseModules = [];
|
||||
@@ -51,7 +52,7 @@ try {
|
||||
middleware: { mount: true },
|
||||
}),
|
||||
LoggerModule,
|
||||
NoopAuditModule,
|
||||
...(enterpriseModules.length > 0 ? [] : [NoopAuditModule]),
|
||||
CoreModule,
|
||||
DatabaseModule,
|
||||
EnvironmentModule,
|
||||
@@ -98,6 +99,7 @@ try {
|
||||
SecurityModule,
|
||||
TelemetryModule,
|
||||
ThrottleModule,
|
||||
OutboundModule,
|
||||
...enterpriseModules,
|
||||
],
|
||||
controllers: [AppController],
|
||||
|
||||
@@ -14,6 +14,7 @@ export const AuditEvent = {
|
||||
USER_ROLE_CHANGED: 'user.role_changed',
|
||||
USER_PASSWORD_CHANGED: 'user.password_changed',
|
||||
USER_PASSWORD_RESET: 'user.password_reset',
|
||||
USER_PASSWORD_RESET_REQUESTED: 'user.password_reset_requested',
|
||||
USER_UPDATED: 'user.updated',
|
||||
USER_DEACTIVATED: 'user.deactivated',
|
||||
USER_ACTIVATED: 'user.activated',
|
||||
@@ -69,6 +70,7 @@ export const AuditEvent = {
|
||||
PAGE_RESTRICTION_REMOVED: 'page.restriction_removed',
|
||||
PAGE_PERMISSION_ADDED: 'page.permission_added',
|
||||
PAGE_PERMISSION_REMOVED: 'page.permission_removed',
|
||||
PAGE_PERMISSION_ROLE_CHANGED: 'page.permission_role_changed',
|
||||
// Page verification
|
||||
PAGE_VERIFICATION_CREATED: 'page.verification_created',
|
||||
PAGE_VERIFICATION_UPDATED: 'page.verification_updated',
|
||||
@@ -104,6 +106,16 @@ export const AuditEvent = {
|
||||
// Attachment
|
||||
ATTACHMENT_UPLOADED: 'attachment.uploaded',
|
||||
// ATTACHMENT_DELETED: 'attachment.deleted',
|
||||
|
||||
// SIEM streaming
|
||||
SIEM_DESTINATION_CREATED: 'siem_destination.created',
|
||||
SIEM_DESTINATION_UPDATED: 'siem_destination.updated',
|
||||
SIEM_DESTINATION_DELETED: 'siem_destination.deleted',
|
||||
SIEM_DESTINATION_TEST: 'siem_destination.test',
|
||||
|
||||
// Template
|
||||
TEMPLATE_CREATED: 'template.created',
|
||||
TEMPLATE_DELETED: 'template.deleted',
|
||||
} as const;
|
||||
|
||||
export type AuditEventType = (typeof AuditEvent)[keyof typeof AuditEvent];
|
||||
@@ -116,7 +128,8 @@ export const EXCLUDED_AUDIT_EVENTS: Set<string> = new Set([
|
||||
AuditEvent.COMMENT_UPDATED,
|
||||
AuditEvent.COMMENT_RESOLVED,
|
||||
AuditEvent.COMMENT_REOPENED,
|
||||
AuditEvent.ATTACHMENT_UPLOADED
|
||||
AuditEvent.ATTACHMENT_UPLOADED,
|
||||
AuditEvent.SIEM_DESTINATION_TEST,
|
||||
]);
|
||||
|
||||
export const AuditResource = {
|
||||
@@ -136,6 +149,8 @@ export const AuditResource = {
|
||||
WORKSPACE_INVITATION: 'workspace_invitation',
|
||||
ATTACHMENT: 'attachment',
|
||||
LICENSE: 'license',
|
||||
SIEM_DESTINATION: 'siem_destination',
|
||||
TEMPLATE: 'template',
|
||||
} as const;
|
||||
|
||||
export type AuditResourceType =
|
||||
|
||||
@@ -26,6 +26,7 @@ export const Feature = {
|
||||
OAUTH: 'oauth',
|
||||
AI_CONTROLS: 'ai:controls',
|
||||
MCP_CONTROLS: 'mcp:controls',
|
||||
SIEM: 'siem',
|
||||
} as const;
|
||||
|
||||
export type FeatureKey = (typeof Feature)[keyof typeof Feature];
|
||||
|
||||
@@ -4,6 +4,7 @@ export const CacheKey = {
|
||||
`perm:space-roles:${userId}:${spaceId}`,
|
||||
PAGE_CAN_EDIT: (userId: string, pageId: string) =>
|
||||
`perm:can-edit:${userId}:${pageId}`,
|
||||
SIEM_LICENSED: (workspaceId: string) => `siem:licensed:${workspaceId}`,
|
||||
};
|
||||
|
||||
// Permission caches dedupe repeated checks within and across short request bursts.
|
||||
|
||||
@@ -219,6 +219,13 @@ export class AuthService {
|
||||
subject: 'Reset your password',
|
||||
template: emailTemplate,
|
||||
});
|
||||
|
||||
this.auditService.log({
|
||||
event: AuditEvent.USER_PASSWORD_RESET_REQUESTED,
|
||||
resourceType: AuditResource.USER,
|
||||
resourceId: user.id,
|
||||
metadata: { source: 'forgot_password' },
|
||||
});
|
||||
}
|
||||
|
||||
async passwordReset(
|
||||
|
||||
@@ -10,6 +10,9 @@ export const NotificationType = {
|
||||
PAGE_VERIFIED: 'page.verified',
|
||||
PAGE_APPROVAL_REQUESTED: 'page.approval_requested',
|
||||
PAGE_APPROVAL_REJECTED: 'page.approval_rejected',
|
||||
SIEM_DESTINATION_FAILING: 'siem_destination.failing',
|
||||
SIEM_DESTINATION_DISABLED: 'siem_destination.disabled',
|
||||
SIEM_DESTINATION_RECOVERED: 'siem_destination.recovered',
|
||||
} as const;
|
||||
|
||||
export type NotificationType =
|
||||
@@ -40,6 +43,9 @@ export const DIRECT_NOTIFICATION_TYPES: NotificationType[] = [
|
||||
NotificationType.COMMENT_RESOLVED,
|
||||
NotificationType.PAGE_USER_MENTION,
|
||||
NotificationType.PAGE_PERMISSION_GRANTED,
|
||||
NotificationType.SIEM_DESTINATION_FAILING,
|
||||
NotificationType.SIEM_DESTINATION_DISABLED,
|
||||
NotificationType.SIEM_DESTINATION_RECOVERED,
|
||||
];
|
||||
|
||||
export const UPDATES_NOTIFICATION_TYPES: NotificationType[] = [
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('siem_destinations')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.notNull().references('workspaces.id').onDelete('cascade'),
|
||||
)
|
||||
.addColumn('name', 'varchar', (col) => col.notNull())
|
||||
.addColumn('type', 'varchar', (col) => col.notNull())
|
||||
.addColumn('enabled', 'boolean', (col) => col.notNull().defaultTo(true))
|
||||
.addColumn('config', 'jsonb', (col) => col.notNull())
|
||||
.addColumn('secrets', 'text', (col) => col.notNull())
|
||||
.addColumn('cursor_created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('cursor_id', 'uuid', (col) =>
|
||||
col.notNull().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
.addColumn('cursor_snapshot', 'text')
|
||||
// Fences cursor writes from stale jobs after configuration changes.
|
||||
.addColumn('version', 'integer', (col) => col.notNull().defaultTo(0))
|
||||
.addColumn('status', 'varchar', (col) => col.notNull().defaultTo('healthy'))
|
||||
.addColumn('consecutive_failures', 'integer', (col) =>
|
||||
col.notNull().defaultTo(0),
|
||||
)
|
||||
.addColumn('next_attempt_at', 'timestamptz')
|
||||
.addColumn('last_delivered_at', 'timestamptz')
|
||||
.addColumn('last_error', 'text')
|
||||
.addColumn('last_error_at', 'timestamptz')
|
||||
.addColumn('failing_since', 'timestamptz')
|
||||
.addColumn('creator_id', 'uuid', (col) =>
|
||||
col.references('users.id').onDelete('set null'),
|
||||
)
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('updated_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.execute();
|
||||
|
||||
await db.schema
|
||||
.createIndex('idx_siem_destinations_workspace_id')
|
||||
.ifNotExists()
|
||||
.on('siem_destinations')
|
||||
.columns(['workspace_id'])
|
||||
.execute();
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_siem_destinations_due
|
||||
ON siem_destinations (next_attempt_at)
|
||||
WHERE enabled = true
|
||||
`.execute(db);
|
||||
|
||||
await db.schema.alterTable('audit').addColumn('user_agent', 'text').execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('audit').dropColumn('user_agent').execute();
|
||||
await db.schema.dropTable('siem_destinations').ifExists().execute();
|
||||
}
|
||||
+26
@@ -74,6 +74,7 @@ export interface Audit {
|
||||
resourceId: string | null;
|
||||
resourceType: string;
|
||||
spaceId: string | null;
|
||||
userAgent: string | null;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
@@ -361,6 +362,30 @@ export interface SpaceMembers {
|
||||
userId: string | null;
|
||||
}
|
||||
|
||||
export interface SiemDestinations {
|
||||
config: Json;
|
||||
consecutiveFailures: Generated<number>;
|
||||
createdAt: Generated<Timestamp>;
|
||||
creatorId: string | null;
|
||||
cursorCreatedAt: Generated<Timestamp>;
|
||||
cursorId: Generated<string>;
|
||||
cursorSnapshot: string | null;
|
||||
enabled: Generated<boolean>;
|
||||
failingSince: Timestamp | null;
|
||||
id: Generated<string>;
|
||||
lastDeliveredAt: Timestamp | null;
|
||||
lastError: string | null;
|
||||
lastErrorAt: Timestamp | null;
|
||||
name: string;
|
||||
nextAttemptAt: Timestamp | null;
|
||||
secrets: string;
|
||||
status: Generated<string>;
|
||||
type: string;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
version: Generated<number>;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface Spaces {
|
||||
createdAt: Generated<Timestamp>;
|
||||
creatorId: string | null;
|
||||
@@ -724,6 +749,7 @@ export interface DB {
|
||||
pages: Pages;
|
||||
scimTokens: ScimTokens;
|
||||
shares: Shares;
|
||||
siemDestinations: SiemDestinations;
|
||||
spaceMembers: SpaceMembers;
|
||||
spaces: Spaces;
|
||||
templates: Templates;
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
UserSessions,
|
||||
ApiKeys,
|
||||
ScimTokens,
|
||||
SiemDestinations,
|
||||
Watchers,
|
||||
Audit as _Audit,
|
||||
Templates,
|
||||
@@ -267,3 +268,8 @@ export type UpdatableBaseRow = Updateable<Omit<BaseRows, 'id'>>;
|
||||
export type BaseView = Selectable<BaseViews>;
|
||||
export type InsertableBaseView = Insertable<BaseViews>;
|
||||
export type UpdatableBaseView = Updateable<Omit<BaseViews, 'id'>>;
|
||||
|
||||
// SIEM destinations
|
||||
export type SiemDestination = Selectable<SiemDestinations>;
|
||||
export type InsertableSiemDestination = Insertable<SiemDestinations>;
|
||||
export type UpdatableSiemDestination = Updateable<Omit<SiemDestinations, 'id'>>;
|
||||
|
||||
+1
-1
Submodule apps/server/src/ee updated: cbf35d363a...c8ca1467df
@@ -385,4 +385,8 @@ export class EnvironmentService {
|
||||
.map((o) => o.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
getAllowedPrivateNetworks(): string {
|
||||
return this.configService.get<string>('ALLOWED_PRIVATE_NETWORKS', 'none');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { Agent } from 'undici';
|
||||
import { OutboundAgentFactory } from './outbound-agent.factory';
|
||||
import { OutboundUrlError } from './outbound-url.guard';
|
||||
|
||||
describe('OutboundAgentFactory', () => {
|
||||
it('validates the URL through the guard and returns a releasable undici Agent', async () => {
|
||||
const validate = jest.fn().mockResolvedValue({ hostname: 'siem.example.com', address: '203.0.113.5', family: 4 });
|
||||
const factory = new OutboundAgentFactory({ validate } as any);
|
||||
|
||||
const lease = await factory.lease('https://siem.example.com/ingest', { caCert: undefined, rejectUnauthorized: true });
|
||||
|
||||
expect(validate).toHaveBeenCalledWith('https://siem.example.com/ingest');
|
||||
expect(lease.dispatcher).toBeInstanceOf(Agent);
|
||||
await expect(lease.release()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('propagates guard rejections', async () => {
|
||||
const validate = jest.fn().mockRejectedValue(new OutboundUrlError('Destination URL must use https'));
|
||||
const factory = new OutboundAgentFactory({ validate } as any);
|
||||
|
||||
await expect(factory.lease('http://siem.example.com')).rejects.toThrow(OutboundUrlError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Agent, Dispatcher } from 'undici';
|
||||
import { OutboundUrlGuard } from './outbound-url.guard';
|
||||
|
||||
export const OUTBOUND_REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
export type OutboundTlsOptions = {
|
||||
caCert?: string; // PEM encoded
|
||||
rejectUnauthorized?: boolean; // Defaults to true; self-hosted only when false.
|
||||
};
|
||||
|
||||
export type AgentLease = {
|
||||
dispatcher: Dispatcher;
|
||||
release: () => Promise<void>;
|
||||
};
|
||||
|
||||
export type IOutboundAgentFactory = {
|
||||
lease(url: string, tls?: OutboundTlsOptions): Promise<AgentLease>;
|
||||
};
|
||||
|
||||
/** Creates a per-request agent pinned to the address validated by the SSRF guard. */
|
||||
@Injectable()
|
||||
export class OutboundAgentFactory implements IOutboundAgentFactory {
|
||||
constructor(private readonly urlGuard: OutboundUrlGuard) {}
|
||||
|
||||
async lease(url: string, tls?: OutboundTlsOptions): Promise<AgentLease> {
|
||||
const pinned = await this.urlGuard.validate(url);
|
||||
|
||||
const lookup = (_hostname: string, options: any, callback: any) => {
|
||||
if (options?.all) {
|
||||
callback(null, [{ address: pinned.address, family: pinned.family }]);
|
||||
} else {
|
||||
callback(null, pinned.address, pinned.family);
|
||||
}
|
||||
};
|
||||
|
||||
const agent = new Agent({
|
||||
connect: {
|
||||
ca: tls?.caCert || undefined,
|
||||
rejectUnauthorized: tls?.rejectUnauthorized ?? true,
|
||||
lookup: lookup as any,
|
||||
timeout: OUTBOUND_REQUEST_TIMEOUT_MS,
|
||||
},
|
||||
headersTimeout: OUTBOUND_REQUEST_TIMEOUT_MS,
|
||||
bodyTimeout: OUTBOUND_REQUEST_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
return {
|
||||
dispatcher: agent,
|
||||
release: async () => {
|
||||
await agent.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import {
|
||||
parseOutboundNetworkPolicy,
|
||||
policyNamesAddress,
|
||||
} from './outbound-network-policy';
|
||||
|
||||
describe('parseOutboundNetworkPolicy', () => {
|
||||
it('parses a bare mode', () => {
|
||||
expect(parseOutboundNetworkPolicy('all')).toMatchObject({
|
||||
mode: 'all',
|
||||
entries: [],
|
||||
invalid: false,
|
||||
});
|
||||
expect(parseOutboundNetworkPolicy('none')).toMatchObject({
|
||||
mode: 'none',
|
||||
entries: [],
|
||||
invalid: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('treats an empty value as none with no entries', () => {
|
||||
for (const raw of ['', ' ', ',,']) {
|
||||
expect(parseOutboundNetworkPolicy(raw)).toMatchObject({
|
||||
mode: 'none',
|
||||
entries: [],
|
||||
invalid: false,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores case and surrounding whitespace on the mode', () => {
|
||||
expect(parseOutboundNetworkPolicy(' ALL ')).toMatchObject({
|
||||
mode: 'all',
|
||||
invalid: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('parses a mode followed by entries', () => {
|
||||
const policy = parseOutboundNetworkPolicy('all,127.0.0.0/8,::1/128');
|
||||
|
||||
expect(policy.mode).toBe('all');
|
||||
expect(policy.entries).toHaveLength(2);
|
||||
expect(policyNamesAddress(policy, '127.0.0.1', 80)).toBe(true);
|
||||
expect(policyNamesAddress(policy, '127.0.0.1', 8088)).toBe(true);
|
||||
expect(policyNamesAddress(policy, '::1', 443)).toBe(true);
|
||||
expect(policyNamesAddress(policy, '10.1.2.3', 443)).toBe(false);
|
||||
});
|
||||
|
||||
it('parses an entry with a port and matches only that port', () => {
|
||||
const policy = parseOutboundNetworkPolicy('none,192.168.1.20/32:8088');
|
||||
|
||||
expect(policy.mode).toBe('none');
|
||||
expect(policyNamesAddress(policy, '192.168.1.20', 8088)).toBe(true);
|
||||
expect(policyNamesAddress(policy, '192.168.1.20', 443)).toBe(false);
|
||||
expect(policyNamesAddress(policy, '192.168.1.21', 8088)).toBe(false);
|
||||
});
|
||||
|
||||
it('parses a bracketed IPv6 entry with a port', () => {
|
||||
const policy = parseOutboundNetworkPolicy('[::1/128]:8088');
|
||||
|
||||
expect(policy.mode).toBe('none');
|
||||
expect(policyNamesAddress(policy, '::1', 8088)).toBe(true);
|
||||
expect(policyNamesAddress(policy, '::1', 80)).toBe(false);
|
||||
});
|
||||
|
||||
it('treats entries without a mode as none plus those entries', () => {
|
||||
const policy = parseOutboundNetworkPolicy('10.0.0.0/8');
|
||||
|
||||
expect(policy.mode).toBe('none');
|
||||
expect(policy.invalid).toBe(false);
|
||||
expect(policyNamesAddress(policy, '10.1.2.3', 443)).toBe(true);
|
||||
expect(policyNamesAddress(policy, '192.168.1.1', 443)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'not-a-cidr',
|
||||
'all,not-a-cidr',
|
||||
'all,10.0.0.0/8,nonsense',
|
||||
'10.0.0.0/33',
|
||||
'10.0.0.0',
|
||||
'::1/129',
|
||||
'::1/128:8088',
|
||||
'10.0.0.0/8:0',
|
||||
'10.0.0.0/8:70000',
|
||||
'[::1/128]:notaport',
|
||||
'0.0.0.0/0',
|
||||
'::/0',
|
||||
'all,0.0.0.0/0',
|
||||
'[::/0]:8088',
|
||||
'192.168.1.20/24',
|
||||
'10.1.0.0/8',
|
||||
'172.16.0.1/12',
|
||||
'fc00::1/7',
|
||||
'[::1/127]',
|
||||
'2001:db8::1/32:8088',
|
||||
])('fails closed on %s', (raw) => {
|
||||
expect(parseOutboundNetworkPolicy(raw)).toMatchObject({
|
||||
mode: 'none',
|
||||
entries: [],
|
||||
invalid: true,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
'10.0.0.0/8',
|
||||
'172.16.0.0/12',
|
||||
'100.64.0.0/10',
|
||||
'192.168.1.20/32',
|
||||
'fc00::/7',
|
||||
'fe80::/10',
|
||||
'::1/128',
|
||||
])('accepts %s, whose address sits on its prefix boundary', (raw) => {
|
||||
expect(parseOutboundNetworkPolicy(raw)).toMatchObject({
|
||||
entries: [expect.anything()],
|
||||
invalid: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts a bracketed IPv6 entry without a port as the unbracketed form', () => {
|
||||
const bracketed = parseOutboundNetworkPolicy('[::1/128]');
|
||||
const bare = parseOutboundNetworkPolicy('::1/128');
|
||||
|
||||
expect(bracketed).toMatchObject({ mode: 'none', invalid: false });
|
||||
for (const port of [80, 443, 8088]) {
|
||||
expect(policyNamesAddress(bracketed, '::1', port)).toBe(
|
||||
policyNamesAddress(bare, '::1', port),
|
||||
);
|
||||
expect(policyNamesAddress(bracketed, '::1', port)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('never names an address when the value is unparseable or the address is not an IP', () => {
|
||||
const policy = parseOutboundNetworkPolicy('all,10.0.0.0/8');
|
||||
|
||||
expect(policyNamesAddress(policy, 'siem.internal', 443)).toBe(false);
|
||||
expect(policyNamesAddress(parseOutboundNetworkPolicy('garbage'), '10.1.2.3', 443)).toBe(false);
|
||||
});
|
||||
|
||||
it('matches an IPv4-mapped IPv6 address against an IPv4 entry', () => {
|
||||
const policy = parseOutboundNetworkPolicy('127.0.0.0/8');
|
||||
|
||||
expect(policyNamesAddress(policy, '::ffff:127.0.0.1', 80)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { BlockList, isIPv4, isIPv6 } from 'node:net';
|
||||
|
||||
export type OutboundPolicyMode = 'all' | 'none';
|
||||
|
||||
export type OutboundPolicyEntry = { list: BlockList; port?: number };
|
||||
|
||||
/** An invalid policy denies all private destinations. */
|
||||
export type OutboundNetworkPolicy = {
|
||||
mode: OutboundPolicyMode;
|
||||
entries: OutboundPolicyEntry[];
|
||||
invalid: boolean;
|
||||
};
|
||||
|
||||
function toBytes(address: string, family: 'ipv4' | 'ipv6'): number[] {
|
||||
if (family === 'ipv4') return address.split('.').map(Number);
|
||||
|
||||
const bytesOf = (part: string): number[] =>
|
||||
part
|
||||
? part.split(':').flatMap((group) => {
|
||||
if (group.includes('.')) return group.split('.').map(Number);
|
||||
const value = parseInt(group, 16);
|
||||
return [value >> 8, value & 0xff];
|
||||
})
|
||||
: [];
|
||||
|
||||
const [head, tail] = address.split('::');
|
||||
const headBytes = bytesOf(head);
|
||||
const tailBytes = address.includes('::') ? bytesOf(tail) : [];
|
||||
const zeros = new Array(16 - headBytes.length - tailBytes.length).fill(0);
|
||||
return [...headBytes, ...zeros, ...tailBytes];
|
||||
}
|
||||
|
||||
function hasHostBits(bytes: number[], prefix: number): boolean {
|
||||
return bytes.some((byte, index) => {
|
||||
const bitsBefore = index * 8;
|
||||
if (bitsBefore >= prefix) return byte !== 0;
|
||||
return (byte & (0xff >> Math.min(8, prefix - bitsBefore))) !== 0;
|
||||
});
|
||||
}
|
||||
|
||||
/** Prefix zero is reserved for the explicit `all` mode. */
|
||||
function parseCidr(
|
||||
raw: string,
|
||||
): { address: string; prefix: number; family: 'ipv4' | 'ipv6' } | null {
|
||||
const [address, prefixRaw] = raw.split('/');
|
||||
if (!prefixRaw) return null;
|
||||
const prefix = Number(prefixRaw);
|
||||
if (!Number.isInteger(prefix) || prefix < 1) return null;
|
||||
const family = isIPv4(address) ? 'ipv4' : isIPv6(address) ? 'ipv6' : null;
|
||||
if (!family) return null;
|
||||
if (prefix > (family === 'ipv4' ? 32 : 128)) return null;
|
||||
if (hasHostBits(toBytes(address, family), prefix)) return null;
|
||||
return { address, prefix, family };
|
||||
}
|
||||
|
||||
/** Parses optional ports without treating IPv6 colons as separators. */
|
||||
function splitPort(token: string): { cidr: string; port?: number } {
|
||||
const bracketed = /^\[(.+)\](?::(\d+))?$/.exec(token);
|
||||
if (bracketed) {
|
||||
const [, cidr, port] = bracketed;
|
||||
return port === undefined ? { cidr } : { cidr, port: Number(port) };
|
||||
}
|
||||
const withPort = /^([^:]+):(\d+)$/.exec(token);
|
||||
if (withPort) return { cidr: withPort[1], port: Number(withPort[2]) };
|
||||
return { cidr: token };
|
||||
}
|
||||
|
||||
function parseEntry(token: string): OutboundPolicyEntry | null {
|
||||
const { cidr: raw, port } = splitPort(token);
|
||||
if (port !== undefined && (port < 1 || port > 65535)) return null;
|
||||
const cidr = parseCidr(raw);
|
||||
if (!cidr) return null;
|
||||
const list = new BlockList();
|
||||
list.addSubnet(cidr.address, cidr.prefix, cidr.family);
|
||||
return { list, port };
|
||||
}
|
||||
|
||||
/** Parses `[all|none,]CIDR[:port],...` and fails closed on invalid input. */
|
||||
export function parseOutboundNetworkPolicy(raw: string): OutboundNetworkPolicy {
|
||||
const tokens = (raw ?? '')
|
||||
.split(',')
|
||||
.map((token) => token.trim())
|
||||
.filter(Boolean);
|
||||
if (tokens.length === 0) return { mode: 'none', entries: [], invalid: false };
|
||||
|
||||
const first = tokens[0].toLowerCase();
|
||||
const hasMode = first === 'all' || first === 'none';
|
||||
const mode: OutboundPolicyMode = hasMode ? first : 'none';
|
||||
|
||||
const entries: OutboundPolicyEntry[] = [];
|
||||
for (const token of hasMode ? tokens.slice(1) : tokens) {
|
||||
const entry = parseEntry(token);
|
||||
if (!entry) return { mode: 'none', entries: [], invalid: true };
|
||||
entries.push(entry);
|
||||
}
|
||||
return { mode, entries, invalid: false };
|
||||
}
|
||||
|
||||
export function policyNamesAddress(
|
||||
policy: OutboundNetworkPolicy,
|
||||
ip: string,
|
||||
port: number,
|
||||
): boolean {
|
||||
const family = isIPv4(ip) ? 'ipv4' : isIPv6(ip) ? 'ipv6' : null;
|
||||
if (!family) return false;
|
||||
return policy.entries.some(
|
||||
(entry) =>
|
||||
(entry.port === undefined || entry.port === port) && entry.list.check(ip, family),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import {
|
||||
isAlwaysBlockedAddress,
|
||||
isHardBlockedAddress,
|
||||
isPrivateAddress,
|
||||
isPrivateNetworkAddress,
|
||||
OutboundUrlError,
|
||||
OutboundUrlGuard,
|
||||
} from './outbound-url.guard';
|
||||
|
||||
function guard(
|
||||
isCloud: boolean,
|
||||
addresses: Array<{ address: string; family: number }>,
|
||||
privateNetworks: string = 'none',
|
||||
) {
|
||||
return new OutboundUrlGuard(
|
||||
{
|
||||
isCloud: () => isCloud,
|
||||
getAllowedPrivateNetworks: () => privateNetworks,
|
||||
} as any,
|
||||
async () => addresses,
|
||||
);
|
||||
}
|
||||
|
||||
function family(ip: string): number {
|
||||
return ip.includes(':') ? 6 : 4;
|
||||
}
|
||||
|
||||
describe('isPrivateAddress', () => {
|
||||
it.each([
|
||||
'127.0.0.1', '10.0.0.5', '172.16.0.1', '172.31.255.255', '192.168.1.1',
|
||||
'169.254.169.254', '100.64.0.1', '0.0.0.0', '224.0.0.1',
|
||||
'::1', '::', 'fe80::1', 'fc00::1', 'fd12::1', 'ff02::1', '::ffff:10.0.0.1',
|
||||
'0:0:0:0:0:0:0:1', '::ffff:a00:1', '::ffff:7f00:1', '0000:0000:0000:0000:0000:0000:0000:0000',
|
||||
'192.0.0.1', '192.0.2.1', '192.88.99.1', '198.18.0.1', '198.51.100.7', '203.0.113.5',
|
||||
'::a00:1', '64:ff9b::a00:1', '64:ff9b:1::a00:1', '100::1', '2001::1', '2001:0:a00:1::1', '2001:db8::1', '2002:a00:1::1', 'fec0::1',
|
||||
])('flags %s as private or reserved', (ip) => {
|
||||
expect(isPrivateAddress(ip)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['8.8.8.8', '172.32.0.1', '2606:4700::1111', '::ffff:8.8.8.8', '::ffff:5db8:d822', '::ffff:8.8.8.8', '2001:4860:4860::8888', '100.128.0.1', '198.17.255.255'])(
|
||||
'allows public %s',
|
||||
(ip) => {
|
||||
expect(isPrivateAddress(ip)).toBe(false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('isAlwaysBlockedAddress / isPrivateNetworkAddress', () => {
|
||||
it.each([
|
||||
'0.0.0.0', '127.0.0.1', '169.254.169.254', '192.0.0.1', '192.0.2.1',
|
||||
'192.88.99.1', '198.18.0.1', '198.51.100.7', '203.0.113.5', '224.0.0.1',
|
||||
'::1', '::', '::ffff:127.0.0.1', '::ffff:0:7f00:1', '64:ff9b::a00:1', '64:ff9b:1::a00:1',
|
||||
'100::1', '2001::1', '2001:db8::1', '2002:a00:1::1', 'fe80::1', 'fec0::1',
|
||||
'ff02::1',
|
||||
])('flags %s as always-blocked but not a private network', (ip) => {
|
||||
expect(isAlwaysBlockedAddress(ip)).toBe(true);
|
||||
expect(isPrivateNetworkAddress(ip)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'10.0.0.5', '172.16.0.1', '172.31.255.255', '192.168.1.1', '100.64.0.1',
|
||||
'fc00::1', 'fd12::1',
|
||||
])('flags %s as a private network but not always-blocked', (ip) => {
|
||||
expect(isPrivateNetworkAddress(ip)).toBe(true);
|
||||
expect(isAlwaysBlockedAddress(ip)).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['8.8.8.8', '172.32.0.1', '2606:4700::1111', '100.128.0.1'])(
|
||||
'allows public %s in both',
|
||||
(ip) => {
|
||||
expect(isAlwaysBlockedAddress(ip)).toBe(false);
|
||||
expect(isPrivateNetworkAddress(ip)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it('the two lists together are exactly isPrivateAddress', () => {
|
||||
for (const ip of ['10.0.0.5', '127.0.0.1', '8.8.8.8', 'fe80::1', 'fc00::1']) {
|
||||
expect(isAlwaysBlockedAddress(ip) || isPrivateNetworkAddress(ip)).toBe(
|
||||
isPrivateAddress(ip),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('OutboundUrlGuard.validate', () => {
|
||||
const publicV4 = { address: '93.184.216.34', family: 4 };
|
||||
|
||||
it('rejects http on cloud', async () => {
|
||||
await expect(guard(true, [publicV4]).validate('http://siem.example.com/x'))
|
||||
.rejects.toThrow(OutboundUrlError);
|
||||
});
|
||||
|
||||
it('rejects hosts that resolve to a private range on cloud', async () => {
|
||||
await expect(
|
||||
guard(true, [publicV4, { address: '10.0.0.5', family: 4 }]).validate('https://siem.example.com'),
|
||||
).rejects.toThrow(/private or reserved/);
|
||||
});
|
||||
|
||||
it('rejects the cloud metadata address literal', async () => {
|
||||
await expect(guard(true, []).validate('https://169.254.169.254/latest'))
|
||||
.rejects.toThrow(/private or reserved/);
|
||||
});
|
||||
|
||||
it('allows LAN hosts and http on self-hosted when private networks are allowed', async () => {
|
||||
const pinned = await guard(false, [{ address: '10.0.5.20', family: 4 }], 'all')
|
||||
.validate('http://splunk.internal:8088/services/collector/event');
|
||||
expect(pinned).toEqual({ hostname: 'splunk.internal', address: '10.0.5.20', family: 4 });
|
||||
});
|
||||
|
||||
it('pins the first resolved address and keeps the hostname for SNI', async () => {
|
||||
const pinned = await guard(true, [{ address: '2606:4700::1111', family: 6 }, publicV4])
|
||||
.validate('https://siem.example.com');
|
||||
expect(pinned).toEqual({ hostname: 'siem.example.com', address: '2606:4700::1111', family: 6 });
|
||||
});
|
||||
|
||||
it('rejects credentials in the URL and unresolvable hosts', async () => {
|
||||
await expect(guard(false, [publicV4]).validate('https://user:pw@siem.example.com'))
|
||||
.rejects.toThrow(/credentials/);
|
||||
await expect(guard(false, []).validate('https://nope.example.com'))
|
||||
.rejects.toThrow(/Could not resolve/);
|
||||
});
|
||||
|
||||
it('marks resolution failures retryable and configuration failures not', async () => {
|
||||
const throwing = new OutboundUrlGuard(
|
||||
{ isCloud: () => false } as any,
|
||||
async () => {
|
||||
throw new Error('EAI_AGAIN');
|
||||
},
|
||||
);
|
||||
|
||||
const dnsError = await throwing
|
||||
.validate('https://siem.example.com')
|
||||
.catch((e) => e);
|
||||
expect(dnsError).toBeInstanceOf(OutboundUrlError);
|
||||
expect(dnsError.retryable).toBe(true);
|
||||
|
||||
const emptyError = await guard(false, [])
|
||||
.validate('https://nope.example.com')
|
||||
.catch((e) => e);
|
||||
expect(emptyError.retryable).toBe(true);
|
||||
|
||||
for (const url of [
|
||||
'not-a-url',
|
||||
'ftp://siem.example.com',
|
||||
'https://user:pw@siem.example.com',
|
||||
]) {
|
||||
const err = await guard(false, [publicV4])
|
||||
.validate(url)
|
||||
.catch((e) => e);
|
||||
expect(err).toBeInstanceOf(OutboundUrlError);
|
||||
expect(err.retryable).toBe(false);
|
||||
}
|
||||
|
||||
const privateError = await guard(true, [{ address: '10.0.0.5', family: 4 }])
|
||||
.validate('https://siem.example.com')
|
||||
.catch((e) => e);
|
||||
expect(privateError.retryable).toBe(false);
|
||||
});
|
||||
|
||||
const hardBlocked = ['169.254.169.254', '0.0.0.0', 'fe80::1', 'ff02::1'];
|
||||
const loopbackOrReserved = ['127.0.0.1', '::1', '::ffff:127.0.0.1', '192.0.2.1'];
|
||||
|
||||
it.each(hardBlocked)(
|
||||
'self-hosted refuses %s under every ALLOWED_PRIVATE_NETWORKS value',
|
||||
async (ip) => {
|
||||
for (const value of ['all', 'none', '169.254.0.0/16', 'all,169.254.0.0/16', 'all,fe80::/10']) {
|
||||
await expect(
|
||||
guard(false, [{ address: ip, family: family(ip) }], value).validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(/link-local, metadata or reserved address .* which is never allowed/);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(loopbackOrReserved)(
|
||||
'self-hosted refuses %s unless an entry names it',
|
||||
async (ip) => {
|
||||
for (const value of ['all', 'none']) {
|
||||
await expect(
|
||||
guard(false, [{ address: ip, family: family(ip) }], value).validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(
|
||||
/resolves to a loopback or reserved address .* Set ALLOWED_PRIVATE_NETWORKS on the server to allow it/,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['10.1.2.3', '192.168.1.10'])(
|
||||
'self-hosted refuses private network %s by default',
|
||||
async (ip) => {
|
||||
await expect(
|
||||
guard(false, [{ address: ip, family: 4 }]).validate('http://siem.internal'),
|
||||
).rejects.toThrow(
|
||||
/resolves to a private address .* Set ALLOWED_PRIVATE_NETWORKS on the server to allow it/,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['10.1.2.3', '192.168.1.10', 'fc00::1', '100.64.0.1'])(
|
||||
'all accepts private network %s',
|
||||
async (ip) => {
|
||||
const pinned = await guard(
|
||||
false,
|
||||
[{ address: ip, family: family(ip) }],
|
||||
'all',
|
||||
).validate('http://siem.internal');
|
||||
expect(pinned.address).toBe(ip);
|
||||
},
|
||||
);
|
||||
|
||||
it('all still refuses loopback, and a loopback entry opts it back in', async () => {
|
||||
await expect(
|
||||
guard(false, [{ address: '127.0.0.1', family: 4 }], 'all').validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(/loopback or reserved/);
|
||||
|
||||
const allowed = await guard(
|
||||
false,
|
||||
[{ address: '127.0.0.1', family: 4 }],
|
||||
'all,127.0.0.0/8',
|
||||
).validate('http://siem.internal');
|
||||
expect(allowed.address).toBe('127.0.0.1');
|
||||
|
||||
const lan = await guard(
|
||||
false,
|
||||
[{ address: '10.1.2.3', family: 4 }],
|
||||
'all,127.0.0.0/8',
|
||||
).validate('http://siem.internal');
|
||||
expect(lan.address).toBe('10.1.2.3');
|
||||
|
||||
await expect(
|
||||
guard(false, [{ address: '::1', family: 6 }], 'all,127.0.0.0/8').validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(/loopback or reserved/);
|
||||
});
|
||||
|
||||
it('an entry with a port matches only that port', async () => {
|
||||
const policy = 'none,192.168.1.20/32:8088';
|
||||
|
||||
const allowed = await guard(
|
||||
false,
|
||||
[{ address: '192.168.1.20', family: 4 }],
|
||||
policy,
|
||||
).validate('https://192.168.1.20:8088/services/collector/event');
|
||||
expect(allowed.address).toBe('192.168.1.20');
|
||||
|
||||
await expect(
|
||||
guard(false, [{ address: '192.168.1.20', family: 4 }], policy).validate(
|
||||
'https://192.168.1.20',
|
||||
),
|
||||
).rejects.toThrow(/private address/);
|
||||
|
||||
await expect(
|
||||
guard(false, [{ address: '192.168.1.21', family: 4 }], policy).validate(
|
||||
'https://192.168.1.21:8088',
|
||||
),
|
||||
).rejects.toThrow(/private address/);
|
||||
});
|
||||
|
||||
it('a bracketed IPv6 entry with a port accepts only that port', async () => {
|
||||
const policy = '[::1/128]:8088';
|
||||
|
||||
const allowed = await guard(
|
||||
false,
|
||||
[{ address: '::1', family: 6 }],
|
||||
policy,
|
||||
).validate('http://[::1]:8088/ingest');
|
||||
expect(allowed).toEqual({ hostname: '::1', address: '::1', family: 6 });
|
||||
|
||||
await expect(
|
||||
guard(false, [{ address: '::1', family: 6 }], policy).validate('http://[::1]/ingest'),
|
||||
).rejects.toThrow(/loopback or reserved/);
|
||||
});
|
||||
|
||||
it('an entry without a port matches every port', async () => {
|
||||
for (const url of ['http://127.0.0.1:8088', 'https://127.0.0.1', 'http://127.0.0.1']) {
|
||||
const allowed = await guard(
|
||||
false,
|
||||
[{ address: '127.0.0.1', family: 4 }],
|
||||
'127.0.0.0/8',
|
||||
).validate(url);
|
||||
expect(allowed.address).toBe('127.0.0.1');
|
||||
}
|
||||
});
|
||||
|
||||
it('entries without a mode none every private network not named', async () => {
|
||||
const allowed = await guard(
|
||||
false,
|
||||
[{ address: '192.168.1.10', family: 4 }],
|
||||
'192.168.1.0/24',
|
||||
).validate('http://siem.internal');
|
||||
expect(allowed.address).toBe('192.168.1.10');
|
||||
|
||||
await expect(
|
||||
guard(false, [{ address: '10.1.2.3', family: 4 }], '192.168.1.0/24').validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(/private address/);
|
||||
});
|
||||
|
||||
it('an unparseable value denies everything private or reserved and logs once per process', async () => {
|
||||
const errorSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
const g = guard(false, [{ address: '10.1.2.3', family: 4 }], 'all,10.0.0.0/8, not-a-cidr');
|
||||
|
||||
await expect(g.validate('http://siem.internal')).rejects.toThrow(/private address/);
|
||||
await expect(g.validate('http://siem.internal')).rejects.toThrow(/private address/);
|
||||
|
||||
expect(errorSpy).toHaveBeenCalledTimes(1);
|
||||
expect(errorSpy.mock.calls[0][0]).toMatch(/ALLOWED_PRIVATE_NETWORKS/);
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('cloud ignores ALLOWED_PRIVATE_NETWORKS and always refuses private and reserved ranges', async () => {
|
||||
for (const ip of [...hardBlocked, ...loopbackOrReserved, '10.1.2.3', '192.168.1.10']) {
|
||||
for (const value of ['all', 'none', '127.0.0.0/8', 'all,10.0.0.0/8']) {
|
||||
await expect(
|
||||
guard(true, [{ address: ip, family: family(ip) }], value).validate(
|
||||
'https://siem.example.com',
|
||||
),
|
||||
).rejects.toThrow(/private or reserved/);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses a resolved address that is not an IP address', async () => {
|
||||
await expect(
|
||||
guard(false, [{ address: 'not-an-ip', family: 4 }], 'all').validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(/is not an IP address/);
|
||||
});
|
||||
|
||||
it('refuses the whole host when any one of its addresses is refused', async () => {
|
||||
await expect(
|
||||
guard(
|
||||
false,
|
||||
[publicV4, { address: '10.1.2.3', family: 4 }],
|
||||
'none',
|
||||
).validate('http://siem.internal'),
|
||||
).rejects.toThrow(/private address/);
|
||||
|
||||
await expect(
|
||||
guard(
|
||||
false,
|
||||
[publicV4, { address: '127.0.0.1', family: 4 }],
|
||||
'all',
|
||||
).validate('http://siem.internal'),
|
||||
).rejects.toThrow(/loopback or reserved/);
|
||||
|
||||
await expect(
|
||||
guard(
|
||||
false,
|
||||
[publicV4, { address: '169.254.169.254', family: 4 }],
|
||||
'all',
|
||||
).validate('http://siem.internal'),
|
||||
).rejects.toThrow(/never allowed/);
|
||||
});
|
||||
|
||||
it('refuses an IPv4-translated loopback address even when private networks are allowed', async () => {
|
||||
await expect(
|
||||
guard(false, [{ address: '::ffff:0:7f00:1', family: 6 }], 'all').validate(
|
||||
'http://siem.internal',
|
||||
),
|
||||
).rejects.toThrow(/loopback or reserved/);
|
||||
});
|
||||
|
||||
it('a public address is allowed in every mode', async () => {
|
||||
const errorSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
for (const value of ['all', 'none', '', '192.168.1.0/24', 'garbage']) {
|
||||
await expect(
|
||||
guard(false, [publicV4], value).validate('http://siem.example.com'),
|
||||
).resolves.toMatchObject({ address: publicV4.address });
|
||||
}
|
||||
await expect(
|
||||
guard(true, [publicV4], 'all').validate('https://siem.example.com'),
|
||||
).resolves.toMatchObject({ address: publicV4.address });
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isHardBlockedAddress', () => {
|
||||
it.each([
|
||||
'0.0.0.0', '169.254.169.254', '224.0.0.1', '255.255.255.255',
|
||||
'::', 'fe80::1', 'ff02::1',
|
||||
])('flags %s as hard-blocked', (ip) => {
|
||||
expect(isHardBlockedAddress(ip)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['127.0.0.1', '::1', '192.0.2.1', 'fec0::1', '8.8.8.8'])(
|
||||
'does not flag %s as hard-blocked (it may still be always-blocked)',
|
||||
(ip) => {
|
||||
expect(isHardBlockedAddress(ip)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it('is a subset of isAlwaysBlockedAddress', () => {
|
||||
for (const ip of ['0.0.0.0', '169.254.169.254', 'fe80::1', 'ff02::1']) {
|
||||
expect(isAlwaysBlockedAddress(ip)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,231 @@
|
||||
import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
|
||||
import { promises as dns } from 'node:dns';
|
||||
import { BlockList, isIPv4, isIPv6 } from 'node:net';
|
||||
import { EnvironmentService } from '../environment/environment.service';
|
||||
import {
|
||||
OutboundNetworkPolicy,
|
||||
parseOutboundNetworkPolicy,
|
||||
policyNamesAddress,
|
||||
} from './outbound-network-policy';
|
||||
|
||||
export const OUTBOUND_LOOKUP = 'OUTBOUND_LOOKUP';
|
||||
|
||||
export type ResolvedAddress = { address: string; family: number };
|
||||
export type LookupFn = (hostname: string) => Promise<ResolvedAddress[]>;
|
||||
export type PinnedAddress = { hostname: string; address: string; family: 4 | 6 };
|
||||
|
||||
/** A rejected URL. Only transient resolution failures are retryable. */
|
||||
export class OutboundUrlError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly retryable: boolean = false,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'OutboundUrlError';
|
||||
}
|
||||
}
|
||||
|
||||
export const defaultLookup: LookupFn = async (hostname) => {
|
||||
const results = await dns.lookup(hostname, { all: true });
|
||||
return results.map((r) => ({ address: r.address, family: r.family }));
|
||||
};
|
||||
|
||||
// Reserved ranges blocked unless explicitly allowed on self-hosted deployments.
|
||||
const ALWAYS_BLOCKED = new BlockList();
|
||||
ALWAYS_BLOCKED.addSubnet('0.0.0.0', 8, 'ipv4'); // "this" network / unspecified
|
||||
ALWAYS_BLOCKED.addSubnet('127.0.0.0', 8, 'ipv4');
|
||||
ALWAYS_BLOCKED.addSubnet('169.254.0.0', 16, 'ipv4'); // link-local / cloud metadata
|
||||
ALWAYS_BLOCKED.addSubnet('192.0.0.0', 24, 'ipv4'); // IETF protocol assignments
|
||||
ALWAYS_BLOCKED.addSubnet('192.0.2.0', 24, 'ipv4'); // TEST-NET-1
|
||||
ALWAYS_BLOCKED.addSubnet('192.88.99.0', 24, 'ipv4'); // deprecated 6to4 relay anycast
|
||||
ALWAYS_BLOCKED.addSubnet('198.18.0.0', 15, 'ipv4'); // benchmarking
|
||||
ALWAYS_BLOCKED.addSubnet('198.51.100.0', 24, 'ipv4'); // TEST-NET-2
|
||||
ALWAYS_BLOCKED.addSubnet('203.0.113.0', 24, 'ipv4'); // TEST-NET-3
|
||||
ALWAYS_BLOCKED.addRange('224.0.0.0', '255.255.255.255', 'ipv4'); // multicast + reserved
|
||||
ALWAYS_BLOCKED.addSubnet('::', 96, 'ipv6'); // deprecated IPv4-compatible
|
||||
ALWAYS_BLOCKED.addSubnet('::ffff:0:0:0', 96, 'ipv6'); // IPv4-translated (SIIT): ::ffff:0:7f00:1 is 127.0.0.1
|
||||
ALWAYS_BLOCKED.addSubnet('::', 128, 'ipv6'); // unspecified
|
||||
ALWAYS_BLOCKED.addSubnet('::1', 128, 'ipv6'); // loopback
|
||||
ALWAYS_BLOCKED.addSubnet('64:ff9b::', 96, 'ipv6'); // NAT64 well-known prefix
|
||||
ALWAYS_BLOCKED.addSubnet('64:ff9b:1::', 48, 'ipv6'); // NAT64 local-use
|
||||
ALWAYS_BLOCKED.addSubnet('100::', 64, 'ipv6'); // discard-only
|
||||
ALWAYS_BLOCKED.addSubnet('2001::', 32, 'ipv6'); // Teredo
|
||||
ALWAYS_BLOCKED.addSubnet('2001:db8::', 32, 'ipv6'); // documentation
|
||||
ALWAYS_BLOCKED.addSubnet('2002::', 16, 'ipv6'); // 6to4
|
||||
ALWAYS_BLOCKED.addSubnet('fe80::', 10, 'ipv6'); // link-local
|
||||
ALWAYS_BLOCKED.addSubnet('fec0::', 10, 'ipv6'); // deprecated site-local
|
||||
ALWAYS_BLOCKED.addSubnet('ff00::', 8, 'ipv6'); // multicast
|
||||
|
||||
// Private ranges that self-hosted deployments can allow.
|
||||
const PRIVATE_NETWORKS = new BlockList();
|
||||
PRIVATE_NETWORKS.addSubnet('10.0.0.0', 8, 'ipv4');
|
||||
PRIVATE_NETWORKS.addSubnet('100.64.0.0', 10, 'ipv4'); // CGNAT
|
||||
PRIVATE_NETWORKS.addSubnet('172.16.0.0', 12, 'ipv4');
|
||||
PRIVATE_NETWORKS.addSubnet('192.168.0.0', 16, 'ipv4');
|
||||
PRIVATE_NETWORKS.addSubnet('fc00::', 7, 'ipv6'); // unique-local
|
||||
|
||||
// These ranges cannot be allowed by policy.
|
||||
const HARD_BLOCKED = new BlockList();
|
||||
HARD_BLOCKED.addSubnet('0.0.0.0', 8, 'ipv4');
|
||||
HARD_BLOCKED.addSubnet('169.254.0.0', 16, 'ipv4');
|
||||
HARD_BLOCKED.addRange('224.0.0.0', '255.255.255.255', 'ipv4');
|
||||
HARD_BLOCKED.addSubnet('::', 128, 'ipv6');
|
||||
HARD_BLOCKED.addSubnet('fe80::', 10, 'ipv6');
|
||||
HARD_BLOCKED.addSubnet('ff00::', 8, 'ipv6');
|
||||
|
||||
/** Returns true for reserved or transition ranges. Invalid input is blocked. */
|
||||
export function isAlwaysBlockedAddress(ip: string): boolean {
|
||||
if (isIPv4(ip)) return ALWAYS_BLOCKED.check(ip, 'ipv4');
|
||||
if (isIPv6(ip)) return ALWAYS_BLOCKED.check(ip, 'ipv6');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Returns true for ranges that policy cannot allow. Invalid input is blocked. */
|
||||
export function isHardBlockedAddress(ip: string): boolean {
|
||||
if (isIPv4(ip)) return HARD_BLOCKED.check(ip, 'ipv4');
|
||||
if (isIPv6(ip)) return HARD_BLOCKED.check(ip, 'ipv6');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Returns true for private network ranges. Invalid input is blocked. */
|
||||
export function isPrivateNetworkAddress(ip: string): boolean {
|
||||
if (isIPv4(ip)) return PRIVATE_NETWORKS.check(ip, 'ipv4');
|
||||
if (isIPv6(ip)) return PRIVATE_NETWORKS.check(ip, 'ipv6');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Returns true for addresses blocked by cloud deployments. */
|
||||
export function isPrivateAddress(ip: string): boolean {
|
||||
return isAlwaysBlockedAddress(ip) || isPrivateNetworkAddress(ip);
|
||||
}
|
||||
|
||||
type Refusal = {
|
||||
address: string;
|
||||
kind: 'not-an-ip' | 'hard-blocked' | 'private' | 'reserved';
|
||||
};
|
||||
|
||||
function findRefusal(
|
||||
resolved: ResolvedAddress[],
|
||||
port: number,
|
||||
policy: OutboundNetworkPolicy,
|
||||
): Refusal | undefined {
|
||||
for (const { address } of resolved) {
|
||||
// Reject invalid resolver output before policy checks.
|
||||
if (!isIPv4(address) && !isIPv6(address)) return { address, kind: 'not-an-ip' };
|
||||
if (isHardBlockedAddress(address)) return { address, kind: 'hard-blocked' };
|
||||
if (policyNamesAddress(policy, address, port)) continue;
|
||||
if (isPrivateNetworkAddress(address)) {
|
||||
if (policy.mode === 'all') continue;
|
||||
return { address, kind: 'private' };
|
||||
}
|
||||
if (isAlwaysBlockedAddress(address)) return { address, kind: 'reserved' };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function describeRefusal(hostname: string, { address, kind }: Refusal): string {
|
||||
if (kind === 'not-an-ip') {
|
||||
return `Destination host "${hostname}" resolved to "${address}", which is not an IP address`;
|
||||
}
|
||||
if (kind === 'hard-blocked') {
|
||||
return `Destination host "${hostname}" resolves to a link-local, metadata or reserved address (${address}), which is never allowed`;
|
||||
}
|
||||
const description =
|
||||
kind === 'private' ? 'a private address' : 'a loopback or reserved address';
|
||||
return `Destination host "${hostname}" resolves to ${description} (${address}). Set ALLOWED_PRIVATE_NETWORKS on the server to allow it`;
|
||||
}
|
||||
|
||||
function effectivePort(url: URL): number {
|
||||
if (url.port) return Number(url.port);
|
||||
return url.protocol === 'https:' ? 443 : 80;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class OutboundUrlGuard {
|
||||
private readonly logger = new Logger(OutboundUrlGuard.name);
|
||||
private readonly lookup: LookupFn;
|
||||
private cachedPolicy?: { raw: string; policy: OutboundNetworkPolicy };
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
@Optional() @Inject(OUTBOUND_LOOKUP) lookup?: LookupFn,
|
||||
) {
|
||||
this.lookup = lookup ?? defaultLookup;
|
||||
}
|
||||
|
||||
/** Caches the parsed policy and logs each invalid value once. */
|
||||
private resolvePolicy(): OutboundNetworkPolicy {
|
||||
const raw = this.environmentService.getAllowedPrivateNetworks();
|
||||
if (this.cachedPolicy?.raw !== raw) {
|
||||
const policy = parseOutboundNetworkPolicy(raw);
|
||||
if (policy.invalid) {
|
||||
this.logger.error(
|
||||
`Invalid ALLOWED_PRIVATE_NETWORKS value "${raw}"; refusing every private and reserved destination`,
|
||||
);
|
||||
}
|
||||
this.cachedPolicy = { raw, policy };
|
||||
}
|
||||
return this.cachedPolicy.policy;
|
||||
}
|
||||
|
||||
/** Validates the URL and returns the address used to pin the connection. */
|
||||
async validate(rawUrl: string): Promise<PinnedAddress> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new OutboundUrlError('Destination URL is not a valid URL');
|
||||
}
|
||||
|
||||
const isCloud = this.environmentService.isCloud();
|
||||
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
||||
throw new OutboundUrlError('Destination URL must use http or https');
|
||||
}
|
||||
if (isCloud && url.protocol !== 'https:') {
|
||||
throw new OutboundUrlError('Destination URL must use https');
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new OutboundUrlError('Destination URL must not contain credentials');
|
||||
}
|
||||
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, '');
|
||||
let resolved: ResolvedAddress[];
|
||||
if (isIPv4(hostname) || isIPv6(hostname)) {
|
||||
resolved = [{ address: hostname, family: isIPv4(hostname) ? 4 : 6 }];
|
||||
} else {
|
||||
try {
|
||||
resolved = await this.lookup(hostname);
|
||||
} catch {
|
||||
throw new OutboundUrlError(
|
||||
`Could not resolve destination host "${hostname}"`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (resolved.length === 0) {
|
||||
throw new OutboundUrlError(
|
||||
`Could not resolve destination host "${hostname}"`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
if (isCloud) {
|
||||
const blocked = resolved.find((r) => isPrivateAddress(r.address));
|
||||
if (blocked) {
|
||||
throw new OutboundUrlError(
|
||||
`Destination host "${hostname}" resolves to a private or reserved address (${blocked.address}), which is not allowed`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
const refusal = findRefusal(
|
||||
resolved,
|
||||
effectivePort(url),
|
||||
this.resolvePolicy(),
|
||||
);
|
||||
if (refusal) throw new OutboundUrlError(describeRefusal(hostname, refusal));
|
||||
}
|
||||
|
||||
const pick = resolved[0];
|
||||
return { hostname, address: pick.address, family: pick.family === 6 ? 6 : 4 };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { OutboundAgentFactory } from './outbound-agent.factory';
|
||||
import { OutboundUrlGuard } from './outbound-url.guard';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [OutboundUrlGuard, OutboundAgentFactory],
|
||||
exports: [OutboundUrlGuard, OutboundAgentFactory],
|
||||
})
|
||||
export class OutboundModule {}
|
||||
@@ -10,6 +10,7 @@ export enum QueueName {
|
||||
NOTIFICATION_QUEUE = '{notification-queue}',
|
||||
AUDIT_QUEUE = '{audit-queue}',
|
||||
BASE_QUEUE = '{base-queue}',
|
||||
SIEM_QUEUE = '{siem-queue}',
|
||||
}
|
||||
|
||||
export enum QueueJob {
|
||||
@@ -83,6 +84,9 @@ export enum QueueJob {
|
||||
AUDIT_LOG = 'audit-log',
|
||||
AUDIT_CLEANUP = 'audit-cleanup',
|
||||
|
||||
SIEM_SWEEP = 'siem-sweep',
|
||||
SIEM_DELIVER = 'siem-deliver',
|
||||
|
||||
PDF_EXPORT_TASK = 'pdf-export-task',
|
||||
PDF_EXPORT_CLEANUP = 'pdf-export-cleanup',
|
||||
|
||||
|
||||
@@ -94,6 +94,14 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
|
||||
attempts: 3,
|
||||
},
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.SIEM_QUEUE,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 1,
|
||||
},
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.BASE_QUEUE,
|
||||
defaultJobOptions: {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
OAUTH_REGISTER_THROTTLER,
|
||||
OAUTH_TOKEN_THROTTLER,
|
||||
OAUTH_AUTHORIZE_THROTTLER,
|
||||
SIEM_TEST_THROTTLER,
|
||||
} from './throttler-names';
|
||||
import Redis from 'ioredis';
|
||||
|
||||
@@ -27,6 +28,7 @@ import Redis from 'ioredis';
|
||||
{ name: OAUTH_REGISTER_THROTTLER, ttl: 3_600_000, limit: 10 },
|
||||
{ name: OAUTH_TOKEN_THROTTLER, ttl: 60_000, limit: 60 },
|
||||
{ name: OAUTH_AUTHORIZE_THROTTLER, ttl: 60_000, limit: 30 },
|
||||
{ name: SIEM_TEST_THROTTLER, ttl: 60_000, limit: 10 },
|
||||
],
|
||||
errorMessage: 'Too many requests',
|
||||
storage: new ThrottlerStorageRedisService(
|
||||
|
||||
@@ -3,6 +3,7 @@ export const AI_CHAT_THROTTLER = 'ai-chat';
|
||||
export const OAUTH_REGISTER_THROTTLER = 'oauth-register';
|
||||
export const OAUTH_TOKEN_THROTTLER = 'oauth-token';
|
||||
export const OAUTH_AUTHORIZE_THROTTLER = 'oauth-authorize';
|
||||
export const SIEM_TEST_THROTTLER = 'siem-test';
|
||||
|
||||
// Every named throttler must appear here; spread it in @SkipThrottle and re-enable per name with false.
|
||||
export const ALL_NAMED_THROTTLERS_SKIPPED: Record<string, boolean> = {
|
||||
@@ -11,4 +12,5 @@ export const ALL_NAMED_THROTTLERS_SKIPPED: Record<string, boolean> = {
|
||||
[OAUTH_REGISTER_THROTTLER]: true,
|
||||
[OAUTH_TOKEN_THROTTLER]: true,
|
||||
[OAUTH_AUTHORIZE_THROTTLER]: true,
|
||||
[SIEM_TEST_THROTTLER]: true,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { Section, Text } from 'react-email';
|
||||
import * as React from 'react';
|
||||
import { content, paragraph } from '../css/styles';
|
||||
import { EmailButton, MailBody } from '../partials/partials';
|
||||
|
||||
type Props = {
|
||||
destinationName: string;
|
||||
destinationType: string;
|
||||
lastError: string;
|
||||
failingSince: string;
|
||||
settingsLink: string;
|
||||
};
|
||||
|
||||
export const SiemDestinationDisabledEmail = ({
|
||||
destinationName,
|
||||
destinationType,
|
||||
lastError,
|
||||
failingSince,
|
||||
settingsLink,
|
||||
}: Props) => {
|
||||
return (
|
||||
<MailBody>
|
||||
<Section style={content}>
|
||||
<Text style={paragraph}>Hi there,</Text>
|
||||
<Text style={paragraph}>
|
||||
Your SIEM destination <strong>{destinationName}</strong> (
|
||||
{destinationType}) has been failing since {failingSince} and was
|
||||
disabled after 24 hours of failed deliveries.
|
||||
</Text>
|
||||
<Text style={paragraph}>Last error: {lastError}</Text>
|
||||
<Text style={paragraph}>
|
||||
Your events are kept and delivery resumes from where it stopped when
|
||||
you re-enable it.
|
||||
</Text>
|
||||
</Section>
|
||||
<EmailButton href={settingsLink}>View destination</EmailButton>
|
||||
</MailBody>
|
||||
);
|
||||
};
|
||||
|
||||
export default SiemDestinationDisabledEmail;
|
||||
@@ -0,0 +1,41 @@
|
||||
import { Section, Text } from 'react-email';
|
||||
import * as React from 'react';
|
||||
import { content, paragraph } from '../css/styles';
|
||||
import { EmailButton, MailBody } from '../partials/partials';
|
||||
|
||||
type Props = {
|
||||
destinationName: string;
|
||||
destinationType: string;
|
||||
lastError: string;
|
||||
failingSince: string;
|
||||
settingsLink: string;
|
||||
};
|
||||
|
||||
export const SiemDestinationFailingEmail = ({
|
||||
destinationName,
|
||||
destinationType,
|
||||
lastError,
|
||||
failingSince,
|
||||
settingsLink,
|
||||
}: Props) => {
|
||||
return (
|
||||
<MailBody>
|
||||
<Section style={content}>
|
||||
<Text style={paragraph}>Hi there,</Text>
|
||||
<Text style={paragraph}>
|
||||
Docmost cannot deliver audit events to your SIEM destination{' '}
|
||||
<strong>{destinationName}</strong> ({destinationType}).
|
||||
</Text>
|
||||
<Text style={paragraph}>Last error: {lastError}</Text>
|
||||
<Text style={paragraph}>Failing since {failingSince}.</Text>
|
||||
<Text style={paragraph}>
|
||||
Docmost keeps retrying every 30 minutes. If the destination is still
|
||||
failing 24 hours after it started, it is disabled automatically.
|
||||
</Text>
|
||||
</Section>
|
||||
<EmailButton href={settingsLink}>View destination</EmailButton>
|
||||
</MailBody>
|
||||
);
|
||||
};
|
||||
|
||||
export default SiemDestinationFailingEmail;
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
import { Section, Text } from 'react-email';
|
||||
import * as React from 'react';
|
||||
import { content, paragraph } from '../css/styles';
|
||||
import { EmailButton, MailBody } from '../partials/partials';
|
||||
|
||||
type Props = {
|
||||
destinationName: string;
|
||||
destinationType: string;
|
||||
settingsLink: string;
|
||||
};
|
||||
|
||||
export const SiemDestinationRecoveredEmail = ({
|
||||
destinationName,
|
||||
destinationType,
|
||||
settingsLink,
|
||||
}: Props) => {
|
||||
return (
|
||||
<MailBody>
|
||||
<Section style={content}>
|
||||
<Text style={paragraph}>Hi there,</Text>
|
||||
<Text style={paragraph}>
|
||||
Your SIEM destination <strong>{destinationName}</strong> (
|
||||
{destinationType}) is delivering audit events again.
|
||||
</Text>
|
||||
<Text style={paragraph}>
|
||||
Events buffered during the outage were delivered from where the stream
|
||||
stopped.
|
||||
</Text>
|
||||
</Section>
|
||||
<EmailButton href={settingsLink}>View destination</EmailButton>
|
||||
</MailBody>
|
||||
);
|
||||
};
|
||||
|
||||
export default SiemDestinationRecoveredEmail;
|
||||
Reference in New Issue
Block a user