import { spawn } from 'node:child_process'; import { createReadStream, createWriteStream, existsSync } from 'node:fs'; import { chmod, mkdir, mkdtemp, readdir, readFile, rename, rm, stat, writeFile, copyFile } from 'node:fs/promises'; import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { join, basename } from 'node:path'; import mysql from 'mysql2/promise'; import { config } from '../core/config.js'; import { query, one } from '../core/db.js'; import { audit, verifyAuditChain } from '../core/audit.js'; import { enqueue } from '../core/jobs.js'; import { fileName, parseName, selectDeletions, type Keep } from './retention.js'; import { getBackupPassword } from './settings.js'; import { buildRemote, loadTargets, friendly, type Remote } from './targets.js'; /** Konfiguration aus /etc/kundencenter/backup.env (nur Namen/Pfade, keine Zugangsdaten der Ziele; die liegen in der rclone-Konfiguration). */ export interface BackupConfig { dir: string; recipient: string | null; identity: string | null; remotes: string[]; rclone: string; rcloneConfig: string | null; keep: Keep; statusFile: string; envDir: string } export function loadBackupConfig(env = process.env): BackupConfig { return { dir: env.BACKUP_DIR ?? '/var/backups/kundencenter', recipient: env.BACKUP_AGE_RECIPIENT || null, identity: env.BACKUP_AGE_IDENTITY || null, remotes: (env.BACKUP_REMOTES ?? '').split(',').map((s) => s.trim()).filter(Boolean), rclone: env.BACKUP_RCLONE_BIN ?? 'rclone', rcloneConfig: env.RCLONE_CONFIG || null, keep: { daily: Number(env.BACKUP_KEEP_DAILY ?? 14), weekly: Number(env.BACKUP_KEEP_WEEKLY ?? 8), monthly: Number(env.BACKUP_KEEP_MONTHLY ?? 12) }, statusFile: env.BACKUP_STATUS_FILE ?? '/var/lib/kundencenter/backup-status.json', envDir: env.KC_BACKUP_ENV_DIR ?? '/etc/kundencenter', }; } /** Externes Programm ohne Shell starten. Fehlertext enthält nur stderr (ohne Umgebung/Passwörter). */ function run(cmd: string, args: string[], o: { env?: Record; stdin?: string; stdinText?: string; stdout?: string; input?: string } = {}): Promise { return new Promise((resolve, reject) => { const p = spawn(cmd, args, { env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '/root', ...(o.env ?? {}) }, stdio: [o.stdin || o.stdinText !== undefined ? 'pipe' : 'ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; if (o.stdout) p.stdout!.pipe(createWriteStream(o.stdout, { mode: 0o600 })); else p.stdout!.on('data', (d) => (out += d)); p.stderr!.on('data', (d) => { err += d; if (err.length > 4000) err = err.slice(-4000); }); if (o.stdin) createReadStream(o.stdin).pipe(p.stdin!); else if (o.stdinText !== undefined) { p.stdin!.on('error', () => undefined); p.stdin!.end(o.stdinText); } p.on('error', (e) => reject(new Error(`${basename(cmd)} konnte nicht gestartet werden: ${e.message}`))); p.on('close', (code) => (code === 0 ? resolve(out) : reject(new Error(`${basename(cmd)} Fehler (Exit ${code}): ${err.trim().split('\n').slice(-3).join(' | ')}`)))); }); } /** Passwortverschlüsselung mit gpg (AES-256, Integritätsschutz). Das Passwort geht über stdin, nie über Argumente. Eigenes Arbeitsverzeichnis, kein Agent-Rückstand. */ async function gpgEncrypt(work: string, input: string, output: string, password: string): Promise { const home = await mkdtemp(join(work, 'gnupg-')); await chmod(home, 0o700); await run('gpg', ['--homedir', home, '--batch', '--yes', '--pinentry-mode', 'loopback', '--passphrase-fd', '0', '--symmetric', '--cipher-algo', 'AES256', '--s2k-mode', '3', '--s2k-count', '65011712', '--s2k-digest-algo', 'SHA512', '-o', output, input], { stdinText: password }); } async function gpgDecrypt(work: string, input: string, output: string, password: string): Promise { const home = await mkdtemp(join(work, 'gnupg-')); await chmod(home, 0o700); await run('gpg', ['--homedir', home, '--batch', '--yes', '--pinentry-mode', 'loopback', '--passphrase-fd', '0', '-d', '-o', output, input], { stdinText: password }); } const sha256 = async (f: string) => { const h = createHash('sha256'); for await (const c of createReadStream(f)) h.update(c); return h.digest('hex'); }; const dbEnv = () => ({ MYSQL_PWD: config.db.password }); const dbArgs = () => ['-h', config.db.host, '-P', String(config.db.port), '-u', config.db.user]; const rcloneArgs = (c: BackupConfig) => (c.rcloneConfig ? ['--config', c.rcloneConfig] : []); export interface RunSummary { at: string; ok: boolean; file?: string; sizeBytes?: number; durationMs: number; label?: string; error?: string } export interface Status { running?: { action: 'backup' | 'restore-test'; since: string } | null; history?: RunSummary[]; lastRun?: { at: string; ok: boolean; file?: string; sizeBytes?: number; durationMs: number; label?: string; targets: { name: string; ok: boolean; error?: string }[]; error?: string }; lastRestoreTest?: { at: string; ok: boolean; file?: string; durationMs: number; checks: Record; error?: string }; } export async function readStatus(c: BackupConfig): Promise { try { return JSON.parse(await readFile(c.statusFile, 'utf8')); } catch { return {}; } } async function writeStatus(c: BackupConfig, patch: Partial): Promise { const s = { ...(await readStatus(c)), ...patch }; if (patch.lastRun) { const r = patch.lastRun; s.history = [{ at: r.at, ok: r.ok, file: r.file, sizeBytes: r.sizeBytes, durationMs: r.durationMs, label: r.label, error: r.error }, ...((await readStatus(c)).history ?? [])].slice(0, 30); } const tmp = `${c.statusFile}.tmp`; await mkdir(join(c.statusFile, '..'), { recursive: true }); await writeFile(tmp, JSON.stringify(s, null, 2), { mode: 0o644 }); await rename(tmp, c.statusFile); } const alertOnce = (event: string, detail: string) => enqueue('discord.notify', { event, detail: detail.slice(0, 200) }, { idempotencyKey: `${event}:${new Date().toISOString().slice(0, 13)}` }).catch(() => undefined); const COUNT_TABLES = ['users', 'organizations', 'memberships', 'orders', 'contracts', 'products', 'resources', 'connector_instances', 'audit_events', 'jobs']; async function tableCounts(q: (sql: string) => Promise): Promise> { const out: Record = {}; for (const t of COUNT_TABLES) out[t] = Number((await q(`SELECT COUNT(*) AS n FROM \`${t}\``))[0].n); return out; } /** Erstellt ein verschlüsseltes Backup (DB-Dump + Konfiguration/Schlüssel), lädt es zu allen Zielen hoch und räumt nach Aufbewahrungsregeln auf. */ export async function runBackup(c: BackupConfig, now = new Date()): Promise> { const t0 = Date.now(); const targets: { name: string; ok: boolean; error?: string }[] = []; const dests: { label: string; remote: string; env: Record }[] = []; const built: Remote[] = []; const pw = await getBackupPassword().catch(() => null); const name = fileName(now, pw ? 'gpg' : 'age'); await mkdir(c.dir, { recursive: true, mode: 0o700 }); await writeStatus(c, { running: { action: 'backup', since: now.toISOString() } }).catch(() => undefined); const work = await mkdtemp(join(c.dir, '.work-')); try { await mkdir(join(work, 'config')); await run('mysqldump', [...dbArgs(), '--single-transaction', '--routines', '--triggers', '--events', '--no-tablespaces', '--default-character-set=utf8mb4', config.db.database], { env: dbEnv(), stdout: join(work, 'db.sql') }); // Konfiguration inkl. Master-Schlüssel (ohne den privaten Backup-Schlüssel!): ohne KC_SECRET_KEY sind TOTP-/Connector-Geheimnisse unlesbar // Nur die Dateien, die das Kundencenter zum Betrieb braucht (nicht z. B. Plane-Zugang, nie den privaten Backup-Schlüssel) for (const f of ['app.env', 'db.env', 'discord.env']) if (existsSync(join(c.envDir, f))) await copyFile(join(c.envDir, f), join(work, 'config', f)); const counts = await tableCounts((sql) => query(sql)); const migrations = (await query('SELECT name FROM schema_migrations ORDER BY name')).map((r) => r.name as string); await writeFile(join(work, 'manifest.json'), JSON.stringify({ version: 1, createdAt: now.toISOString(), database: config.db.database, counts, migrations, dumpSha256: await sha256(join(work, 'db.sql')) }, null, 2)); await run('tar', ['-czf', join(work, 'archive.tar.gz'), '-C', work, 'db.sql', 'config', 'manifest.json']); const out = join(c.dir, name); if (pw) await gpgEncrypt(work, join(work, 'archive.tar.gz'), out, pw.password); else if (c.recipient) await run('age', ['-r', c.recipient, '-o', out, join(work, 'archive.tar.gz')]); else throw new Error('Keine Verschlüsselung eingerichtet: bitte unter Einstellungen → Backup ein Passwort festlegen.'); await chmod(out, 0o600); // nur Besitzer const size = (await stat(out)).size; if (size < 512) throw new Error('Backup-Datei ist unplausibel klein'); await writeFile(`${out}.sha256`, `${await sha256(out)} ${name}\n`, { mode: 0o600 }); targets.push({ name: `lokal (${c.dir})`, ok: true }); // Ziele: in der Oberfläche definierte (Datenbank) plus ggf. Altbestand aus BACKUP_REMOTES try { for (const t of await loadTargets(true)) { try { const r = await buildRemote(t, c.rclone); built.push(r); dests.push({ label: t.name, remote: r.remote, env: r.env }); } catch (e) { targets.push({ name: t.name, ok: false, error: friendly(e) }); } } } catch (e) { targets.push({ name: 'Ziele laden', ok: false, error: friendly(e) }); } for (const r of c.remotes) dests.push({ label: r, remote: r, env: c.rcloneConfig ? { RCLONE_CONFIG: c.rcloneConfig } : {} }); const okDests: typeof dests = []; for (const d of dests) { try { const dest = `${d.remote.replace(/\/+$/, '')}/${name}`; await run(c.rclone, ['copyto', out, dest, '--retries', '3', '--low-level-retries', '5', '--timeout', '120s', '--contimeout', '30s'], { env: d.env }); const ls = JSON.parse(await run(c.rclone, ['lsjson', dest], { env: d.env })) as { Size: number }[]; if (ls[0]?.Size !== size) throw new Error(`Größe am Ziel weicht ab (${ls[0]?.Size ?? 'fehlt'} statt ${size})`); await run(c.rclone, ['copyto', `${out}.sha256`, `${dest}.sha256`], { env: d.env }); targets.push({ name: d.label, ok: true }); okDests.push(d); } catch (e) { targets.push({ name: d.label, ok: false, error: friendly(e) }); } } await applyRetention(c, now, okDests, targets); const failed = targets.filter((t) => !t.ok); const res = { at: now.toISOString(), ok: failed.length === 0, file: name, sizeBytes: size, durationMs: Date.now() - t0, label: parseName(name)?.label, targets, ...(failed.length ? { error: `${failed.length} Ziel(e) fehlgeschlagen` } : {}) }; await writeStatus(c, { lastRun: res, running: null }); await audit({ actorType: 'system', action: 'backup.run', resourceType: 'backup', resourceId: name, result: res.ok ? 'success' : 'failure', errorClass: res.ok ? undefined : 'target_failed', after: { sizeBytes: size, targets: targets.map((t) => ({ name: t.name, ok: t.ok })) } }).catch(() => undefined); if (!res.ok) await alertOnce('backup.failed', res.error ?? 'Ziel fehlgeschlagen'); return res; } catch (e) { const res = { at: now.toISOString(), ok: false, durationMs: Date.now() - t0, targets, error: (e as Error).message.slice(0, 400) }; await writeStatus(c, { lastRun: res, running: null }).catch(() => undefined); await audit({ actorType: 'system', action: 'backup.run', resourceType: 'backup', result: 'failure', errorClass: 'backup_error' }).catch(() => undefined); await alertOnce('backup.failed', res.error); return res; } finally { await rm(work, { recursive: true, force: true }); for (const r of built) await r.cleanup().catch(() => undefined); } } async function applyRetention(c: BackupConfig, now: Date, dests: { label: string; remote: string; env: Record }[], targets: { name: string; ok: boolean; error?: string }[]): Promise { const localNames = (await readdir(c.dir)).filter((n) => parseName(n)); for (const n of selectDeletions(localNames, now, c.keep)) { await rm(join(c.dir, n), { force: true }); await rm(join(c.dir, `${n}.sha256`), { force: true }); } for (const d of dests) { const t = targets.find((x) => x.name === d.label); try { const base = d.remote.replace(/\/+$/, ''); const names = (await run(c.rclone, ['lsf', d.remote, '--files-only'], { env: d.env })).split('\n').map((x) => x.trim()).filter(Boolean); for (const n of selectDeletions(names.filter((x) => parseName(x)), now, c.keep)) { await run(c.rclone, ['deletefile', `${base}/${n}`], { env: d.env }); await run(c.rclone, ['deletefile', `${base}/${n}.sha256`], { env: d.env }).catch(() => undefined); } } catch (e) { if (t) t.error = `Aufräumen fehlgeschlagen: ${friendly(e)}`; } } } /** Neueste lokale Sicherung finden. */ export async function latestLocal(c: BackupConfig): Promise { const names = (await readdir(c.dir)).map(parseName).filter((x): x is NonNullable> => !!x).sort((a, b) => b.at.getTime() - a.at.getTime()); return names[0]?.name ?? null; } /** * Wiederherstellungstest: entschlüsselt eine Sicherung, spielt sie in eine Wegwerf-Datenbank ein und prüft Prüfsumme, Zeilenzahlen, * Migrationen, Audit-Hash-Kette und ob die gesicherten Geheimnisse mit dem gesicherten Master-Schlüssel entschlüsselbar sind. Produktivdaten bleiben unberührt. */ export async function runRestoreTest(c: BackupConfig, file?: string): Promise> { const t0 = Date.now(); const checks: Record = {}; const scratch = `${config.db.database}_restoretest`; const work = await mkdtemp(join(tmpdir(), 'kc-restore-')); await writeStatus(c, { running: { action: 'restore-test', since: new Date().toISOString() } }).catch(() => undefined); let name = file; try { name = name ?? (await latestLocal(c)) ?? undefined; if (!name) throw new Error('Keine Sicherung gefunden'); const src = join(c.dir, name); checks.datei = name; const expect = (await readFile(`${src}.sha256`, 'utf8')).split(/\s+/)[0]; checks.pruefsumme = (await sha256(src)) === expect; if (!checks.pruefsumme) throw new Error('Prüfsumme der Sicherung stimmt nicht'); if (name.endsWith('.gpg')) { const pw = await getBackupPassword(); if (!pw) throw new Error('Kein Backup-Passwort gespeichert: die Sicherung ist passwortverschlüsselt.'); try { await gpgDecrypt(work, src, join(work, 'archive.tar.gz'), pw.password); } catch { throw new Error('Entschlüsselung fehlgeschlagen: Das gespeicherte Passwort passt nicht zu dieser Sicherung (wurde das Passwort inzwischen geändert?).'); } } else { if (!c.identity || !existsSync(c.identity)) throw new Error('Privater Backup-Schlüssel (BACKUP_AGE_IDENTITY) nicht vorhanden'); await run('age', ['-d', '-i', c.identity, '-o', join(work, 'archive.tar.gz'), src]); } await run('tar', ['-xzf', join(work, 'archive.tar.gz'), '-C', work]); const manifest = JSON.parse(await readFile(join(work, 'manifest.json'), 'utf8')); checks.dumpPruefsumme = (await sha256(join(work, 'db.sql'))) === manifest.dumpSha256; if (!checks.dumpPruefsumme) throw new Error('Prüfsumme des Datenbank-Dumps stimmt nicht'); const admin = await mysql.createConnection({ host: config.db.host, port: config.db.port, user: config.db.user, password: config.db.password }); try { await admin.query(`DROP DATABASE IF EXISTS \`${scratch}\``); await admin.query(`CREATE DATABASE \`${scratch}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`); } finally { await admin.end(); } await run('mysql', [...dbArgs(), scratch], { env: dbEnv(), stdin: join(work, 'db.sql') }); const conn = await mysql.createConnection({ host: config.db.host, port: config.db.port, user: config.db.user, password: config.db.password, database: scratch, timezone: 'Z' }); // UTC wie im Betrieb, sonst stimmt die Hash-Kette der Zeitstempel nicht try { const q = async (sql: string) => (await conn.query(sql))[0] as any[]; const counts = await tableCounts(q); const mism = Object.entries(manifest.counts as Record).filter(([t, n]) => counts[t] !== n); checks.zeilenzahlen = mism.length === 0; if (mism.length) throw new Error(`Zeilenzahlen weichen ab: ${mism.map(([t]) => t).join(', ')}`); const mig = (await q('SELECT name FROM schema_migrations ORDER BY name')).map((r) => r.name); checks.migrationen = JSON.stringify(mig) === JSON.stringify(manifest.migrations); if (!checks.migrationen) throw new Error('Migrationen weichen ab'); const chain = await verifyAuditChain(q); checks.auditKette = chain.brokenAt === null; checks.auditEintraege = chain.checked; if (chain.brokenAt !== null) throw new Error(`Audit-Kette defekt ab Eintrag ${chain.brokenAt}`); // Geheimnisse mit dem GESICHERTEN Schlüssel entschlüsseln (beweist, dass die Schlüsselsicherung brauchbar ist) const key = /^KC_SECRET_KEY=(.+)$/m.exec(await readFile(join(work, 'config', 'app.env'), 'utf8'))?.[1]; if (!key) throw new Error('Master-Schlüssel fehlt in der Sicherung'); const { createDecipheriv } = await import('node:crypto'); const dec = (blob: string) => { const [v, iv, tag, ct] = blob.split(':'); const d = createDecipheriv('aes-256-gcm', Buffer.from(key, 'base64'), Buffer.from(iv!, 'base64url')); d.setAuthTag(Buffer.from(tag!, 'base64url')); return Buffer.concat([d.update(Buffer.from(ct!, 'base64url')), d.final()]).length > 0 && v === 'v1'; }; const sample = [...(await q('SELECT secret_enc AS s FROM mfa_totp LIMIT 3')), ...(await q('SELECT secrets_enc AS s FROM connector_instances WHERE secrets_enc IS NOT NULL LIMIT 3'))]; checks.geheimnisseGeprueft = sample.length; checks.geheimnisseOk = sample.every((r) => dec(r.s)); if (!checks.geheimnisseOk) throw new Error('Gesicherte Geheimnisse sind mit dem gesicherten Schlüssel nicht entschlüsselbar'); } finally { await conn.end(); } const res = { at: new Date().toISOString(), ok: true, file: name, durationMs: Date.now() - t0, checks }; await writeStatus(c, { lastRestoreTest: res, running: null }); await audit({ actorType: 'system', action: 'backup.restore_test', resourceType: 'backup', resourceId: name, result: 'success' }).catch(() => undefined); return res; } catch (e) { const res = { at: new Date().toISOString(), ok: false, file: name, durationMs: Date.now() - t0, checks, error: (e as Error).message.slice(0, 400) }; await writeStatus(c, { lastRestoreTest: res, running: null }).catch(() => undefined); await audit({ actorType: 'system', action: 'backup.restore_test', resourceType: 'backup', resourceId: name, result: 'failure', errorClass: 'restore_test_failed' }).catch(() => undefined); await alertOnce('restoretest.failed', res.error); return res; } finally { await rm(work, { recursive: true, force: true }); try { const a = await mysql.createConnection({ host: config.db.host, port: config.db.port, user: config.db.user, password: config.db.password }); await a.query(`DROP DATABASE IF EXISTS \`${scratch}\``); await a.end(); } catch { /* Aufräumen best effort */ } } }