kundencenter/apps/api/src/ops/backup.ts

210 lines
19 KiB
TypeScript
Raw Normal View History

import { spawn } from 'node:child_process';
import { createReadStream, createWriteStream, existsSync } from 'node:fs';
import { chmod, mkdir, mkdtemp, readdir, readFile, rename, rm, stat, writeFile, copyFile } from 'node:fs/promises';
import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import { join, basename } from 'node:path';
import mysql from 'mysql2/promise';
import { config } from '../core/config.js';
import { query, one } from '../core/db.js';
import { audit, verifyAuditChain } from '../core/audit.js';
import { enqueue } from '../core/jobs.js';
import { fileName, parseName, selectDeletions, type Keep } from './retention.js';
import { getBackupPassword } from './settings.js';
import { buildRemote, loadTargets, friendly, type Remote } from './targets.js';
/** Konfiguration aus /etc/kundencenter/backup.env (nur Namen/Pfade, keine Zugangsdaten der Ziele; die liegen in der rclone-Konfiguration). */
export interface BackupConfig { dir: string; recipient: string | null; identity: string | null; remotes: string[]; rclone: string; rcloneConfig: string | null; keep: Keep; statusFile: string; envDir: string }
export function loadBackupConfig(env = process.env): BackupConfig {
return {
dir: env.BACKUP_DIR ?? '/var/backups/kundencenter', recipient: env.BACKUP_AGE_RECIPIENT || null, identity: env.BACKUP_AGE_IDENTITY || null,
remotes: (env.BACKUP_REMOTES ?? '').split(',').map((s) => s.trim()).filter(Boolean), rclone: env.BACKUP_RCLONE_BIN ?? 'rclone', rcloneConfig: env.RCLONE_CONFIG || null,
keep: { daily: Number(env.BACKUP_KEEP_DAILY ?? 14), weekly: Number(env.BACKUP_KEEP_WEEKLY ?? 8), monthly: Number(env.BACKUP_KEEP_MONTHLY ?? 12) },
statusFile: env.BACKUP_STATUS_FILE ?? '/var/lib/kundencenter/backup-status.json', envDir: env.KC_BACKUP_ENV_DIR ?? '/etc/kundencenter',
};
}
/** Externes Programm ohne Shell starten. Fehlertext enthält nur stderr (ohne Umgebung/Passwörter). */
function run(cmd: string, args: string[], o: { env?: Record<string, string>; stdin?: string; stdinText?: string; stdout?: string; input?: string } = {}): Promise<string> {
return new Promise((resolve, reject) => {
const p = spawn(cmd, args, { env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '/root', ...(o.env ?? {}) }, stdio: [o.stdin || o.stdinText !== undefined ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
let out = ''; let err = '';
if (o.stdout) p.stdout!.pipe(createWriteStream(o.stdout, { mode: 0o600 })); else p.stdout!.on('data', (d) => (out += d));
p.stderr!.on('data', (d) => { err += d; if (err.length > 4000) err = err.slice(-4000); });
if (o.stdin) createReadStream(o.stdin).pipe(p.stdin!); else if (o.stdinText !== undefined) { p.stdin!.on('error', () => undefined); p.stdin!.end(o.stdinText); }
p.on('error', (e) => reject(new Error(`${basename(cmd)} konnte nicht gestartet werden: ${e.message}`)));
p.on('close', (code) => (code === 0 ? resolve(out) : reject(new Error(`${basename(cmd)} Fehler (Exit ${code}): ${err.trim().split('\n').slice(-3).join(' | ')}`))));
});
}
/** Passwortverschlüsselung mit gpg (AES-256, Integritätsschutz). Das Passwort geht über stdin, nie über Argumente. Eigenes Arbeitsverzeichnis, kein Agent-Rückstand. */
async function gpgEncrypt(work: string, input: string, output: string, password: string): Promise<void> {
const home = await mkdtemp(join(work, 'gnupg-')); await chmod(home, 0o700);
await run('gpg', ['--homedir', home, '--batch', '--yes', '--pinentry-mode', 'loopback', '--passphrase-fd', '0', '--symmetric', '--cipher-algo', 'AES256', '--s2k-mode', '3', '--s2k-count', '65011712', '--s2k-digest-algo', 'SHA512', '-o', output, input], { stdinText: password });
}
async function gpgDecrypt(work: string, input: string, output: string, password: string): Promise<void> {
const home = await mkdtemp(join(work, 'gnupg-')); await chmod(home, 0o700);
await run('gpg', ['--homedir', home, '--batch', '--yes', '--pinentry-mode', 'loopback', '--passphrase-fd', '0', '-d', '-o', output, input], { stdinText: password });
}
const sha256 = async (f: string) => { const h = createHash('sha256'); for await (const c of createReadStream(f)) h.update(c); return h.digest('hex'); };
const dbEnv = () => ({ MYSQL_PWD: config.db.password });
const dbArgs = () => ['-h', config.db.host, '-P', String(config.db.port), '-u', config.db.user];
const rcloneArgs = (c: BackupConfig) => (c.rcloneConfig ? ['--config', c.rcloneConfig] : []);
export interface RunSummary { at: string; ok: boolean; file?: string; sizeBytes?: number; durationMs: number; label?: string; error?: string }
export interface Status {
running?: { action: 'backup' | 'restore-test'; since: string } | null;
history?: RunSummary[];
lastRun?: { at: string; ok: boolean; file?: string; sizeBytes?: number; durationMs: number; label?: string; targets: { name: string; ok: boolean; error?: string }[]; error?: string };
lastRestoreTest?: { at: string; ok: boolean; file?: string; durationMs: number; checks: Record<string, string | number | boolean>; error?: string };
}
export async function readStatus(c: BackupConfig): Promise<Status> { try { return JSON.parse(await readFile(c.statusFile, 'utf8')); } catch { return {}; } }
async function writeStatus(c: BackupConfig, patch: Partial<Status>): Promise<void> {
const s = { ...(await readStatus(c)), ...patch };
if (patch.lastRun) { const r = patch.lastRun; s.history = [{ at: r.at, ok: r.ok, file: r.file, sizeBytes: r.sizeBytes, durationMs: r.durationMs, label: r.label, error: r.error }, ...((await readStatus(c)).history ?? [])].slice(0, 30); } const tmp = `${c.statusFile}.tmp`;
await mkdir(join(c.statusFile, '..'), { recursive: true }); await writeFile(tmp, JSON.stringify(s, null, 2), { mode: 0o644 }); await rename(tmp, c.statusFile);
}
const alertOnce = (event: string, detail: string) => enqueue('discord.notify', { event, detail: detail.slice(0, 200) }, { idempotencyKey: `${event}:${new Date().toISOString().slice(0, 13)}` }).catch(() => undefined);
const COUNT_TABLES = ['users', 'organizations', 'memberships', 'orders', 'contracts', 'products', 'resources', 'connector_instances', 'audit_events', 'jobs'];
async function tableCounts(q: (sql: string) => Promise<any[]>): Promise<Record<string, number>> {
const out: Record<string, number> = {};
for (const t of COUNT_TABLES) out[t] = Number((await q(`SELECT COUNT(*) AS n FROM \`${t}\``))[0].n);
return out;
}
/** Erstellt ein verschlüsseltes Backup (DB-Dump + Konfiguration/Schlüssel), lädt es zu allen Zielen hoch und räumt nach Aufbewahrungsregeln auf. */
export async function runBackup(c: BackupConfig, now = new Date()): Promise<NonNullable<Status['lastRun']>> {
const t0 = Date.now(); const targets: { name: string; ok: boolean; error?: string }[] = []; const dests: { label: string; remote: string; env: Record<string, string> }[] = []; const built: Remote[] = [];
const pw = await getBackupPassword().catch(() => null); const name = fileName(now, pw ? 'gpg' : 'age');
await mkdir(c.dir, { recursive: true, mode: 0o700 });
await writeStatus(c, { running: { action: 'backup', since: now.toISOString() } }).catch(() => undefined);
const work = await mkdtemp(join(c.dir, '.work-'));
try {
await mkdir(join(work, 'config'));
await run('mysqldump', [...dbArgs(), '--single-transaction', '--routines', '--triggers', '--events', '--no-tablespaces', '--default-character-set=utf8mb4', config.db.database], { env: dbEnv(), stdout: join(work, 'db.sql') });
// Konfiguration inkl. Master-Schlüssel (ohne den privaten Backup-Schlüssel!): ohne KC_SECRET_KEY sind TOTP-/Connector-Geheimnisse unlesbar
// Nur die Dateien, die das Kundencenter zum Betrieb braucht (nicht z. B. Plane-Zugang, nie den privaten Backup-Schlüssel)
for (const f of ['app.env', 'db.env', 'discord.env']) if (existsSync(join(c.envDir, f))) await copyFile(join(c.envDir, f), join(work, 'config', f));
const counts = await tableCounts((sql) => query(sql));
const migrations = (await query('SELECT name FROM schema_migrations ORDER BY name')).map((r) => r.name as string);
await writeFile(join(work, 'manifest.json'), JSON.stringify({ version: 1, createdAt: now.toISOString(), database: config.db.database, counts, migrations, dumpSha256: await sha256(join(work, 'db.sql')) }, null, 2));
await run('tar', ['-czf', join(work, 'archive.tar.gz'), '-C', work, 'db.sql', 'config', 'manifest.json']);
const out = join(c.dir, name);
if (pw) await gpgEncrypt(work, join(work, 'archive.tar.gz'), out, pw.password);
else if (c.recipient) await run('age', ['-r', c.recipient, '-o', out, join(work, 'archive.tar.gz')]);
else throw new Error('Keine Verschlüsselung eingerichtet: bitte unter Einstellungen → Backup ein Passwort festlegen.');
await chmod(out, 0o600); // nur Besitzer
const size = (await stat(out)).size;
if (size < 512) throw new Error('Backup-Datei ist unplausibel klein');
await writeFile(`${out}.sha256`, `${await sha256(out)} ${name}\n`, { mode: 0o600 });
targets.push({ name: `lokal (${c.dir})`, ok: true });
// Ziele: in der Oberfläche definierte (Datenbank) plus ggf. Altbestand aus BACKUP_REMOTES
try { for (const t of await loadTargets(true)) { try { const r = await buildRemote(t, c.rclone); built.push(r); dests.push({ label: t.name, remote: r.remote, env: r.env }); } catch (e) { targets.push({ name: t.name, ok: false, error: friendly(e) }); } } }
catch (e) { targets.push({ name: 'Ziele laden', ok: false, error: friendly(e) }); }
for (const r of c.remotes) dests.push({ label: r, remote: r, env: c.rcloneConfig ? { RCLONE_CONFIG: c.rcloneConfig } : {} });
const okDests: typeof dests = [];
for (const d of dests) {
try {
const dest = `${d.remote.replace(/\/+$/, '')}/${name}`;
await run(c.rclone, ['copyto', out, dest, '--retries', '3', '--low-level-retries', '5', '--timeout', '120s', '--contimeout', '30s'], { env: d.env });
const ls = JSON.parse(await run(c.rclone, ['lsjson', dest], { env: d.env })) as { Size: number }[];
if (ls[0]?.Size !== size) throw new Error(`Größe am Ziel weicht ab (${ls[0]?.Size ?? 'fehlt'} statt ${size})`);
await run(c.rclone, ['copyto', `${out}.sha256`, `${dest}.sha256`], { env: d.env });
targets.push({ name: d.label, ok: true }); okDests.push(d);
} catch (e) { targets.push({ name: d.label, ok: false, error: friendly(e) }); }
}
await applyRetention(c, now, okDests, targets);
const failed = targets.filter((t) => !t.ok);
const res = { at: now.toISOString(), ok: failed.length === 0, file: name, sizeBytes: size, durationMs: Date.now() - t0, label: parseName(name)?.label, targets, ...(failed.length ? { error: `${failed.length} Ziel(e) fehlgeschlagen` } : {}) };
await writeStatus(c, { lastRun: res, running: null });
await audit({ actorType: 'system', action: 'backup.run', resourceType: 'backup', resourceId: name, result: res.ok ? 'success' : 'failure', errorClass: res.ok ? undefined : 'target_failed', after: { sizeBytes: size, targets: targets.map((t) => ({ name: t.name, ok: t.ok })) } }).catch(() => undefined);
if (!res.ok) await alertOnce('backup.failed', res.error ?? 'Ziel fehlgeschlagen');
return res;
} catch (e) {
const res = { at: now.toISOString(), ok: false, durationMs: Date.now() - t0, targets, error: (e as Error).message.slice(0, 400) };
await writeStatus(c, { lastRun: res, running: null }).catch(() => undefined);
await audit({ actorType: 'system', action: 'backup.run', resourceType: 'backup', result: 'failure', errorClass: 'backup_error' }).catch(() => undefined);
await alertOnce('backup.failed', res.error);
return res;
} finally { await rm(work, { recursive: true, force: true }); for (const r of built) await r.cleanup().catch(() => undefined); }
}
async function applyRetention(c: BackupConfig, now: Date, dests: { label: string; remote: string; env: Record<string, string> }[], targets: { name: string; ok: boolean; error?: string }[]): Promise<void> {
const localNames = (await readdir(c.dir)).filter((n) => parseName(n));
for (const n of selectDeletions(localNames, now, c.keep)) { await rm(join(c.dir, n), { force: true }); await rm(join(c.dir, `${n}.sha256`), { force: true }); }
for (const d of dests) {
const t = targets.find((x) => x.name === d.label);
try {
const base = d.remote.replace(/\/+$/, '');
const names = (await run(c.rclone, ['lsf', d.remote, '--files-only'], { env: d.env })).split('\n').map((x) => x.trim()).filter(Boolean);
for (const n of selectDeletions(names.filter((x) => parseName(x)), now, c.keep)) { await run(c.rclone, ['deletefile', `${base}/${n}`], { env: d.env }); await run(c.rclone, ['deletefile', `${base}/${n}.sha256`], { env: d.env }).catch(() => undefined); }
} catch (e) { if (t) t.error = `Aufräumen fehlgeschlagen: ${friendly(e)}`; }
}
}
/** Neueste lokale Sicherung finden. */
export async function latestLocal(c: BackupConfig): Promise<string | null> {
const names = (await readdir(c.dir)).map(parseName).filter((x): x is NonNullable<ReturnType<typeof parseName>> => !!x).sort((a, b) => b.at.getTime() - a.at.getTime());
return names[0]?.name ?? null;
}
/**
* Wiederherstellungstest: entschlüsselt eine Sicherung, spielt sie in eine Wegwerf-Datenbank ein und prüft Prüfsumme, Zeilenzahlen,
* Migrationen, Audit-Hash-Kette und ob die gesicherten Geheimnisse mit dem gesicherten Master-Schlüssel entschlüsselbar sind. Produktivdaten bleiben unberührt.
*/
export async function runRestoreTest(c: BackupConfig, file?: string): Promise<NonNullable<Status['lastRestoreTest']>> {
const t0 = Date.now(); const checks: Record<string, string | number | boolean> = {}; const scratch = `${config.db.database}_restoretest`;
const work = await mkdtemp(join(tmpdir(), 'kc-restore-'));
await writeStatus(c, { running: { action: 'restore-test', since: new Date().toISOString() } }).catch(() => undefined);
let name = file;
try {
name = name ?? (await latestLocal(c)) ?? undefined; if (!name) throw new Error('Keine Sicherung gefunden');
const src = join(c.dir, name); checks.datei = name;
const expect = (await readFile(`${src}.sha256`, 'utf8')).split(/\s+/)[0]; checks.pruefsumme = (await sha256(src)) === expect;
if (!checks.pruefsumme) throw new Error('Prüfsumme der Sicherung stimmt nicht');
if (name.endsWith('.gpg')) {
const pw = await getBackupPassword(); if (!pw) throw new Error('Kein Backup-Passwort gespeichert: die Sicherung ist passwortverschlüsselt.');
try { await gpgDecrypt(work, src, join(work, 'archive.tar.gz'), pw.password); } catch { throw new Error('Entschlüsselung fehlgeschlagen: Das gespeicherte Passwort passt nicht zu dieser Sicherung (wurde das Passwort inzwischen geändert?).'); }
} else {
if (!c.identity || !existsSync(c.identity)) throw new Error('Privater Backup-Schlüssel (BACKUP_AGE_IDENTITY) nicht vorhanden');
await run('age', ['-d', '-i', c.identity, '-o', join(work, 'archive.tar.gz'), src]);
}
await run('tar', ['-xzf', join(work, 'archive.tar.gz'), '-C', work]);
const manifest = JSON.parse(await readFile(join(work, 'manifest.json'), 'utf8')); checks.dumpPruefsumme = (await sha256(join(work, 'db.sql'))) === manifest.dumpSha256;
if (!checks.dumpPruefsumme) throw new Error('Prüfsumme des Datenbank-Dumps stimmt nicht');
const admin = await mysql.createConnection({ host: config.db.host, port: config.db.port, user: config.db.user, password: config.db.password });
try {
await admin.query(`DROP DATABASE IF EXISTS \`${scratch}\``); await admin.query(`CREATE DATABASE \`${scratch}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
} finally { await admin.end(); }
await run('mysql', [...dbArgs(), scratch], { env: dbEnv(), stdin: join(work, 'db.sql') });
const conn = await mysql.createConnection({ host: config.db.host, port: config.db.port, user: config.db.user, password: config.db.password, database: scratch, timezone: 'Z' }); // UTC wie im Betrieb, sonst stimmt die Hash-Kette der Zeitstempel nicht
try {
const q = async (sql: string) => (await conn.query(sql))[0] as any[];
const counts = await tableCounts(q); const mism = Object.entries(manifest.counts as Record<string, number>).filter(([t, n]) => counts[t] !== n);
checks.zeilenzahlen = mism.length === 0; if (mism.length) throw new Error(`Zeilenzahlen weichen ab: ${mism.map(([t]) => t).join(', ')}`);
const mig = (await q('SELECT name FROM schema_migrations ORDER BY name')).map((r) => r.name); checks.migrationen = JSON.stringify(mig) === JSON.stringify(manifest.migrations); if (!checks.migrationen) throw new Error('Migrationen weichen ab');
const chain = await verifyAuditChain(q); checks.auditKette = chain.brokenAt === null; checks.auditEintraege = chain.checked; if (chain.brokenAt !== null) throw new Error(`Audit-Kette defekt ab Eintrag ${chain.brokenAt}`);
// Geheimnisse mit dem GESICHERTEN Schlüssel entschlüsseln (beweist, dass die Schlüsselsicherung brauchbar ist)
const key = /^KC_SECRET_KEY=(.+)$/m.exec(await readFile(join(work, 'config', 'app.env'), 'utf8'))?.[1];
if (!key) throw new Error('Master-Schlüssel fehlt in der Sicherung');
const { createDecipheriv } = await import('node:crypto');
const dec = (blob: string) => { const [v, iv, tag, ct] = blob.split(':'); const d = createDecipheriv('aes-256-gcm', Buffer.from(key, 'base64'), Buffer.from(iv!, 'base64url')); d.setAuthTag(Buffer.from(tag!, 'base64url')); return Buffer.concat([d.update(Buffer.from(ct!, 'base64url')), d.final()]).length > 0 && v === 'v1'; };
const sample = [...(await q('SELECT secret_enc AS s FROM mfa_totp LIMIT 3')), ...(await q('SELECT secrets_enc AS s FROM connector_instances WHERE secrets_enc IS NOT NULL LIMIT 3'))];
checks.geheimnisseGeprueft = sample.length; checks.geheimnisseOk = sample.every((r) => dec(r.s)); if (!checks.geheimnisseOk) throw new Error('Gesicherte Geheimnisse sind mit dem gesicherten Schlüssel nicht entschlüsselbar');
} finally { await conn.end(); }
const res = { at: new Date().toISOString(), ok: true, file: name, durationMs: Date.now() - t0, checks };
await writeStatus(c, { lastRestoreTest: res, running: null });
await audit({ actorType: 'system', action: 'backup.restore_test', resourceType: 'backup', resourceId: name, result: 'success' }).catch(() => undefined);
return res;
} catch (e) {
const res = { at: new Date().toISOString(), ok: false, file: name, durationMs: Date.now() - t0, checks, error: (e as Error).message.slice(0, 400) };
await writeStatus(c, { lastRestoreTest: res, running: null }).catch(() => undefined);
await audit({ actorType: 'system', action: 'backup.restore_test', resourceType: 'backup', resourceId: name, result: 'failure', errorClass: 'restore_test_failed' }).catch(() => undefined);
await alertOnce('restoretest.failed', res.error);
return res;
} finally {
await rm(work, { recursive: true, force: true });
try { const a = await mysql.createConnection({ host: config.db.host, port: config.db.port, user: config.db.user, password: config.db.password }); await a.query(`DROP DATABASE IF EXISTS \`${scratch}\``); await a.end(); } catch { /* Aufräumen best effort */ }
}
}