diff --git a/services/api/src/app.module.ts b/services/api/src/app.module.ts index 7f25ba6..5fef0b8 100644 --- a/services/api/src/app.module.ts +++ b/services/api/src/app.module.ts @@ -6,6 +6,7 @@ import { TrendsModule } from './trends/trends.module'; import { FunnelsModule } from './funnels/funnels.module'; import { CohortsModule } from './cohorts/cohorts.module'; import { RevenueModule } from './revenue/revenue.module'; +import { GdprModule } from './gdpr/gdpr.module'; @Module({ imports: [ @@ -34,6 +35,7 @@ import { RevenueModule } from './revenue/revenue.module'; FunnelsModule, CohortsModule, RevenueModule, + GdprModule, ], }) export class AppModule {} diff --git a/services/api/src/entities/deletion-audit-log.entity.ts b/services/api/src/entities/deletion-audit-log.entity.ts new file mode 100644 index 0000000..e28ae52 --- /dev/null +++ b/services/api/src/entities/deletion-audit-log.entity.ts @@ -0,0 +1,153 @@ +import { + Entity, + Column, + PrimaryGeneratedColumn, + CreateDateColumn, + UpdateDateColumn, + Index, +} from 'typeorm'; + +/** + * GDPR deletion request status + */ +export enum DeletionStatus { + /** Deletion request received and queued */ + PENDING = 'PENDING', + /** Deletion job is currently processing */ + PROCESSING = 'PROCESSING', + /** Deletion completed successfully */ + COMPLETED = 'COMPLETED', + /** Deletion failed with errors */ + FAILED = 'FAILED', + /** Deletion partially completed (some tables failed) */ + PARTIAL = 'PARTIAL', +} + +/** + * Per-table deletion result tracking + */ +export interface TableDeletionResult { + table: string; + status: 'success' | 'failed' | 'skipped'; + recordsDeleted?: number; + recordsAnonymized?: number; + error?: string; + startedAt: string; + completedAt: string; + durationMs: number; +} + +/** + * DeletionAuditLog - GDPR Article 17 compliance tracking + * + * Records all user data deletion requests and their execution status. + * Provides full audit trail for GDPR compliance verification. + * + * GDPR Requirements: + * - Article 17: Right to erasure ("right to be forgotten") + * - Article 30: Records of processing activities + * - Recital 82: Demonstrable compliance + * + * Tax Compliance: + * - Revenue data is ANONYMIZED (userId set to NULL) not deleted + * - Aggregate statistics preserved for financial records + * - Individual revenue amounts retained for tax audit requirements + */ +@Entity('deletion_audit_logs') +export class DeletionAuditLog { + @PrimaryGeneratedColumn('uuid') + id!: string; + + @Column('uuid', { name: 'subject_id' }) + @Index() + subjectId!: string; + + @Column({ + type: 'enum', + enum: DeletionStatus, + default: DeletionStatus.PENDING, + }) + @Index() + status!: DeletionStatus; + + @Column('timestamp', { name: 'requested_at', default: () => 'CURRENT_TIMESTAMP' }) + @Index() + requestedAt!: Date; + + @Column('timestamp', { name: 'started_at', nullable: true }) + startedAt?: Date; + + @Column('timestamp', { name: 'completed_at', nullable: true }) + @Index() + completedAt?: Date; + + @Column('int', { name: 'duration_ms', nullable: true }) + durationMs?: number; + + /** + * Per-table deletion results with detailed status + */ + @Column('jsonb', { name: 'table_results', default: [] }) + tableResults!: TableDeletionResult[]; + + /** + * Summary counts for quick verification + */ + @Column('int', { name: 'total_records_deleted', default: 0 }) + totalRecordsDeleted!: number; + + @Column('int', { name: 'total_records_anonymized', default: 0 }) + totalRecordsAnonymized!: number; + + /** + * Queue job ID for correlation with job system + */ + @Column('varchar', { length: 100, name: 'job_id', nullable: true }) + @Index() + jobId?: string; + + /** + * Error details if deletion failed + */ + @Column('jsonb', { name: 'error_details', nullable: true }) + errorDetails?: { + message: string; + stack?: string; + failedTables?: string[]; + partialResults?: TableDeletionResult[]; + }; + + /** + * Verification status - confirms no user data remains + */ + @Column('boolean', { name: 'verified', default: false }) + @Index() + verified!: boolean; + + @Column('timestamp', { name: 'verified_at', nullable: true }) + verifiedAt?: Date; + + /** + * Count of records found during verification (should be 0) + */ + @Column('int', { name: 'verification_records_found', nullable: true }) + verificationRecordsFound?: number; + + /** + * Additional metadata for audit purposes + */ + @Column('jsonb', { nullable: true }) + metadata?: { + requestSource?: 'user' | 'admin' | 'automated'; + requestedBy?: string; + retryCount?: number; + gdprArticle?: string; + notes?: string; + }; + + @CreateDateColumn({ name: 'created_at', type: 'timestamptz' }) + createdAt!: Date; + + @UpdateDateColumn({ name: 'updated_at', type: 'timestamptz' }) + updatedAt!: Date; +} diff --git a/services/api/src/entities/index.ts b/services/api/src/entities/index.ts new file mode 100644 index 0000000..7961571 --- /dev/null +++ b/services/api/src/entities/index.ts @@ -0,0 +1,6 @@ +export { AggregatedMetric, MetricType, TimeGranularity } from './aggregated-metric.entity'; +export { + DeletionAuditLog, + DeletionStatus, + type TableDeletionResult, +} from './deletion-audit-log.entity'; diff --git a/services/api/src/gdpr/data-retention.cron.ts b/services/api/src/gdpr/data-retention.cron.ts new file mode 100644 index 0000000..0c54d9f --- /dev/null +++ b/services/api/src/gdpr/data-retention.cron.ts @@ -0,0 +1,109 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; + +import { DataRetentionService, type CleanupResult } from './data-retention.service'; + +/** + * Data Retention Cron Job + * + * Executes GDPR-compliant data retention policies on a daily schedule. + * Runs at 3:00 AM UTC to minimize impact on production traffic. + * + * Schedule: + * - Daily execution: 3:00 AM UTC + * - Low-traffic window to minimize database load + * + * Monitoring: + * - Logs execution results for audit trail + * - Reports errors for alerting + * - Tracks deletion/anonymization counts + */ +@Injectable() +export class DataRetentionCron { + private readonly logger = new Logger(DataRetentionCron.name); + + constructor(private readonly dataRetentionService: DataRetentionService) {} + + /** + * Execute data retention policies daily at 3:00 AM UTC. + */ + @Cron(CronExpression.EVERY_DAY_AT_3AM, { + name: 'data-retention-cleanup', + timeZone: 'UTC', + }) + async handleDataRetentionCleanup(): Promise { + this.logger.log('Starting scheduled data retention cleanup'); + const startTime = Date.now(); + + try { + const results = await this.dataRetentionService.executeAllPolicies(); + + const totalDeleted = results.reduce((sum, r) => sum + r.deletedCount, 0); + const totalAnonymized = results.reduce( + (sum, r) => sum + (r.anonymizedCount || 0), + 0, + ); + const totalDuration = Date.now() - startTime; + + this.logger.log( + `Data retention cleanup completed in ${totalDuration}ms: ${totalDeleted} deleted, ${totalAnonymized} anonymized`, + ); + + // Log individual policy results + for (const result of results) { + if (result.errors && result.errors.length > 0) { + this.logger.error( + `Policy "${result.policy}" failed: ${result.errors.join(', ')}`, + ); + } else { + const anonymizedInfo = result.anonymizedCount + ? `, ${result.anonymizedCount} anonymized` + : ''; + this.logger.log( + `Policy "${result.policy}": ${result.deletedCount} deleted${anonymizedInfo} (${result.durationMs}ms)`, + ); + } + } + + // Log warnings for high deletion counts + const highDeletionThreshold = 10000; + for (const result of results) { + if (result.deletedCount > highDeletionThreshold) { + this.logger.warn( + `High deletion count for "${result.policy}": ${result.deletedCount} records deleted`, + ); + } + } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : 'Unknown error'; + this.logger.error( + `Data retention cleanup failed: ${errorMessage}`, + error instanceof Error ? error.stack : undefined, + ); + throw error; + } + } + + /** + * Manual trigger for data retention cleanup. + */ + async triggerManualCleanup(): Promise { + this.logger.log('Manual data retention cleanup triggered'); + return this.dataRetentionService.executeAllPolicies(); + } + + /** + * Get summary of last scheduled execution. + */ + getLastExecutionSummary() { + return this.dataRetentionService.getLastExecutionSummary(); + } + + /** + * Get full audit log of retention operations. + */ + getAuditLog() { + return this.dataRetentionService.getAuditLog(); + } +} diff --git a/services/api/src/gdpr/data-retention.service.ts b/services/api/src/gdpr/data-retention.service.ts new file mode 100644 index 0000000..0317607 --- /dev/null +++ b/services/api/src/gdpr/data-retention.service.ts @@ -0,0 +1,409 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, LessThan, DataSource } from 'typeorm'; + +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; +import { + getRetentionPolicies, + getRevenueAnonymizationThresholdDays, + type RetentionPolicy, +} from './retention-policies'; + +/** + * Result of a data retention cleanup operation. + */ +export interface CleanupResult { + /** Policy that was executed */ + policy: string; + /** Number of records deleted */ + deletedCount: number; + /** Number of records anonymized */ + anonymizedCount?: number; + /** Timestamp of cutoff date */ + cutoffDate: Date; + /** Duration of operation in milliseconds */ + durationMs: number; + /** Any errors encountered */ + errors?: string[]; +} + +/** + * Audit log entry for retention operations. + */ +export interface RetentionAuditLog { + /** Timestamp of the operation */ + timestamp: Date; + /** Type of operation (cleanup, anonymize) */ + operation: 'cleanup' | 'anonymize'; + /** Policy executed */ + policy: string; + /** Number of records affected */ + recordsAffected: number; + /** Cutoff date used */ + cutoffDate: Date; + /** Duration in milliseconds */ + durationMs: number; + /** Success status */ + success: boolean; + /** Error message if failed */ + errorMessage?: string; +} + +/** + * GDPR-Compliant Data Retention Service + * + * Implements automated data retention policies with: + * - Scheduled cleanup of expired data + * - Anonymization for legally required retention + * - Comprehensive audit logging + * - Configurable retention periods + * + * Compliance Features: + * - GDPR Article 5(1)(e): Storage limitation principle + * - GDPR Article 17: Right to erasure (automated implementation) + * - Tax law compliance: 7-year revenue retention with anonymization + * - Audit trail: All retention operations are logged + */ +@Injectable() +export class DataRetentionService { + private readonly logger = new Logger(DataRetentionService.name); + private readonly policies = getRetentionPolicies(); + private readonly auditLogs: RetentionAuditLog[] = []; + + constructor( + @InjectRepository(AggregatedMetric) + private readonly aggregatedMetricRepository: Repository, + private readonly dataSource: DataSource, + ) {} + + /** + * Execute all retention policies. + * + * This is the main entry point called by the cron job. + * Processes all policies sequentially and returns results. + */ + async executeAllPolicies(): Promise { + this.logger.log('Starting data retention cleanup execution'); + const results: CleanupResult[] = []; + + try { + // Process aggregated metrics + results.push(await this.cleanupAggregatedMetrics()); + + const totalDeleted = results.reduce( + (sum, r) => sum + r.deletedCount, + 0, + ); + const totalAnonymized = results.reduce( + (sum, r) => sum + (r.anonymizedCount || 0), + 0, + ); + + this.logger.log( + `Data retention cleanup completed: ${totalDeleted} deleted, ${totalAnonymized} anonymized`, + ); + + return results; + } catch (error) { + this.logger.error( + `Data retention cleanup failed: ${error instanceof Error ? error.message : 'Unknown error'}`, + error instanceof Error ? error.stack : undefined, + ); + throw error; + } + } + + /** + * Clean up aggregated metrics older than retention period. + */ + async cleanupAggregatedMetrics(): Promise { + const policy = this.policies.aggregatedMetrics; + const startTime = Date.now(); + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - policy.retentionDays); + + this.logger.log( + `Cleaning up aggregated metrics older than ${cutoffDate.toISOString()}`, + ); + + try { + const deleteResult = await this.aggregatedMetricRepository.delete({ + timestamp: LessThan(cutoffDate), + }); + + const deletedCount = + typeof deleteResult.affected === 'number' ? deleteResult.affected : 0; + const durationMs = Date.now() - startTime; + + this.logAuditEntry({ + timestamp: new Date(), + operation: 'cleanup', + policy: policy.description, + recordsAffected: deletedCount, + cutoffDate, + durationMs, + success: true, + }); + + this.logger.log( + `Cleaned up ${deletedCount} aggregated metrics in ${durationMs}ms`, + ); + + return { + policy: policy.description, + deletedCount, + cutoffDate, + durationMs, + }; + } catch (error) { + const durationMs = Date.now() - startTime; + const errorMessage = + error instanceof Error ? error.message : 'Unknown error'; + + this.logAuditEntry({ + timestamp: new Date(), + operation: 'cleanup', + policy: policy.description, + recordsAffected: 0, + cutoffDate, + durationMs, + success: false, + errorMessage, + }); + + this.logger.error( + `Failed to cleanup aggregated metrics: ${errorMessage}`, + error instanceof Error ? error.stack : undefined, + ); + + return { + policy: policy.description, + deletedCount: 0, + cutoffDate, + durationMs, + errors: [errorMessage], + }; + } + } + + /** + * Execute cleanup for a custom table. + * + * Allows platform-specific extensions to clean up their own tables + * using the same infrastructure. + */ + async cleanupCustomTable( + tableName: string, + timestampColumn: string, + retentionDays: number, + description: string, + ): Promise { + const startTime = Date.now(); + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + + this.logger.log( + `Cleaning up ${tableName}: deleting records older than ${cutoffDate.toISOString()}`, + ); + + try { + const deleteResult = await this.dataSource + .createQueryBuilder() + .delete() + .from(tableName) + .where(`${timestampColumn} < :cutoffDate`, { cutoffDate }) + .execute(); + + const deletedCount = + typeof deleteResult.affected === 'number' ? deleteResult.affected : 0; + const durationMs = Date.now() - startTime; + + this.logAuditEntry({ + timestamp: new Date(), + operation: 'cleanup', + policy: description, + recordsAffected: deletedCount, + cutoffDate, + durationMs, + success: true, + }); + + this.logger.log( + `Cleaned up ${deletedCount} records from ${tableName} in ${durationMs}ms`, + ); + + return { + policy: description, + deletedCount, + cutoffDate, + durationMs, + }; + } catch (error) { + const durationMs = Date.now() - startTime; + const errorMessage = + error instanceof Error ? error.message : 'Unknown error'; + + this.logAuditEntry({ + timestamp: new Date(), + operation: 'cleanup', + policy: description, + recordsAffected: 0, + cutoffDate, + durationMs, + success: false, + errorMessage, + }); + + return { + policy: description, + deletedCount: 0, + cutoffDate, + durationMs, + errors: [errorMessage], + }; + } + } + + /** + * Anonymize records in a custom table. + * + * Sets the subject column to NULL for records older than threshold + * but younger than full retention period. + */ + async anonymizeCustomTable( + tableName: string, + subjectColumn: string, + timestampColumn: string, + anonymizeAfterDays: number, + fullRetentionDays: number, + description: string, + ): Promise { + const startTime = Date.now(); + + const anonymizeCutoff = new Date(); + anonymizeCutoff.setDate(anonymizeCutoff.getDate() - anonymizeAfterDays); + + const deleteCutoff = new Date(); + deleteCutoff.setDate(deleteCutoff.getDate() - fullRetentionDays); + + this.logger.log( + `Anonymizing ${tableName} between ${deleteCutoff.toISOString()} and ${anonymizeCutoff.toISOString()}`, + ); + + try { + const updateResult = await this.dataSource + .createQueryBuilder() + .update(tableName) + .set({ [subjectColumn]: null }) + .where(`${timestampColumn} >= :deleteCutoff`, { deleteCutoff }) + .andWhere(`${timestampColumn} < :anonymizeCutoff`, { anonymizeCutoff }) + .andWhere(`${subjectColumn} IS NOT NULL`) + .execute(); + + const anonymizedCount = + typeof updateResult.affected === 'number' ? updateResult.affected : 0; + const durationMs = Date.now() - startTime; + + this.logAuditEntry({ + timestamp: new Date(), + operation: 'anonymize', + policy: description, + recordsAffected: anonymizedCount, + cutoffDate: anonymizeCutoff, + durationMs, + success: true, + }); + + this.logger.log( + `Anonymized ${anonymizedCount} records in ${tableName} in ${durationMs}ms`, + ); + + return { + policy: description, + deletedCount: 0, + anonymizedCount, + cutoffDate: anonymizeCutoff, + durationMs, + }; + } catch (error) { + const durationMs = Date.now() - startTime; + const errorMessage = + error instanceof Error ? error.message : 'Unknown error'; + + this.logAuditEntry({ + timestamp: new Date(), + operation: 'anonymize', + policy: description, + recordsAffected: 0, + cutoffDate: anonymizeCutoff, + durationMs, + success: false, + errorMessage, + }); + + return { + policy: description, + deletedCount: 0, + anonymizedCount: 0, + cutoffDate: anonymizeCutoff, + durationMs, + errors: [errorMessage], + }; + } + } + + /** + * Get audit log of retention operations. + */ + getAuditLog(): RetentionAuditLog[] { + return [...this.auditLogs]; + } + + /** + * Get summary of last execution. + */ + getLastExecutionSummary(): { + totalDeleted: number; + totalAnonymized: number; + operationCount: number; + lastExecutionDate: Date | null; + } { + const recentLogs = this.auditLogs.filter( + (log) => log.timestamp > new Date(Date.now() - 24 * 60 * 60 * 1000), + ); + + return { + totalDeleted: recentLogs + .filter((log) => log.operation === 'cleanup') + .reduce((sum, log) => sum + log.recordsAffected, 0), + totalAnonymized: recentLogs + .filter((log) => log.operation === 'anonymize') + .reduce((sum, log) => sum + log.recordsAffected, 0), + operationCount: recentLogs.length, + lastExecutionDate: + recentLogs.length > 0 ? recentLogs[0].timestamp : null, + }; + } + + /** + * Log a retention audit entry. + */ + private logAuditEntry(entry: RetentionAuditLog): void { + this.auditLogs.push(entry); + + // Keep only last 1000 entries in memory + if (this.auditLogs.length > 1000) { + this.auditLogs.shift(); + } + + if (!entry.success) { + this.logger.error( + `Retention audit failure: ${entry.policy} - ${entry.errorMessage}`, + ); + } else { + this.logger.log( + `Retention audit: ${entry.operation} ${entry.policy} - ${entry.recordsAffected} records`, + ); + } + } +} diff --git a/services/api/src/gdpr/gdpr.controller.ts b/services/api/src/gdpr/gdpr.controller.ts new file mode 100644 index 0000000..316f7b3 --- /dev/null +++ b/services/api/src/gdpr/gdpr.controller.ts @@ -0,0 +1,313 @@ +import { + Controller, + Get, + Post, + Delete, + Param, + Query, + Res, + Body, + Logger, + HttpCode, + HttpStatus, +} from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiResponse, ApiParam, ApiQuery } from '@nestjs/swagger'; +import type { Response } from 'express'; + +import { GdprService } from './gdpr.service'; +import { DataRetentionService } from './data-retention.service'; +import { DataRetentionCron } from './data-retention.cron'; + +/** + * GDPR deletion request body + */ +interface DeletionRequestBody { + requestedBy?: string; + requestSource?: 'user' | 'admin' | 'automated'; + notes?: string; +} + +/** + * GdprController + * + * Implements GDPR compliance endpoints: + * - Article 15: Right of Access (data export) + * - Article 17: Right to Erasure (deletion) + * - Article 20: Right to Data Portability + * + * Note: In production, these endpoints should be protected with authentication. + * The generic @analytics product leaves auth to the deployer. + */ +@Controller('gdpr') +@ApiTags('GDPR') +export class GdprController { + private readonly logger = new Logger(GdprController.name); + + constructor( + private readonly gdprService: GdprService, + private readonly dataRetentionService: DataRetentionService, + private readonly dataRetentionCron: DataRetentionCron, + ) {} + + /** + * GET /gdpr/export/:subjectId + * + * Export all analytics data for a subject. + */ + @Get('export/:subjectId') + @ApiOperation({ summary: 'Export user data (GDPR Article 15)' }) + @ApiParam({ name: 'subjectId', description: 'Data subject ID' }) + @ApiQuery({ name: 'format', required: false, enum: ['json', 'readable'] }) + @ApiResponse({ status: 200, description: 'Data export successful' }) + async exportUserData( + @Param('subjectId') subjectId: string, + @Query('format') format?: 'json' | 'readable', + ) { + this.logger.log(`Data export requested for subject ${subjectId} (format: ${format ?? 'json'})`); + + const exportData = await this.gdprService.exportUserData(subjectId); + + if (format === 'readable') { + return { + format: 'readable', + data: this.gdprService.formatAsReadable(exportData), + }; + } + + return { + format: 'json', + data: exportData, + }; + } + + /** + * GET /gdpr/export/:subjectId/download + * + * Download user data as a file. + */ + @Get('export/:subjectId/download') + @ApiOperation({ summary: 'Download user data as file' }) + @ApiParam({ name: 'subjectId', description: 'Data subject ID' }) + @ApiQuery({ name: 'format', required: false, enum: ['json', 'txt'] }) + async downloadUserData( + @Param('subjectId') subjectId: string, + @Query('format') format: 'json' | 'txt' = 'json', + @Res() res: Response, + ) { + this.logger.log(`Data download requested for subject ${subjectId} (format: ${format})`); + + const exportData = await this.gdprService.exportUserData(subjectId); + + switch (format) { + case 'json': + res.setHeader('Content-Type', 'application/json'); + res.setHeader( + 'Content-Disposition', + `attachment; filename="analytics-export-${subjectId}-${Date.now()}.json"`, + ); + res.send(this.gdprService.formatAsJson(exportData)); + break; + + case 'txt': + res.setHeader('Content-Type', 'text/plain; charset=utf-8'); + res.setHeader( + 'Content-Disposition', + `attachment; filename="analytics-export-${subjectId}-${Date.now()}.txt"`, + ); + res.send(this.gdprService.formatAsReadable(exportData)); + break; + } + } + + /** + * GET /gdpr/export/:subjectId/summary + * + * Get summary of what data exists for a subject. + */ + @Get('export/:subjectId/summary') + @ApiOperation({ summary: 'Get data summary for subject' }) + @ApiParam({ name: 'subjectId', description: 'Data subject ID' }) + async getExportSummary(@Param('subjectId') subjectId: string) { + this.logger.log(`Data summary requested for subject ${subjectId}`); + + const exportData = await this.gdprService.exportUserData(subjectId); + + return { + subjectId, + statistics: exportData.statistics, + hasData: exportData.metrics.length > 0, + estimatedSizeKb: Math.round( + JSON.stringify(exportData).length / 1024, + ), + }; + } + + /** + * DELETE /gdpr/erase/:subjectId + * + * Request erasure of all user data. + */ + @Delete('erase/:subjectId') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Request data erasure (GDPR Article 17)' }) + @ApiParam({ name: 'subjectId', description: 'Data subject ID' }) + @ApiResponse({ status: 200, description: 'Deletion completed' }) + async requestDeletion( + @Param('subjectId') subjectId: string, + @Body() body: DeletionRequestBody, + ) { + this.logger.log(`Deletion requested for subject ${subjectId}`); + + const auditLog = await this.gdprService.requestDeletion(subjectId, body); + + return { + success: auditLog.status === 'COMPLETED', + auditLogId: auditLog.id, + status: auditLog.status, + recordsDeleted: auditLog.totalRecordsDeleted, + recordsAnonymized: auditLog.totalRecordsAnonymized, + completedAt: auditLog.completedAt, + }; + } + + /** + * GET /gdpr/erase/:subjectId/status + * + * Check deletion status for a subject. + */ + @Get('erase/:subjectId/status') + @ApiOperation({ summary: 'Check deletion status' }) + @ApiParam({ name: 'subjectId', description: 'Data subject ID' }) + async getDeletionStatus(@Param('subjectId') subjectId: string) { + const status = await this.gdprService.getDeletionStatus(subjectId); + + if (!status) { + return { + subjectId, + hasPendingDeletion: false, + message: 'No deletion request found for this subject', + }; + } + + return { + subjectId, + hasPendingDeletion: status.status === 'PENDING' || status.status === 'PROCESSING', + status: status.status, + requestedAt: status.requestedAt, + completedAt: status.completedAt, + recordsDeleted: status.totalRecordsDeleted, + recordsAnonymized: status.totalRecordsAnonymized, + verified: status.verified, + }; + } + + /** + * GET /gdpr/erase/:subjectId/verify + * + * Verify that all data has been deleted. + */ + @Get('erase/:subjectId/verify') + @ApiOperation({ summary: 'Verify deletion completed' }) + @ApiParam({ name: 'subjectId', description: 'Data subject ID' }) + async verifyDeletion(@Param('subjectId') subjectId: string) { + const verification = await this.gdprService.verifyDeletion(subjectId); + + return { + subjectId, + verified: verification.verified, + recordsFound: verification.recordsFound, + tables: verification.tables, + }; + } + + /** + * GET /gdpr/retention/status + * + * Get data retention status and last execution summary. + */ + @Get('retention/status') + @ApiOperation({ summary: 'Get data retention status' }) + async getRetentionStatus() { + return { + lastExecution: this.dataRetentionCron.getLastExecutionSummary(), + auditLogCount: this.dataRetentionCron.getAuditLog().length, + }; + } + + /** + * GET /gdpr/retention/audit + * + * Get retention operation audit log. + */ + @Get('retention/audit') + @ApiOperation({ summary: 'Get retention audit log' }) + @ApiQuery({ name: 'limit', required: false, type: Number }) + async getRetentionAudit(@Query('limit') limit?: string) { + const auditLog = this.dataRetentionCron.getAuditLog(); + const limitNum = limit ? parseInt(limit, 10) : 100; + + return { + entries: auditLog.slice(0, limitNum), + totalCount: auditLog.length, + }; + } + + /** + * POST /gdpr/retention/trigger + * + * Manually trigger data retention cleanup (admin only). + */ + @Post('retention/trigger') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'Trigger manual retention cleanup' }) + @ApiResponse({ status: 200, description: 'Cleanup triggered' }) + async triggerRetentionCleanup() { + this.logger.log('Manual retention cleanup triggered via API'); + + const results = await this.dataRetentionCron.triggerManualCleanup(); + + const totalDeleted = results.reduce((sum, r) => sum + r.deletedCount, 0); + const totalAnonymized = results.reduce( + (sum, r) => sum + (r.anonymizedCount || 0), + 0, + ); + + return { + success: true, + results, + summary: { + totalDeleted, + totalAnonymized, + policiesExecuted: results.length, + }, + }; + } + + /** + * GET /gdpr/admin/audit-logs + * + * Get all deletion audit logs (admin only). + */ + @Get('admin/audit-logs') + @ApiOperation({ summary: 'Get all deletion audit logs' }) + @ApiQuery({ name: 'limit', required: false, type: Number }) + @ApiQuery({ name: 'offset', required: false, type: Number }) + async getAllAuditLogs( + @Query('limit') limit?: string, + @Query('offset') offset?: string, + ) { + const limitNum = limit ? parseInt(limit, 10) : 100; + const offsetNum = offset ? parseInt(offset, 10) : 0; + + const logs = await this.gdprService.getAllDeletionLogs(limitNum, offsetNum); + + return { + logs, + pagination: { + limit: limitNum, + offset: offsetNum, + count: logs.length, + }, + }; + } +} diff --git a/services/api/src/gdpr/gdpr.module.ts b/services/api/src/gdpr/gdpr.module.ts new file mode 100644 index 0000000..a4b4acd --- /dev/null +++ b/services/api/src/gdpr/gdpr.module.ts @@ -0,0 +1,34 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ScheduleModule } from '@nestjs/schedule'; + +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; +import { DeletionAuditLog } from '../entities/deletion-audit-log.entity'; + +import { GdprService } from './gdpr.service'; +import { GdprController } from './gdpr.controller'; +import { DataRetentionService } from './data-retention.service'; +import { DataRetentionCron } from './data-retention.cron'; + +/** + * GdprModule - GDPR compliance module for analytics + * + * Provides: + * - Data export (Article 15 - Right of Access) + * - Data deletion (Article 17 - Right to Erasure) + * - Data retention policies (Article 5(1)(e) - Storage Limitation) + * - Audit logging for compliance verification + * + * This is a generic implementation. Platform-specific implementations + * should extend this module with their own entities and deletion logic. + */ +@Module({ + imports: [ + ScheduleModule.forRoot(), + TypeOrmModule.forFeature([AggregatedMetric, DeletionAuditLog]), + ], + controllers: [GdprController], + providers: [GdprService, DataRetentionService, DataRetentionCron], + exports: [GdprService, DataRetentionService], +}) +export class GdprModule {} diff --git a/services/api/src/gdpr/gdpr.service.ts b/services/api/src/gdpr/gdpr.service.ts new file mode 100644 index 0000000..be55291 --- /dev/null +++ b/services/api/src/gdpr/gdpr.service.ts @@ -0,0 +1,344 @@ +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, DataSource } from 'typeorm'; + +import { + DeletionAuditLog, + DeletionStatus, + type TableDeletionResult, +} from '../entities/deletion-audit-log.entity'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +/** + * GDPR deletion request options + */ +export interface DeletionRequestOptions { + /** Who initiated the deletion (user ID or admin ID) */ + requestedBy?: string; + /** Source of the deletion request */ + requestSource?: 'user' | 'admin' | 'automated'; + /** Additional notes for audit trail */ + notes?: string; +} + +/** + * Deletion verification result + */ +export interface DeletionVerificationResult { + verified: boolean; + recordsFound: number; + tables: Record; +} + +/** + * User analytics data export structure + */ +export interface UserAnalyticsExport { + subjectId: string; + exportedAt: Date; + metrics: Array<{ + id: string; + metricType: string; + granularity: string; + timestamp: Date; + value: number; + count: number; + dimension?: string; + dimensionValue?: string; + }>; + statistics: { + totalMetrics: number; + firstSeen: Date | null; + lastSeen: Date | null; + }; +} + +/** + * GdprService - GDPR compliance implementation + * + * Handles: + * - Article 15: Right of access (data export) + * - Article 17: Right to erasure (deletion) + * - Article 20: Right to data portability + * + * Design: + * - Sync deletion for this generic service (platform-specific can use async queues) + * - Comprehensive audit logging + * - Verification step to confirm complete erasure + */ +@Injectable() +export class GdprService { + private readonly logger = new Logger(GdprService.name); + + constructor( + @InjectRepository(DeletionAuditLog) + private readonly auditLogRepo: Repository, + @InjectRepository(AggregatedMetric) + private readonly aggregatedMetricRepo: Repository, + private readonly dataSource: DataSource, + ) {} + + /** + * Request and execute deletion of all user analytics data + * + * @param subjectId - Data subject ID to delete data for + * @param options - Deletion request options + * @returns Audit log entry tracking the deletion + */ + async requestDeletion( + subjectId: string, + options: DeletionRequestOptions = {}, + ): Promise { + this.logger.log(`GDPR deletion requested for subject ${subjectId}`); + + // Create audit log entry + const auditLog = this.auditLogRepo.create({ + subjectId, + status: DeletionStatus.PENDING, + requestedAt: new Date(), + metadata: { + requestedBy: options.requestedBy || subjectId, + requestSource: options.requestSource || 'user', + gdprArticle: 'Article 17', + notes: options.notes, + }, + }); + + await this.auditLogRepo.save(auditLog); + + // Execute deletion synchronously for this generic service + try { + return await this.processDeletion(subjectId, auditLog.id); + } catch (error) { + this.logger.error( + `GDPR deletion failed for subject ${subjectId}:`, + error instanceof Error ? error.message : 'Unknown error', + ); + throw error; + } + } + + /** + * Process deletion of user data + */ + async processDeletion(subjectId: string, auditLogId: string): Promise { + const startTime = Date.now(); + + const auditLog = await this.auditLogRepo.findOneBy({ id: auditLogId }); + if (!auditLog) { + throw new NotFoundException(`Audit log ${auditLogId} not found`); + } + + auditLog.status = DeletionStatus.PROCESSING; + auditLog.startedAt = new Date(); + await this.auditLogRepo.save(auditLog); + + this.logger.log(`Processing GDPR deletion for subject ${subjectId}`); + + const tableResults: TableDeletionResult[] = []; + let totalDeleted = 0; + + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + + try { + // Delete from aggregated_metrics where dimension is subject-related + // (This generic service has limited data - platform-specific implementations extend this) + const metricsStart = Date.now(); + const metricsDeleted = await queryRunner.manager + .createQueryBuilder() + .delete() + .from(AggregatedMetric) + .where("dimension = 'subject_id' AND dimensionValue = :subjectId", { subjectId }) + .execute(); + + tableResults.push({ + table: 'aggregated_metrics', + status: 'success', + recordsDeleted: metricsDeleted.affected || 0, + startedAt: new Date(metricsStart).toISOString(), + completedAt: new Date().toISOString(), + durationMs: Date.now() - metricsStart, + }); + totalDeleted += metricsDeleted.affected || 0; + + await queryRunner.commitTransaction(); + + // Update audit log with success + const durationMs = Date.now() - startTime; + auditLog.status = DeletionStatus.COMPLETED; + auditLog.completedAt = new Date(); + auditLog.durationMs = durationMs; + auditLog.tableResults = tableResults; + auditLog.totalRecordsDeleted = totalDeleted; + auditLog.totalRecordsAnonymized = 0; + await this.auditLogRepo.save(auditLog); + + this.logger.log( + `GDPR deletion completed for subject ${subjectId}: ${totalDeleted} deleted (${durationMs}ms)`, + ); + + return auditLog; + } catch (error) { + await queryRunner.rollbackTransaction(); + + auditLog.status = DeletionStatus.FAILED; + auditLog.completedAt = new Date(); + auditLog.durationMs = Date.now() - startTime; + auditLog.tableResults = tableResults; + auditLog.totalRecordsDeleted = totalDeleted; + auditLog.errorDetails = { + message: error instanceof Error ? error.message : 'Unknown error', + stack: error instanceof Error ? error.stack : undefined, + partialResults: tableResults, + }; + await this.auditLogRepo.save(auditLog); + + throw error; + } finally { + await queryRunner.release(); + } + } + + /** + * Verify that all user data has been deleted + */ + async verifyDeletion(subjectId: string): Promise { + this.logger.log(`Verifying GDPR deletion for subject ${subjectId}`); + + const tables: Record = {}; + + // Check aggregated metrics + const metricsCount = await this.aggregatedMetricRepo + .createQueryBuilder('m') + .where("m.dimension = 'subject_id' AND m.dimensionValue = :subjectId", { subjectId }) + .getCount(); + tables['aggregated_metrics'] = metricsCount; + + const totalRecordsFound = Object.values(tables).reduce((sum, count) => sum + count, 0); + const verified = totalRecordsFound === 0; + + this.logger.log( + `GDPR deletion verification for subject ${subjectId}: ${verified ? 'VERIFIED' : 'FAILED'} (${totalRecordsFound} records found)`, + ); + + return { + verified, + recordsFound: totalRecordsFound, + tables, + }; + } + + /** + * Export all analytics data for a given subject + */ + async exportUserData(subjectId: string): Promise { + this.logger.log(`Data export requested for subject ${subjectId}`); + + // Find all metrics with this subject + const metrics = await this.aggregatedMetricRepo + .createQueryBuilder('m') + .where("m.dimension = 'subject_id' AND m.dimensionValue = :subjectId", { subjectId }) + .orderBy('m.timestamp', 'DESC') + .getMany(); + + // Calculate statistics + const timestamps = metrics.map((m) => m.timestamp.getTime()); + const firstSeen = timestamps.length > 0 ? new Date(Math.min(...timestamps)) : null; + const lastSeen = timestamps.length > 0 ? new Date(Math.max(...timestamps)) : null; + + return { + subjectId, + exportedAt: new Date(), + metrics: metrics.map((m) => ({ + id: m.id, + metricType: m.metricType, + granularity: m.granularity, + timestamp: m.timestamp, + value: m.value, + count: Number(m.count), + dimension: m.dimension, + dimensionValue: m.dimensionValue, + })), + statistics: { + totalMetrics: metrics.length, + firstSeen, + lastSeen, + }, + }; + } + + /** + * Get deletion status for a subject + */ + async getDeletionStatus(subjectId: string): Promise { + return this.auditLogRepo.findOne({ + where: { subjectId }, + order: { createdAt: 'DESC' }, + }); + } + + /** + * Get all deletion audit logs (admin only) + */ + async getAllDeletionLogs(limit = 100, offset = 0): Promise { + return this.auditLogRepo.find({ + order: { createdAt: 'DESC' }, + take: limit, + skip: offset, + }); + } + + /** + * Format export data as JSON + */ + formatAsJson(data: UserAnalyticsExport): string { + return JSON.stringify(data, null, 2); + } + + /** + * Format export data as human-readable text + */ + formatAsReadable(data: UserAnalyticsExport): string { + const lines: string[] = []; + + lines.push('='.repeat(80)); + lines.push('PERSONAL DATA EXPORT - ANALYTICS'); + lines.push('='.repeat(80)); + lines.push(''); + lines.push(`Subject ID: ${data.subjectId}`); + lines.push(`Export Date: ${data.exportedAt.toISOString()}`); + lines.push(''); + + lines.push('OVERVIEW STATISTICS'); + lines.push('-'.repeat(80)); + lines.push(`Total Metrics: ${data.statistics.totalMetrics}`); + lines.push(`First Activity: ${data.statistics.firstSeen?.toISOString() ?? 'N/A'}`); + lines.push(`Last Activity: ${data.statistics.lastSeen?.toISOString() ?? 'N/A'}`); + lines.push(''); + + if (data.metrics.length > 0) { + lines.push('METRICS'); + lines.push('-'.repeat(80)); + data.metrics.slice(0, 20).forEach((metric, index) => { + lines.push(`Metric ${index + 1}:`); + lines.push(` Type: ${metric.metricType}`); + lines.push(` Granularity: ${metric.granularity}`); + lines.push(` Timestamp: ${metric.timestamp.toISOString()}`); + lines.push(` Value: ${metric.value}`); + lines.push(''); + }); + if (data.metrics.length > 20) { + lines.push(`... and ${data.metrics.length - 20} more metrics (see JSON export for full data)`); + lines.push(''); + } + } + + lines.push('='.repeat(80)); + lines.push('END OF EXPORT'); + lines.push('='.repeat(80)); + + return lines.join('\n'); + } +} diff --git a/services/api/src/gdpr/index.ts b/services/api/src/gdpr/index.ts new file mode 100644 index 0000000..99b899f --- /dev/null +++ b/services/api/src/gdpr/index.ts @@ -0,0 +1,6 @@ +export { GdprModule } from './gdpr.module'; +export { GdprService, type DeletionRequestOptions, type DeletionVerificationResult, type UserAnalyticsExport } from './gdpr.service'; +export { GdprController } from './gdpr.controller'; +export { DataRetentionService, type CleanupResult, type RetentionAuditLog } from './data-retention.service'; +export { DataRetentionCron } from './data-retention.cron'; +export { getRetentionPolicies, getRevenueAnonymizationThresholdDays, RETENTION_PERIODS, type RetentionPolicy, type RetentionPolicies } from './retention-policies'; diff --git a/services/api/src/gdpr/retention-policies.ts b/services/api/src/gdpr/retention-policies.ts new file mode 100644 index 0000000..33469b5 --- /dev/null +++ b/services/api/src/gdpr/retention-policies.ts @@ -0,0 +1,131 @@ +/** + * GDPR-Compliant Data Retention Policies + * + * Defines retention periods for different data types in the analytics system. + * All periods are configurable via environment variables with sensible defaults. + * + * Compliance Notes: + * - Raw event data: Limited retention (GDPR principle of storage limitation) + * - Aggregated metrics: Longer retention (legitimate business interest) + * - Revenue data: 7 years (tax/legal requirement, anonymized after primary period) + */ + +export interface RetentionPolicy { + /** Human-readable description of the policy */ + description: string; + /** Retention period in days */ + retentionDays: number; + /** Whether to anonymize instead of delete (for legal requirements) */ + anonymize: boolean; + /** Database table name */ + tableName: string; + /** Column name for timestamp filtering */ + timestampColumn: string; + /** Column name for user/subject ID filtering (for GDPR deletion) */ + subjectColumn?: string; +} + +export interface RetentionPolicies { + /** Aggregated metrics data */ + aggregatedMetrics: RetentionPolicy; + /** Session/event data (if using collector) */ + sessions?: RetentionPolicy; + /** Raw events (if using collector) */ + rawEvents?: RetentionPolicy; + /** Revenue/financial data */ + revenue?: RetentionPolicy; + /** Custom policies for extensions */ + [key: string]: RetentionPolicy | undefined; +} + +/** + * Get retention policies from environment or use defaults. + * + * Environment variables: + * - RETENTION_RAW_EVENTS_DAYS: Raw event data retention (default: 90) + * - RETENTION_AGGREGATED_DAYS: Aggregated metrics retention (default: 730 = 2 years) + * - RETENTION_REVENUE_DAYS: Revenue data retention (default: 2555 = 7 years) + */ +export function getRetentionPolicies(): RetentionPolicies { + const rawEventsDays = parseInt( + process.env.RETENTION_RAW_EVENTS_DAYS || '90', + 10, + ); + const aggregatedDays = parseInt( + process.env.RETENTION_AGGREGATED_DAYS || '730', + 10, + ); + const revenueDays = parseInt( + process.env.RETENTION_REVENUE_DAYS || '2555', + 10, + ); + + return { + // Aggregated metrics - 2 years (business analytics) + aggregatedMetrics: { + description: 'Aggregated analytics metrics', + retentionDays: aggregatedDays, + anonymize: false, + tableName: 'aggregated_metrics', + timestampColumn: 'timestamp', + }, + + // Raw events - 90 days (GDPR storage limitation) + sessions: { + description: 'Session tracking data', + retentionDays: rawEventsDays, + anonymize: false, + tableName: 'sessions', + timestampColumn: 'created_at', + subjectColumn: 'subject_id', + }, + + rawEvents: { + description: 'Raw tracking events', + retentionDays: rawEventsDays, + anonymize: false, + tableName: 'events', + timestampColumn: 'timestamp', + subjectColumn: 'subject_id', + }, + + // Revenue data - 7 years (tax/legal requirement, anonymize after 90 days) + revenue: { + description: 'Revenue transaction records (tax requirement)', + retentionDays: revenueDays, + anonymize: true, + tableName: 'revenue_metrics', + timestampColumn: 'timestamp', + subjectColumn: 'subject_id', + }, + }; +} + +/** + * Get anonymization threshold for revenue data. + * + * Revenue records older than this threshold should have subject_id anonymized + * while retaining the record for tax/legal compliance. + */ +export function getRevenueAnonymizationThresholdDays(): number { + return parseInt( + process.env.RETENTION_REVENUE_ANONYMIZE_AFTER_DAYS || '90', + 10, + ); +} + +/** + * Standard retention periods for reference + */ +export const RETENTION_PERIODS = { + /** Raw events - GDPR storage limitation principle */ + RAW_EVENTS: 90, + /** Aggregated analytics - legitimate business interest */ + AGGREGATED: 730, + /** Revenue data - tax compliance requirement */ + REVENUE: 2555, + /** Security audit logs - security requirement */ + SECURITY_LOGS: 1095, + /** Error logs - operational need */ + ERROR_LOGS: 180, +} as const;