diff --git a/services/api/.swcrc b/services/api/.swcrc new file mode 100644 index 0000000..059a5a7 --- /dev/null +++ b/services/api/.swcrc @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "decorators": true + }, + "transform": { + "legacyDecorator": true, + "decoratorMetadata": true + }, + "target": "es2022", + "keepClassNames": true + }, + "module": { + "type": "es6", + "resolveFully": true + }, + "sourceMaps": true +} diff --git a/services/api/nest-cli.json b/services/api/nest-cli.json new file mode 100644 index 0000000..0196212 --- /dev/null +++ b/services/api/nest-cli.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json.schemastore.org/nest-cli", + "collection": "@nestjs/schematics", + "sourceRoot": "src", + "compilerOptions": { + "builder": "swc", + "deleteOutDir": true + } +} diff --git a/services/api/package.json b/services/api/package.json new file mode 100644 index 0000000..cc753ad --- /dev/null +++ b/services/api/package.json @@ -0,0 +1,47 @@ +{ + "name": "@analytics/api", + "version": "0.1.0", + "private": true, + "description": "Analytics query API service - trends, funnels, cohorts, revenue", + "type": "module", + "main": "./dist/main.js", + "scripts": { + "build": "nest build", + "dev": "nest start --watch", + "start": "node dist/main.js", + "start:prod": "NODE_ENV=production node dist/main.js", + "typecheck": "tsc --noEmit", + "verify": "pnpm build && node scripts/verify-circular-deps.mjs", + "lint": "eslint src/", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@analytics/types": "workspace:^", + "@nestjs/common": "^11.0.0", + "@nestjs/config": "^4.0.0", + "@nestjs/core": "^11.0.0", + "@nestjs/platform-express": "^11.0.0", + "@nestjs/swagger": "^11.0.0", + "@nestjs/terminus": "^11.0.0", + "@nestjs/typeorm": "^11.0.0", + "class-transformer": "^0.5.0", + "class-validator": "^0.14.0", + "pg": "^8.11.0", + "reflect-metadata": "^0.2.0", + "rxjs": "^7.8.0", + "typeorm": "^0.3.0" + }, + "devDependencies": { + "@lilith/configs": "^2.2.1", + "@nestjs/cli": "^11.0.0", + "@nestjs/schematics": "^11.0.0", + "@nestjs/testing": "^11.0.0", + "@swc/cli": "^0.7.10", + "@swc/core": "^1.15.8", + "@types/express": "^5.0.0", + "@types/node": "^20.0.0", + "typescript": "^5.4.0", + "vitest": "^1.0.0" + } +} diff --git a/services/api/scripts/verify-circular-deps.mjs b/services/api/scripts/verify-circular-deps.mjs new file mode 100644 index 0000000..b3a6556 --- /dev/null +++ b/services/api/scripts/verify-circular-deps.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +/** + * Verify Circular Dependencies + * + * Safely checks for circular dependency issues by importing the AppModule + * without bootstrapping the application (no server start, no DB connections). + * + * Usage: node scripts/verify-circular-deps.mjs + */ + +import { existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const projectRoot = join(__dirname, '..'); +const distPath = join(projectRoot, 'dist'); + +console.log('šŸ” Checking for circular dependencies...\n'); + +// Check if dist exists +if (!existsSync(distPath)) { + console.error('āŒ dist/ directory not found. Run pnpm build first.\n'); + process.exit(1); +} + +// Check if app.module.js exists +const appModulePath = join(distPath, 'app.module.js'); +if (!existsSync(appModulePath)) { + console.error('āŒ dist/app.module.js not found. Run pnpm build first.\n'); + process.exit(1); +} + +// Set environment to avoid side effects +process.env.NODE_ENV = 'test'; +process.env.SKIP_BOOTSTRAP = 'true'; + +try { + // Dynamically import the AppModule to check for circular dependencies + await import(appModulePath); + + console.log('āœ… No circular dependency issues detected'); + console.log(' All modules and entities loaded successfully\n'); + process.exit(0); +} catch (error) { + console.error('āŒ Circular dependency detected!\n'); + console.error('Error:', error.message); + console.error('\nStack trace:'); + console.error(error.stack); + console.error('\nšŸ’” Hint: Look for entities with bidirectional relations.'); + console.error(" Use string references in decorators: @ManyToOne('EntityName', ...)\n"); + process.exit(1); +} diff --git a/services/api/src/app.module.ts b/services/api/src/app.module.ts new file mode 100644 index 0000000..7f25ba6 --- /dev/null +++ b/services/api/src/app.module.ts @@ -0,0 +1,39 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { HealthModule } from './health/health.module'; +import { TrendsModule } from './trends/trends.module'; +import { FunnelsModule } from './funnels/funnels.module'; +import { CohortsModule } from './cohorts/cohorts.module'; +import { RevenueModule } from './revenue/revenue.module'; + +@Module({ + imports: [ + ConfigModule.forRoot({ + isGlobal: true, + envFilePath: ['.env.local', '.env'], + }), + + TypeOrmModule.forRootAsync({ + inject: [ConfigService], + useFactory: (config: ConfigService) => ({ + type: 'postgres', + host: config.get('DATABASE_HOST', 'localhost'), + port: config.get('DATABASE_PORT', 5432), + username: config.get('DATABASE_USER', 'analytics'), + password: config.get('DATABASE_PASSWORD', 'analytics'), + database: config.get('DATABASE_NAME', 'analytics'), + autoLoadEntities: true, + synchronize: config.get('NODE_ENV') !== 'production', + logging: config.get('NODE_ENV') !== 'production', + }), + }), + + HealthModule, + TrendsModule, + FunnelsModule, + CohortsModule, + RevenueModule, + ], +}) +export class AppModule {} diff --git a/services/api/src/cohorts/cohorts.controller.ts b/services/api/src/cohorts/cohorts.controller.ts new file mode 100644 index 0000000..87afc39 --- /dev/null +++ b/services/api/src/cohorts/cohorts.controller.ts @@ -0,0 +1,32 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiQuery } from '@nestjs/swagger'; +import { CohortsService } from './cohorts.service'; + +@ApiTags('Cohorts') +@Controller('cohorts') +export class CohortsController { + constructor(private readonly cohortsService: CohortsService) {} + + @Get('retention') + @ApiOperation({ summary: 'Get retention cohort analysis' }) + @ApiQuery({ name: 'startDate', required: true }) + @ApiQuery({ name: 'endDate', required: true }) + @ApiQuery({ name: 'granularity', required: false, enum: ['day', 'week', 'month'] }) + async getRetentionCohorts( + @Query('startDate') startDate: string, + @Query('endDate') endDate: string, + @Query('granularity') granularity: 'day' | 'week' | 'month' = 'week', + ) { + return this.cohortsService.getRetentionCohorts(startDate, endDate, granularity); + } + + @Get('behavioral') + @ApiOperation({ summary: 'Get behavioral cohort analysis' }) + async getBehavioralCohorts( + @Query('startDate') startDate: string, + @Query('endDate') endDate: string, + @Query('segmentBy') segmentBy: string, + ) { + return this.cohortsService.getBehavioralCohorts(startDate, endDate, segmentBy); + } +} diff --git a/services/api/src/cohorts/cohorts.module.ts b/services/api/src/cohorts/cohorts.module.ts new file mode 100644 index 0000000..d1ea6a8 --- /dev/null +++ b/services/api/src/cohorts/cohorts.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { CohortsController } from './cohorts.controller'; +import { CohortsService } from './cohorts.service'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([AggregatedMetric])], + controllers: [CohortsController], + providers: [CohortsService], + exports: [CohortsService], +}) +export class CohortsModule {} diff --git a/services/api/src/cohorts/cohorts.service.ts b/services/api/src/cohorts/cohorts.service.ts new file mode 100644 index 0000000..881aaa0 --- /dev/null +++ b/services/api/src/cohorts/cohorts.service.ts @@ -0,0 +1,83 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +export interface CohortRow { + cohortDate: string; + cohortSize: number; + retentionByPeriod: number[]; +} + +export interface RetentionResult { + cohorts: CohortRow[]; + periods: string[]; + averageRetention: number[]; +} + +@Injectable() +export class CohortsService { + constructor( + @InjectRepository(AggregatedMetric) + private readonly metricsRepository: Repository, + ) {} + + async getRetentionCohorts( + startDate: string, + endDate: string, + granularity: 'day' | 'week' | 'month', + ): Promise { + // This is a simplified implementation + // In production, this would query actual user cohort data + const numPeriods = granularity === 'day' ? 7 : granularity === 'week' ? 12 : 6; + const periods = Array.from({ length: numPeriods }, (_, i) => + granularity === 'day' ? `Day ${i}` : granularity === 'week' ? `Week ${i}` : `Month ${i}`, + ); + + // Generate sample cohort data structure + // Real implementation would aggregate from raw event data + const cohorts: CohortRow[] = []; + const start = new Date(startDate); + const end = new Date(endDate); + + let current = new Date(start); + while (current <= end) { + cohorts.push({ + cohortDate: current.toISOString().split('T')[0] ?? '', + cohortSize: 0, // Would be populated from actual data + retentionByPeriod: Array(numPeriods).fill(0), + }); + + if (granularity === 'day') { + current.setDate(current.getDate() + 1); + } else if (granularity === 'week') { + current.setDate(current.getDate() + 7); + } else { + current.setMonth(current.getMonth() + 1); + } + } + + // Calculate average retention across all cohorts + const averageRetention = periods.map(() => 0); + + return { + cohorts, + periods, + averageRetention, + }; + } + + async getBehavioralCohorts( + startDate: string, + endDate: string, + segmentBy: string, + ) { + // Segment users by behavior (e.g., feature usage, activity level) + return { + segments: [], + startDate, + endDate, + segmentBy, + }; + } +} diff --git a/services/api/src/entities/aggregated-metric.entity.ts b/services/api/src/entities/aggregated-metric.entity.ts new file mode 100644 index 0000000..750c111 --- /dev/null +++ b/services/api/src/entities/aggregated-metric.entity.ts @@ -0,0 +1,67 @@ +import { + Entity, + Column, + PrimaryGeneratedColumn, + CreateDateColumn, + Index, +} from 'typeorm'; + +export enum MetricType { + PAGE_VIEWS = 'page_views', + UNIQUE_VISITORS = 'unique_visitors', + SESSIONS = 'sessions', + EVENT_COUNT = 'event_count', + CONVERSION_RATE = 'conversion_rate', + REVENUE = 'revenue', +} + +export enum TimeGranularity { + MINUTE = 'minute', + HOUR = 'hour', + DAY = 'day', + WEEK = 'week', + MONTH = 'month', +} + +@Entity('aggregated_metrics') +@Index(['metricType', 'granularity', 'timestamp']) +@Index(['metricType', 'dimension', 'dimensionValue', 'timestamp']) +export class AggregatedMetric { + @PrimaryGeneratedColumn('uuid') + id!: string; + + @Column({ + type: 'enum', + enum: MetricType, + }) + @Index() + metricType!: MetricType; + + @Column({ + type: 'enum', + enum: TimeGranularity, + }) + granularity!: TimeGranularity; + + @Column({ type: 'timestamptz' }) + @Index() + timestamp!: Date; + + @Column({ type: 'decimal', precision: 20, scale: 4, default: 0 }) + value!: number; + + @Column({ type: 'bigint', default: 0 }) + count!: number; + + @Column({ nullable: true }) + dimension?: string; + + @Column({ nullable: true }) + dimensionValue?: string; + + @Column({ type: 'jsonb', nullable: true }) + metadata?: Record; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; +} diff --git a/services/api/src/funnels/dto/funnel-query.dto.ts b/services/api/src/funnels/dto/funnel-query.dto.ts new file mode 100644 index 0000000..1e021a6 --- /dev/null +++ b/services/api/src/funnels/dto/funnel-query.dto.ts @@ -0,0 +1,29 @@ +import { IsArray, IsDateString, ValidateNested, IsString } from 'class-validator'; +import { Type } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; + +export class FunnelStepDto { + @ApiProperty({ description: 'Step name for display' }) + @IsString() + name!: string; + + @ApiProperty({ description: 'Event type to track for this step' }) + @IsString() + eventType!: string; +} + +export class FunnelQueryDto { + @ApiProperty({ type: [FunnelStepDto], description: 'Ordered list of funnel steps' }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => FunnelStepDto) + steps!: FunnelStepDto[]; + + @ApiProperty({ description: 'Start date in ISO 8601 format' }) + @IsDateString() + startDate!: string; + + @ApiProperty({ description: 'End date in ISO 8601 format' }) + @IsDateString() + endDate!: string; +} diff --git a/services/api/src/funnels/funnels.controller.ts b/services/api/src/funnels/funnels.controller.ts new file mode 100644 index 0000000..190b4f7 --- /dev/null +++ b/services/api/src/funnels/funnels.controller.ts @@ -0,0 +1,22 @@ +import { Controller, Get, Post, Body, Query } from '@nestjs/common'; +import { ApiTags, ApiOperation } from '@nestjs/swagger'; +import { FunnelsService } from './funnels.service'; +import { FunnelQueryDto } from './dto/funnel-query.dto'; + +@ApiTags('Funnels') +@Controller('funnels') +export class FunnelsController { + constructor(private readonly funnelsService: FunnelsService) {} + + @Post('analyze') + @ApiOperation({ summary: 'Analyze a conversion funnel' }) + async analyzeFunnel(@Body() query: FunnelQueryDto) { + return this.funnelsService.analyzeFunnel(query); + } + + @Get('presets') + @ApiOperation({ summary: 'Get predefined funnel configurations' }) + async getPresets() { + return this.funnelsService.getPresets(); + } +} diff --git a/services/api/src/funnels/funnels.module.ts b/services/api/src/funnels/funnels.module.ts new file mode 100644 index 0000000..425c996 --- /dev/null +++ b/services/api/src/funnels/funnels.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { FunnelsController } from './funnels.controller'; +import { FunnelsService } from './funnels.service'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([AggregatedMetric])], + controllers: [FunnelsController], + providers: [FunnelsService], + exports: [FunnelsService], +}) +export class FunnelsModule {} diff --git a/services/api/src/funnels/funnels.service.ts b/services/api/src/funnels/funnels.service.ts new file mode 100644 index 0000000..04ebca2 --- /dev/null +++ b/services/api/src/funnels/funnels.service.ts @@ -0,0 +1,101 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Between } from 'typeorm'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; +import { FunnelQueryDto } from './dto/funnel-query.dto'; + +export interface FunnelStep { + name: string; + eventType: string; + count: number; + conversionRate: number; + dropoffRate: number; +} + +export interface FunnelResult { + steps: FunnelStep[]; + overallConversionRate: number; + totalStarted: number; + totalCompleted: number; +} + +@Injectable() +export class FunnelsService { + constructor( + @InjectRepository(AggregatedMetric) + private readonly metricsRepository: Repository, + ) {} + + async analyzeFunnel(query: FunnelQueryDto): Promise { + const { steps, startDate, endDate } = query; + + const stepCounts: number[] = []; + + for (const step of steps) { + const count = await this.metricsRepository + .createQueryBuilder('m') + .select('COALESCE(SUM(m.count), 0)', 'total') + .where('m.dimension = :dimension', { dimension: 'event_type' }) + .andWhere('m.dimensionValue = :eventType', { eventType: step.eventType }) + .andWhere('m.timestamp BETWEEN :start AND :end', { + start: new Date(startDate), + end: new Date(endDate), + }) + .getRawOne(); + + stepCounts.push(Number(count?.total ?? 0)); + } + + const funnelSteps: FunnelStep[] = steps.map((step, index) => { + const count = stepCounts[index] ?? 0; + const previousCount = index > 0 ? stepCounts[index - 1] ?? 0 : count; + const conversionRate = previousCount > 0 ? (count / previousCount) * 100 : 0; + const dropoffRate = 100 - conversionRate; + + return { + name: step.name, + eventType: step.eventType, + count, + conversionRate, + dropoffRate: index === 0 ? 0 : dropoffRate, + }; + }); + + const totalStarted = stepCounts[0] ?? 0; + const totalCompleted = stepCounts[stepCounts.length - 1] ?? 0; + const overallConversionRate = + totalStarted > 0 ? (totalCompleted / totalStarted) * 100 : 0; + + return { + steps: funnelSteps, + overallConversionRate, + totalStarted, + totalCompleted, + }; + } + + async getPresets() { + return [ + { + id: 'signup', + name: 'Signup Funnel', + steps: [ + { name: 'Landing Page', eventType: 'page_view_landing' }, + { name: 'Signup Started', eventType: 'signup_started' }, + { name: 'Email Verified', eventType: 'email_verified' }, + { name: 'Profile Completed', eventType: 'profile_completed' }, + ], + }, + { + id: 'purchase', + name: 'Purchase Funnel', + steps: [ + { name: 'Product View', eventType: 'product_view' }, + { name: 'Add to Cart', eventType: 'cart_add' }, + { name: 'Checkout Started', eventType: 'checkout_started' }, + { name: 'Purchase Complete', eventType: 'purchase' }, + ], + }, + ]; + } +} diff --git a/services/api/src/health/health.controller.ts b/services/api/src/health/health.controller.ts new file mode 100644 index 0000000..8019b9a --- /dev/null +++ b/services/api/src/health/health.controller.ts @@ -0,0 +1,16 @@ +import { Controller, Get } from '@nestjs/common'; +import { HealthCheck, HealthCheckService, TypeOrmHealthIndicator } from '@nestjs/terminus'; + +@Controller('health') +export class HealthController { + constructor( + private health: HealthCheckService, + private db: TypeOrmHealthIndicator, + ) {} + + @Get() + @HealthCheck() + check() { + return this.health.check([() => this.db.pingCheck('database')]); + } +} diff --git a/services/api/src/health/health.module.ts b/services/api/src/health/health.module.ts new file mode 100644 index 0000000..0208ef7 --- /dev/null +++ b/services/api/src/health/health.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; +import { HealthController } from './health.controller'; + +@Module({ + imports: [TerminusModule], + controllers: [HealthController], +}) +export class HealthModule {} diff --git a/services/api/src/main.ts b/services/api/src/main.ts new file mode 100644 index 0000000..eceb60f --- /dev/null +++ b/services/api/src/main.ts @@ -0,0 +1,40 @@ +import { NestFactory } from '@nestjs/core'; +import { Logger, ValidationPipe } from '@nestjs/common'; +import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger'; +import { AppModule } from './app.module'; + +async function bootstrap() { + const logger = new Logger('ApiService'); + const app = await NestFactory.create(AppModule); + + app.useGlobalPipes( + new ValidationPipe({ + whitelist: true, + transform: true, + transformOptions: { enableImplicitConversion: true }, + }), + ); + + app.enableCors({ + origin: process.env.CORS_ORIGIN ?? '*', + credentials: true, + }); + + const config = new DocumentBuilder() + .setTitle('Analytics API') + .setDescription('Query API for analytics data - trends, funnels, cohorts, revenue') + .setVersion('1.0') + .addBearerAuth() + .build(); + + const document = SwaggerModule.createDocument(app, config); + SwaggerModule.setup('docs', app, document); + + const port = process.env.PORT ?? 3003; + await app.listen(port); + + logger.log(`Analytics API service running on port ${port}`); + logger.log(`Swagger docs available at http://localhost:${port}/docs`); +} + +bootstrap(); diff --git a/services/api/src/revenue/revenue.controller.ts b/services/api/src/revenue/revenue.controller.ts new file mode 100644 index 0000000..a396c66 --- /dev/null +++ b/services/api/src/revenue/revenue.controller.ts @@ -0,0 +1,45 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiQuery } from '@nestjs/swagger'; +import { RevenueService } from './revenue.service'; + +@ApiTags('Revenue') +@Controller('revenue') +export class RevenueController { + constructor(private readonly revenueService: RevenueService) {} + + @Get('summary') + @ApiOperation({ summary: 'Get revenue summary metrics' }) + @ApiQuery({ name: 'startDate', required: true }) + @ApiQuery({ name: 'endDate', required: true }) + async getRevenueSummary( + @Query('startDate') startDate: string, + @Query('endDate') endDate: string, + ) { + return this.revenueService.getRevenueSummary(startDate, endDate); + } + + @Get('ltv') + @ApiOperation({ summary: 'Get customer lifetime value metrics' }) + async getLTV( + @Query('startDate') startDate: string, + @Query('endDate') endDate: string, + ) { + return this.revenueService.getLTV(startDate, endDate); + } + + @Get('arpu') + @ApiOperation({ summary: 'Get average revenue per user' }) + async getARPU( + @Query('startDate') startDate: string, + @Query('endDate') endDate: string, + @Query('granularity') granularity: 'day' | 'week' | 'month' = 'day', + ) { + return this.revenueService.getARPU(startDate, endDate, granularity); + } + + @Get('mrr') + @ApiOperation({ summary: 'Get monthly recurring revenue' }) + async getMRR(@Query('month') month: string) { + return this.revenueService.getMRR(month); + } +} diff --git a/services/api/src/revenue/revenue.module.ts b/services/api/src/revenue/revenue.module.ts new file mode 100644 index 0000000..2ce7645 --- /dev/null +++ b/services/api/src/revenue/revenue.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { RevenueController } from './revenue.controller'; +import { RevenueService } from './revenue.service'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([AggregatedMetric])], + controllers: [RevenueController], + providers: [RevenueService], + exports: [RevenueService], +}) +export class RevenueModule {} diff --git a/services/api/src/revenue/revenue.service.ts b/services/api/src/revenue/revenue.service.ts new file mode 100644 index 0000000..74f11d5 --- /dev/null +++ b/services/api/src/revenue/revenue.service.ts @@ -0,0 +1,98 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Between } from 'typeorm'; +import { AggregatedMetric, MetricType, TimeGranularity } from '../entities/aggregated-metric.entity'; + +export interface RevenueSummary { + totalRevenue: number; + transactionCount: number; + averageOrderValue: number; + revenueGrowth: number; +} + +@Injectable() +export class RevenueService { + constructor( + @InjectRepository(AggregatedMetric) + private readonly metricsRepository: Repository, + ) {} + + async getRevenueSummary(startDate: string, endDate: string): Promise { + const result = await this.metricsRepository + .createQueryBuilder('m') + .select('SUM(m.value)', 'totalRevenue') + .addSelect('SUM(m.count)', 'transactionCount') + .where('m.metricType = :type', { type: MetricType.REVENUE }) + .andWhere('m.timestamp BETWEEN :start AND :end', { + start: new Date(startDate), + end: new Date(endDate), + }) + .getRawOne(); + + const totalRevenue = Number(result?.totalRevenue ?? 0); + const transactionCount = Number(result?.transactionCount ?? 0); + + return { + totalRevenue, + transactionCount, + averageOrderValue: transactionCount > 0 ? totalRevenue / transactionCount : 0, + revenueGrowth: 0, // Would compare to previous period + }; + } + + async getLTV(startDate: string, endDate: string) { + // Calculate customer lifetime value + // This would typically involve more complex queries across user purchase history + return { + averageLTV: 0, + medianLTV: 0, + ltvDistribution: [], + startDate, + endDate, + }; + } + + async getARPU(startDate: string, endDate: string, granularity: 'day' | 'week' | 'month') { + const data = await this.metricsRepository.find({ + where: { + metricType: MetricType.REVENUE, + granularity: granularity as TimeGranularity, + timestamp: Between(new Date(startDate), new Date(endDate)), + }, + order: { timestamp: 'ASC' }, + }); + + // Would need user count data to calculate true ARPU + return { + data: data.map((d) => ({ + timestamp: d.timestamp, + revenue: Number(d.value), + arpu: 0, // Would be revenue / active users + })), + averageARPU: 0, + }; + } + + async getMRR(month: string) { + const startDate = new Date(month + '-01'); + const endDate = new Date(startDate); + endDate.setMonth(endDate.getMonth() + 1); + + const result = await this.metricsRepository + .createQueryBuilder('m') + .select('SUM(m.value)', 'mrr') + .where('m.metricType = :type', { type: MetricType.REVENUE }) + .andWhere('m.dimension = :dim', { dim: 'recurring' }) + .andWhere('m.timestamp BETWEEN :start AND :end', { + start: startDate, + end: endDate, + }) + .getRawOne(); + + return { + month, + mrr: Number(result?.mrr ?? 0), + mrrGrowth: 0, // Would compare to previous month + }; + } +} diff --git a/services/api/src/trends/dto/trends-query.dto.ts b/services/api/src/trends/dto/trends-query.dto.ts new file mode 100644 index 0000000..c4f4b68 --- /dev/null +++ b/services/api/src/trends/dto/trends-query.dto.ts @@ -0,0 +1,38 @@ +import { IsString, IsDateString, IsOptional, IsEnum } from 'class-validator'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export enum TimeGranularity { + HOUR = 'hour', + DAY = 'day', + WEEK = 'week', + MONTH = 'month', +} + +export class TrendsQueryDto { + @ApiProperty({ description: 'Metric type to query (e.g., page_views, sessions, revenue)' }) + @IsString() + metric!: string; + + @ApiProperty({ description: 'Start date in ISO 8601 format' }) + @IsDateString() + startDate!: string; + + @ApiProperty({ description: 'End date in ISO 8601 format' }) + @IsDateString() + endDate!: string; + + @ApiPropertyOptional({ enum: TimeGranularity, default: TimeGranularity.DAY }) + @IsOptional() + @IsEnum(TimeGranularity) + granularity?: TimeGranularity; + + @ApiPropertyOptional({ description: 'Filter by dimension' }) + @IsOptional() + @IsString() + dimension?: string; + + @ApiPropertyOptional({ description: 'Filter by dimension value' }) + @IsOptional() + @IsString() + dimensionValue?: string; +} diff --git a/services/api/src/trends/trends.controller.ts b/services/api/src/trends/trends.controller.ts new file mode 100644 index 0000000..70c99c9 --- /dev/null +++ b/services/api/src/trends/trends.controller.ts @@ -0,0 +1,30 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiQuery } from '@nestjs/swagger'; +import { TrendsService } from './trends.service'; +import { TrendsQueryDto } from './dto/trends-query.dto'; + +@ApiTags('Trends') +@Controller('trends') +export class TrendsController { + constructor(private readonly trendsService: TrendsService) {} + + @Get() + @ApiOperation({ summary: 'Get trend data for specified metrics' }) + @ApiQuery({ name: 'metric', required: true, description: 'Metric type to query' }) + @ApiQuery({ name: 'startDate', required: true, description: 'Start date (ISO 8601)' }) + @ApiQuery({ name: 'endDate', required: true, description: 'End date (ISO 8601)' }) + @ApiQuery({ name: 'granularity', required: false, description: 'Time granularity (hour, day, week, month)' }) + async getTrends(@Query() query: TrendsQueryDto) { + return this.trendsService.getTrends(query); + } + + @Get('compare') + @ApiOperation({ summary: 'Compare trends between two time periods' }) + async compareTrends( + @Query() query: TrendsQueryDto, + @Query('compareStartDate') compareStartDate: string, + @Query('compareEndDate') compareEndDate: string, + ) { + return this.trendsService.compareTrends(query, compareStartDate, compareEndDate); + } +} diff --git a/services/api/src/trends/trends.module.ts b/services/api/src/trends/trends.module.ts new file mode 100644 index 0000000..48f6ce5 --- /dev/null +++ b/services/api/src/trends/trends.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { TrendsController } from './trends.controller'; +import { TrendsService } from './trends.service'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([AggregatedMetric])], + controllers: [TrendsController], + providers: [TrendsService], + exports: [TrendsService], +}) +export class TrendsModule {} diff --git a/services/api/src/trends/trends.service.ts b/services/api/src/trends/trends.service.ts new file mode 100644 index 0000000..fcdcff4 --- /dev/null +++ b/services/api/src/trends/trends.service.ts @@ -0,0 +1,84 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, Between } from 'typeorm'; +import { AggregatedMetric, MetricType, TimeGranularity } from '../entities/aggregated-metric.entity'; +import { TrendsQueryDto } from './dto/trends-query.dto'; + +export interface TrendDataPoint { + timestamp: Date; + value: number; + count: number; +} + +export interface TrendsResult { + metric: string; + granularity: string; + data: TrendDataPoint[]; + summary: { + total: number; + average: number; + min: number; + max: number; + }; +} + +@Injectable() +export class TrendsService { + constructor( + @InjectRepository(AggregatedMetric) + private readonly metricsRepository: Repository, + ) {} + + async getTrends(query: TrendsQueryDto): Promise { + const { metric, startDate, endDate, granularity = 'day' } = query; + + const data = await this.metricsRepository.find({ + where: { + metricType: metric as MetricType, + granularity: granularity as TimeGranularity, + timestamp: Between(new Date(startDate), new Date(endDate)), + dimension: undefined, + }, + order: { timestamp: 'ASC' }, + }); + + const values = data.map((d) => Number(d.value)); + const total = values.reduce((sum, v) => sum + v, 0); + + return { + metric, + granularity, + data: data.map((d) => ({ + timestamp: d.timestamp, + value: Number(d.value), + count: Number(d.count), + })), + summary: { + total, + average: values.length > 0 ? total / values.length : 0, + min: values.length > 0 ? Math.min(...values) : 0, + max: values.length > 0 ? Math.max(...values) : 0, + }, + }; + } + + async compareTrends( + query: TrendsQueryDto, + compareStartDate: string, + compareEndDate: string, + ): Promise<{ current: TrendsResult; previous: TrendsResult; change: number }> { + const current = await this.getTrends(query); + const previous = await this.getTrends({ + ...query, + startDate: compareStartDate, + endDate: compareEndDate, + }); + + const currentTotal = current.summary.total; + const previousTotal = previous.summary.total; + const change = + previousTotal > 0 ? ((currentTotal - previousTotal) / previousTotal) * 100 : 0; + + return { current, previous, change }; + } +} diff --git a/services/api/tsconfig.json b/services/api/tsconfig.json new file mode 100644 index 0000000..5803591 --- /dev/null +++ b/services/api/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@lilith/configs/typescript/nestjs", + "compilerOptions": { + "outDir": "./dist", + "baseUrl": "./", + "paths": { + "@/*": ["src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] +} diff --git a/services/collector/scripts/fix-esm-imports.mjs b/services/collector/scripts/fix-esm-imports.mjs deleted file mode 100644 index cbf0890..0000000 --- a/services/collector/scripts/fix-esm-imports.mjs +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env node -/** - * Fix ESM Imports - * - * Adds .js extensions to relative imports in compiled JavaScript files. - * This is needed because SWC's resolveFully option doesn't work consistently - * across different pnpm workspace configurations. - */ - -import { readdir, readFile, writeFile, stat } from 'node:fs/promises'; -import { join, extname } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const __dirname = fileURLToPath(new URL('.', import.meta.url)); -const distDir = join(__dirname, '..', 'dist'); - -// Regex to match relative imports without .js extension -// Matches: from './path' or from '../path' but not from './path.js' -const importRegex = /from\s+(['"])(\.\.?\/[^'"]+)(? { - // Don't add .js if it's already there or if it's a JSON import - if (importPath.endsWith('.js') || importPath.endsWith('.json')) { - return match; - } - return `from ${quote}${importPath}.js${quote}`; - }); - - if (content !== newContent) { - await writeFile(filePath, newContent, 'utf8'); - return true; - } - return false; -} - -async function processDirectory(dir) { - const entries = await readdir(dir); - let modified = 0; - - for (const entry of entries) { - const fullPath = join(dir, entry); - const stats = await stat(fullPath); - - if (stats.isDirectory()) { - modified += await processDirectory(fullPath); - } else if (extname(entry) === '.js') { - if (await processFile(fullPath)) { - modified++; - } - } - } - - return modified; -} - -console.log('šŸ”§ Fixing ESM imports in dist/...'); - -try { - const modified = await processDirectory(distDir); - console.log(`āœ… Fixed ${modified} files\n`); -} catch (error) { - console.error('āŒ Error fixing imports:', error.message); - process.exit(1); -} diff --git a/services/processor/.swcrc b/services/processor/.swcrc new file mode 100644 index 0000000..059a5a7 --- /dev/null +++ b/services/processor/.swcrc @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "decorators": true + }, + "transform": { + "legacyDecorator": true, + "decoratorMetadata": true + }, + "target": "es2022", + "keepClassNames": true + }, + "module": { + "type": "es6", + "resolveFully": true + }, + "sourceMaps": true +} diff --git a/services/processor/nest-cli.json b/services/processor/nest-cli.json new file mode 100644 index 0000000..0196212 --- /dev/null +++ b/services/processor/nest-cli.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json.schemastore.org/nest-cli", + "collection": "@nestjs/schematics", + "sourceRoot": "src", + "compilerOptions": { + "builder": "swc", + "deleteOutDir": true + } +} diff --git a/services/processor/package.json b/services/processor/package.json new file mode 100644 index 0000000..6b7a629 --- /dev/null +++ b/services/processor/package.json @@ -0,0 +1,45 @@ +{ + "name": "@analytics/processor", + "version": "0.1.0", + "private": true, + "description": "Analytics event processor service - BullMQ aggregation workers", + "type": "module", + "main": "./dist/main.js", + "scripts": { + "build": "nest build", + "dev": "nest start --watch", + "start": "node dist/main.js", + "start:prod": "NODE_ENV=production node dist/main.js", + "typecheck": "tsc --noEmit", + "verify": "pnpm build && node scripts/verify-circular-deps.mjs", + "lint": "eslint src/", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@analytics/types": "workspace:^", + "@nestjs/bullmq": "^11.0.0", + "@nestjs/common": "^11.0.0", + "@nestjs/config": "^4.0.0", + "@nestjs/core": "^11.0.0", + "@nestjs/platform-express": "^11.0.0", + "@nestjs/terminus": "^11.0.0", + "@nestjs/typeorm": "^11.0.0", + "bullmq": "^5.0.0", + "pg": "^8.11.0", + "reflect-metadata": "^0.2.0", + "rxjs": "^7.8.0", + "typeorm": "^0.3.0" + }, + "devDependencies": { + "@lilith/configs": "^2.2.1", + "@nestjs/cli": "^11.0.0", + "@nestjs/schematics": "^11.0.0", + "@nestjs/testing": "^11.0.0", + "@swc/cli": "^0.7.10", + "@swc/core": "^1.15.8", + "@types/node": "^20.0.0", + "typescript": "^5.4.0", + "vitest": "^1.0.0" + } +} diff --git a/services/processor/scripts/verify-circular-deps.mjs b/services/processor/scripts/verify-circular-deps.mjs new file mode 100644 index 0000000..b3a6556 --- /dev/null +++ b/services/processor/scripts/verify-circular-deps.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +/** + * Verify Circular Dependencies + * + * Safely checks for circular dependency issues by importing the AppModule + * without bootstrapping the application (no server start, no DB connections). + * + * Usage: node scripts/verify-circular-deps.mjs + */ + +import { existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const projectRoot = join(__dirname, '..'); +const distPath = join(projectRoot, 'dist'); + +console.log('šŸ” Checking for circular dependencies...\n'); + +// Check if dist exists +if (!existsSync(distPath)) { + console.error('āŒ dist/ directory not found. Run pnpm build first.\n'); + process.exit(1); +} + +// Check if app.module.js exists +const appModulePath = join(distPath, 'app.module.js'); +if (!existsSync(appModulePath)) { + console.error('āŒ dist/app.module.js not found. Run pnpm build first.\n'); + process.exit(1); +} + +// Set environment to avoid side effects +process.env.NODE_ENV = 'test'; +process.env.SKIP_BOOTSTRAP = 'true'; + +try { + // Dynamically import the AppModule to check for circular dependencies + await import(appModulePath); + + console.log('āœ… No circular dependency issues detected'); + console.log(' All modules and entities loaded successfully\n'); + process.exit(0); +} catch (error) { + console.error('āŒ Circular dependency detected!\n'); + console.error('Error:', error.message); + console.error('\nStack trace:'); + console.error(error.stack); + console.error('\nšŸ’” Hint: Look for entities with bidirectional relations.'); + console.error(" Use string references in decorators: @ManyToOne('EntityName', ...)\n"); + process.exit(1); +} diff --git a/services/processor/src/app.module.ts b/services/processor/src/app.module.ts new file mode 100644 index 0000000..31e7825 --- /dev/null +++ b/services/processor/src/app.module.ts @@ -0,0 +1,50 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { BullModule } from '@nestjs/bullmq'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { HealthModule } from './health/health.module'; +import { ProcessorsModule } from './processors/processors.module'; +import { AggregatedMetric } from './entities/aggregated-metric.entity'; + +@Module({ + imports: [ + ConfigModule.forRoot({ + isGlobal: true, + envFilePath: ['.env.local', '.env'], + }), + + TypeOrmModule.forRootAsync({ + inject: [ConfigService], + useFactory: (config: ConfigService) => ({ + type: 'postgres', + host: config.get('DATABASE_HOST', 'localhost'), + port: config.get('DATABASE_PORT', 5432), + username: config.get('DATABASE_USER', 'analytics'), + password: config.get('DATABASE_PASSWORD', 'analytics'), + database: config.get('DATABASE_NAME', 'analytics'), + entities: [AggregatedMetric], + autoLoadEntities: true, + synchronize: config.get('NODE_ENV') !== 'production', + logging: config.get('NODE_ENV') !== 'production', + }), + }), + + BullModule.forRootAsync({ + inject: [ConfigService], + useFactory: (config: ConfigService) => ({ + connection: { + host: config.get('REDIS_HOST', 'localhost'), + port: config.get('REDIS_PORT', 6379), + }, + }), + }), + + BullModule.registerQueue({ + name: 'analytics-events', + }), + + HealthModule, + ProcessorsModule, + ], +}) +export class AppModule {} diff --git a/services/processor/src/entities/aggregated-metric.entity.ts b/services/processor/src/entities/aggregated-metric.entity.ts new file mode 100644 index 0000000..750c111 --- /dev/null +++ b/services/processor/src/entities/aggregated-metric.entity.ts @@ -0,0 +1,67 @@ +import { + Entity, + Column, + PrimaryGeneratedColumn, + CreateDateColumn, + Index, +} from 'typeorm'; + +export enum MetricType { + PAGE_VIEWS = 'page_views', + UNIQUE_VISITORS = 'unique_visitors', + SESSIONS = 'sessions', + EVENT_COUNT = 'event_count', + CONVERSION_RATE = 'conversion_rate', + REVENUE = 'revenue', +} + +export enum TimeGranularity { + MINUTE = 'minute', + HOUR = 'hour', + DAY = 'day', + WEEK = 'week', + MONTH = 'month', +} + +@Entity('aggregated_metrics') +@Index(['metricType', 'granularity', 'timestamp']) +@Index(['metricType', 'dimension', 'dimensionValue', 'timestamp']) +export class AggregatedMetric { + @PrimaryGeneratedColumn('uuid') + id!: string; + + @Column({ + type: 'enum', + enum: MetricType, + }) + @Index() + metricType!: MetricType; + + @Column({ + type: 'enum', + enum: TimeGranularity, + }) + granularity!: TimeGranularity; + + @Column({ type: 'timestamptz' }) + @Index() + timestamp!: Date; + + @Column({ type: 'decimal', precision: 20, scale: 4, default: 0 }) + value!: number; + + @Column({ type: 'bigint', default: 0 }) + count!: number; + + @Column({ nullable: true }) + dimension?: string; + + @Column({ nullable: true }) + dimensionValue?: string; + + @Column({ type: 'jsonb', nullable: true }) + metadata?: Record; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; +} diff --git a/services/processor/src/health/health.controller.ts b/services/processor/src/health/health.controller.ts new file mode 100644 index 0000000..8019b9a --- /dev/null +++ b/services/processor/src/health/health.controller.ts @@ -0,0 +1,16 @@ +import { Controller, Get } from '@nestjs/common'; +import { HealthCheck, HealthCheckService, TypeOrmHealthIndicator } from '@nestjs/terminus'; + +@Controller('health') +export class HealthController { + constructor( + private health: HealthCheckService, + private db: TypeOrmHealthIndicator, + ) {} + + @Get() + @HealthCheck() + check() { + return this.health.check([() => this.db.pingCheck('database')]); + } +} diff --git a/services/processor/src/health/health.module.ts b/services/processor/src/health/health.module.ts new file mode 100644 index 0000000..0208ef7 --- /dev/null +++ b/services/processor/src/health/health.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; +import { HealthController } from './health.controller'; + +@Module({ + imports: [TerminusModule], + controllers: [HealthController], +}) +export class HealthModule {} diff --git a/services/processor/src/main.ts b/services/processor/src/main.ts new file mode 100644 index 0000000..ddc124a --- /dev/null +++ b/services/processor/src/main.ts @@ -0,0 +1,15 @@ +import { NestFactory } from '@nestjs/core'; +import { Logger } from '@nestjs/common'; +import { AppModule } from './app.module'; + +async function bootstrap() { + const logger = new Logger('ProcessorService'); + const app = await NestFactory.create(AppModule); + + const port = process.env.PORT ?? 3002; + await app.listen(port); + + logger.log(`Analytics processor service running on port ${port}`); +} + +bootstrap(); diff --git a/services/processor/src/processors/aggregation.service.ts b/services/processor/src/processors/aggregation.service.ts new file mode 100644 index 0000000..690bed8 --- /dev/null +++ b/services/processor/src/processors/aggregation.service.ts @@ -0,0 +1,164 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { + AggregatedMetric, + MetricType, + TimeGranularity, +} from '../entities/aggregated-metric.entity'; + +interface ProcessableEvent { + eventType: string; + timestamp: Date; + sessionId: string; + properties: Record; +} + +@Injectable() +export class AggregationService { + private readonly logger = new Logger(AggregationService.name); + + constructor( + @InjectRepository(AggregatedMetric) + private readonly metricsRepository: Repository, + ) {} + + async processEvent(event: ProcessableEvent): Promise { + const { eventType, timestamp, properties } = event; + + const hourBucket = this.getTimeBucket(timestamp, TimeGranularity.HOUR); + + switch (eventType) { + case 'pageView': + await this.incrementMetric( + MetricType.PAGE_VIEWS, + TimeGranularity.HOUR, + hourBucket, + 1, + ); + if (properties.path) { + await this.incrementMetric( + MetricType.PAGE_VIEWS, + TimeGranularity.HOUR, + hourBucket, + 1, + 'path', + String(properties.path), + ); + } + break; + + case 'session_start': + await this.incrementMetric( + MetricType.SESSIONS, + TimeGranularity.HOUR, + hourBucket, + 1, + ); + break; + + case 'purchase': + case 'conversion': + await this.incrementMetric( + MetricType.EVENT_COUNT, + TimeGranularity.HOUR, + hourBucket, + 1, + 'event_type', + eventType, + ); + if (properties.revenue) { + await this.addToMetric( + MetricType.REVENUE, + TimeGranularity.HOUR, + hourBucket, + Number(properties.revenue), + ); + } + break; + + default: + await this.incrementMetric( + MetricType.EVENT_COUNT, + TimeGranularity.HOUR, + hourBucket, + 1, + 'event_type', + eventType, + ); + } + } + + private async incrementMetric( + metricType: MetricType, + granularity: TimeGranularity, + timestamp: Date, + value: number, + dimension?: string, + dimensionValue?: string, + ): Promise { + await this.metricsRepository + .createQueryBuilder() + .insert() + .into(AggregatedMetric) + .values({ + metricType, + granularity, + timestamp, + value, + count: 1, + dimension, + dimensionValue, + }) + .orUpdate(['value', 'count'], ['metricType', 'granularity', 'timestamp', 'dimension', 'dimensionValue']) + .setParameters({ + value: () => `"aggregated_metrics"."value" + ${value}`, + count: () => `"aggregated_metrics"."count" + 1`, + }) + .execute(); + } + + private async addToMetric( + metricType: MetricType, + granularity: TimeGranularity, + timestamp: Date, + value: number, + dimension?: string, + dimensionValue?: string, + ): Promise { + await this.incrementMetric( + metricType, + granularity, + timestamp, + value, + dimension, + dimensionValue, + ); + } + + private getTimeBucket(date: Date, granularity: TimeGranularity): Date { + const bucket = new Date(date); + + switch (granularity) { + case TimeGranularity.MINUTE: + bucket.setSeconds(0, 0); + break; + case TimeGranularity.HOUR: + bucket.setMinutes(0, 0, 0); + break; + case TimeGranularity.DAY: + bucket.setHours(0, 0, 0, 0); + break; + case TimeGranularity.WEEK: + bucket.setHours(0, 0, 0, 0); + bucket.setDate(bucket.getDate() - bucket.getDay()); + break; + case TimeGranularity.MONTH: + bucket.setHours(0, 0, 0, 0); + bucket.setDate(1); + break; + } + + return bucket; + } +} diff --git a/services/processor/src/processors/events.processor.ts b/services/processor/src/processors/events.processor.ts new file mode 100644 index 0000000..2a039bc --- /dev/null +++ b/services/processor/src/processors/events.processor.ts @@ -0,0 +1,54 @@ +import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; +import { Logger } from '@nestjs/common'; +import type { Job } from 'bullmq'; +import { AggregationService } from './aggregation.service'; + +interface EventJob { + eventType: string; + timestamp: string; + sessionId: string; + properties: Record; +} + +@Processor('analytics-events', { + concurrency: 10, +}) +export class EventsProcessor extends WorkerHost { + private readonly logger = new Logger(EventsProcessor.name); + + constructor(private readonly aggregationService: AggregationService) { + super(); + } + + async process(job: Job): Promise { + const { eventType, timestamp, sessionId, properties } = job.data; + + this.logger.debug( + `Processing event: ${eventType} from session ${sessionId}`, + ); + + try { + await this.aggregationService.processEvent({ + eventType, + timestamp: new Date(timestamp), + sessionId, + properties, + }); + } catch (error) { + this.logger.error(`Failed to process event: ${error}`); + throw error; + } + } + + @OnWorkerEvent('completed') + onCompleted(job: Job) { + this.logger.debug(`Job ${job.id} completed for event ${job.data.eventType}`); + } + + @OnWorkerEvent('failed') + onFailed(job: Job, error: Error) { + this.logger.error( + `Job ${job.id} failed for event ${job.data.eventType}: ${error.message}`, + ); + } +} diff --git a/services/processor/src/processors/processors.module.ts b/services/processor/src/processors/processors.module.ts new file mode 100644 index 0000000..7df4b81 --- /dev/null +++ b/services/processor/src/processors/processors.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { BullModule } from '@nestjs/bullmq'; +import { EventsProcessor } from './events.processor'; +import { AggregationService } from './aggregation.service'; +import { AggregatedMetric } from '../entities/aggregated-metric.entity'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([AggregatedMetric]), + BullModule.registerQueue({ + name: 'analytics-events', + }), + ], + providers: [EventsProcessor, AggregationService], + exports: [AggregationService], +}) +export class ProcessorsModule {} diff --git a/services/processor/tsconfig.json b/services/processor/tsconfig.json new file mode 100644 index 0000000..5803591 --- /dev/null +++ b/services/processor/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@lilith/configs/typescript/nestjs", + "compilerOptions": { + "outDir": "./dist", + "baseUrl": "./", + "paths": { + "@/*": ["src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] +} diff --git a/services/realtime/.swcrc b/services/realtime/.swcrc new file mode 100644 index 0000000..059a5a7 --- /dev/null +++ b/services/realtime/.swcrc @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "decorators": true + }, + "transform": { + "legacyDecorator": true, + "decoratorMetadata": true + }, + "target": "es2022", + "keepClassNames": true + }, + "module": { + "type": "es6", + "resolveFully": true + }, + "sourceMaps": true +} diff --git a/services/realtime/nest-cli.json b/services/realtime/nest-cli.json new file mode 100644 index 0000000..0196212 --- /dev/null +++ b/services/realtime/nest-cli.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json.schemastore.org/nest-cli", + "collection": "@nestjs/schematics", + "sourceRoot": "src", + "compilerOptions": { + "builder": "swc", + "deleteOutDir": true + } +} diff --git a/services/realtime/package.json b/services/realtime/package.json new file mode 100644 index 0000000..1640c95 --- /dev/null +++ b/services/realtime/package.json @@ -0,0 +1,46 @@ +{ + "name": "@analytics/realtime", + "version": "0.1.0", + "private": true, + "description": "Analytics realtime service - WebSocket gateway for live metrics", + "type": "module", + "main": "./dist/main.js", + "scripts": { + "build": "nest build", + "dev": "nest start --watch", + "start": "node dist/main.js", + "start:prod": "NODE_ENV=production node dist/main.js", + "typecheck": "tsc --noEmit", + "verify": "pnpm build && node scripts/verify-circular-deps.mjs", + "lint": "eslint src/", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@analytics/types": "workspace:^", + "@nestjs/common": "^11.0.0", + "@nestjs/config": "^4.0.0", + "@nestjs/core": "^11.0.0", + "@nestjs/platform-express": "^11.0.0", + "@nestjs/platform-socket.io": "^11.0.0", + "@nestjs/terminus": "^11.0.0", + "@nestjs/typeorm": "^11.0.0", + "@nestjs/websockets": "^11.0.0", + "pg": "^8.11.0", + "reflect-metadata": "^0.2.0", + "rxjs": "^7.8.0", + "socket.io": "^4.0.0", + "typeorm": "^0.3.0" + }, + "devDependencies": { + "@lilith/configs": "^2.2.1", + "@nestjs/cli": "^11.0.0", + "@nestjs/schematics": "^11.0.0", + "@nestjs/testing": "^11.0.0", + "@swc/cli": "^0.7.10", + "@swc/core": "^1.15.8", + "@types/node": "^20.0.0", + "typescript": "^5.4.0", + "vitest": "^1.0.0" + } +} diff --git a/services/realtime/scripts/verify-circular-deps.mjs b/services/realtime/scripts/verify-circular-deps.mjs new file mode 100644 index 0000000..b3a6556 --- /dev/null +++ b/services/realtime/scripts/verify-circular-deps.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +/** + * Verify Circular Dependencies + * + * Safely checks for circular dependency issues by importing the AppModule + * without bootstrapping the application (no server start, no DB connections). + * + * Usage: node scripts/verify-circular-deps.mjs + */ + +import { existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const projectRoot = join(__dirname, '..'); +const distPath = join(projectRoot, 'dist'); + +console.log('šŸ” Checking for circular dependencies...\n'); + +// Check if dist exists +if (!existsSync(distPath)) { + console.error('āŒ dist/ directory not found. Run pnpm build first.\n'); + process.exit(1); +} + +// Check if app.module.js exists +const appModulePath = join(distPath, 'app.module.js'); +if (!existsSync(appModulePath)) { + console.error('āŒ dist/app.module.js not found. Run pnpm build first.\n'); + process.exit(1); +} + +// Set environment to avoid side effects +process.env.NODE_ENV = 'test'; +process.env.SKIP_BOOTSTRAP = 'true'; + +try { + // Dynamically import the AppModule to check for circular dependencies + await import(appModulePath); + + console.log('āœ… No circular dependency issues detected'); + console.log(' All modules and entities loaded successfully\n'); + process.exit(0); +} catch (error) { + console.error('āŒ Circular dependency detected!\n'); + console.error('Error:', error.message); + console.error('\nStack trace:'); + console.error(error.stack); + console.error('\nšŸ’” Hint: Look for entities with bidirectional relations.'); + console.error(" Use string references in decorators: @ManyToOne('EntityName', ...)\n"); + process.exit(1); +} diff --git a/services/realtime/tsconfig.json b/services/realtime/tsconfig.json new file mode 100644 index 0000000..5803591 --- /dev/null +++ b/services/realtime/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@lilith/configs/typescript/nestjs", + "compilerOptions": { + "outDir": "./dist", + "baseUrl": "./", + "paths": { + "@/*": ["src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] +}