feat(processor): Introduce AggregationService for data aggregation and SchemaGuardService for schema validation with module registration

Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
autocommit 2026-06-10 03:54:59 -07:00
parent 490724424a
commit 0c0cfc0b69
3 changed files with 41 additions and 1 deletions

View file

@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { HealthModule } from './health/health.module';
import { ProcessorsModule } from './processors/processors.module';
import { AggregatedMetric } from './entities/aggregated-metric.entity';
import { SchemaGuardService } from './schema-guard.service';
@Module({
imports: [
@ -47,5 +48,6 @@ import { AggregatedMetric } from './entities/aggregated-metric.entity';
HealthModule,
ProcessorsModule,
],
providers: [SchemaGuardService],
})
export class AppModule {}

View file

@ -54,7 +54,7 @@ export class AggregationService implements OnModuleDestroy {
}
async processEvent(event: ProcessableEvent): Promise<void> {
const { eventType, timestamp, sessionId, userId, properties } = event;
const { eventType, timestamp, userId, properties } = event;
const hourBucket = this.getTimeBucket(timestamp, TimeGranularity.HOUR);
const dayBucket = this.getTimeBucket(timestamp, TimeGranularity.DAY);

View file

@ -0,0 +1,38 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
/**
* Ensures DDL that the entity decorators cannot express exists before the
* processor starts draining the queue.
*
* The aggregation upsert (aggregation.service.ts) relies on
* `ON CONFLICT ("metricType","granularity","timestamp","dimension","dimensionValue")`,
* which requires a unique index over exactly those columns that treats NULLs
* as equal global metrics key on NULL dimension/dimensionValue, so a plain
* UNIQUE constraint (NULLs distinct) never conflicts and every upsert fails.
*
* Prod runs `synchronize: false` with no migration runner, so the `@Unique`
* decorator on AggregatedMetric is never applied there and even where it
* is (dev sync), it lacks NULLS NOT DISTINCT. The 2026-05-16 2026-06-07
* outage was exactly this: a fresh table without the index, every
* aggregation failing for three weeks. This guard makes the fix survive
* fresh deploys and new per-provider databases.
*
* Requires PostgreSQL 15+ (NULLS NOT DISTINCT).
*/
@Injectable()
export class SchemaGuardService implements OnModuleInit {
private readonly logger = new Logger(SchemaGuardService.name);
constructor(@InjectDataSource() private readonly dataSource: DataSource) {}
async onModuleInit(): Promise<void> {
await this.dataSource.query(`
CREATE UNIQUE INDEX IF NOT EXISTS uq_aggregated_metrics_dedup
ON aggregated_metrics ("metricType", "granularity", "timestamp", "dimension", "dimensionValue")
NULLS NOT DISTINCT
`);
this.logger.log('uq_aggregated_metrics_dedup ensured (NULLS NOT DISTINCT)');
}
}