API v1 (#6)

* getProviderClients

* getProviderBiggestClientDistribution

* bigInt returned as int now
url format change

* removed -week- from histogramDto

* aggregation retry

* weekly count

* cleanup

* Provider Retrievability

* cleanup, allocators API wireframe

* enabled CORS

* Allocator biggest client distribution (#8)

* Add SQL for getting allocator biggest client distribution

* Implement allocator biggest client distribution API

* Update src/aggregation/aggregation-tasks.service.ts

---------

Co-authored-by: neti-mateusz-mazur <[email protected]>

* cleanup

* histogram results include same number of buckets

* Allocator Retrievability

* getAllocatorSpsCompliance

* Update src/service/allocator/allocator.service.ts

Co-authored-by: Kacper Żuk (Neti) <[email protected]>

* renamed histogram property

* getAllocatorRetrievability average taken from last week

* getProviderRetrievability success average taken from last week

* getProviderRetrievability switched to providers_weekly

* switched Retrievability to only take full week for average calculation

* switched dates to luxon

---------

Co-authored-by: Kacper Żuk (Neti) <[email protected]>

package.json

@@ -21,40 +21,42 @@
   },
   "dependencies": {
     "@nestjs/axios": "^3.0.3",
-    "@nestjs/common": "^10.0.0",
+    "@nestjs/common": "^10.4.1",
     "@nestjs/config": "^3.2.3",
-    "@nestjs/core": "^10.0.0",
-    "@nestjs/platform-express": "^10.0.0",
+    "@nestjs/core": "^10.4.1",
+    "@nestjs/platform-express": "^10.4.1",
     "@nestjs/schedule": "^4.1.0",
     "@prisma/client": "^5.19.1",
     "axios": "^1.7.7",
+    "lodash": "^4.17.21",
     "luxon": "^3.5.0",
-    "reflect-metadata": "^0.2.0",
+    "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^10.0.0",
-    "@nestjs/schematics": "^10.0.0",
-    "@nestjs/testing": "^10.0.0",
-    "@types/express": "^4.17.17",
-    "@types/jest": "^29.5.2",
-    "@types/node": "^20.3.1",
-    "@types/supertest": "^6.0.0",
-    "@typescript-eslint/eslint-plugin": "^8.0.0",
-    "@typescript-eslint/parser": "^8.0.0",
-    "eslint": "^8.42.0",
-    "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-prettier": "^5.0.0",
-    "jest": "^29.5.0",
-    "prettier": "^3.0.0",
+    "@nestjs/cli": "^10.4.5",
+    "@nestjs/schematics": "^10.1.4",
+    "@nestjs/testing": "^10.4.1",
+    "@types/express": "^4.17.21",
+    "@types/jest": "^29.5.12",
+    "@types/lodash": "^4.17.7",
+    "@types/node": "^20.16.5",
+    "@types/supertest": "^6.0.2",
+    "@typescript-eslint/eslint-plugin": "^8.5.0",
+    "@typescript-eslint/parser": "^8.5.0",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.2.1",
+    "jest": "^29.7.0",
+    "prettier": "^3.3.3",
     "prisma": "^5.19.1",
     "source-map-support": "^0.5.21",
     "supertest": "^7.0.0",
-    "ts-jest": "^29.1.0",
-    "ts-loader": "^9.4.3",
-    "ts-node": "^10.9.1",
+    "ts-jest": "^29.2.5",
+    "ts-loader": "^9.5.1",
+    "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.1.3"
+    "typescript": "^5.6.2"
   },
   "jest": {
     "moduleFileExtensions": [

prisma/sql/getAllocatorBiggestClientDistribution.sql

@@ -0,0 +1,23 @@
+with allocators_with_ratio as (
+    select
+        week,
+        allocator,
+        max(sum_of_allocations) / sum(sum_of_allocations) biggest_to_total_ratio
+    from client_allocator_distribution_weekly
+    group by
+        week,
+        allocator
+)
+select
+    week,
+    100 * ceil(biggest_to_total_ratio::float8 * 20) / 20 - 5 as "valueFromExclusive",
+    100 * ceil(biggest_to_total_ratio::float8 * 20) / 20 as "valueToInclusive",
+    count(*)::int as count
+from allocators_with_ratio
+group by
+    week,
+    "valueFromExclusive",
+    "valueToInclusive"
+order by
+    week,
+    "valueFromExclusive";

prisma/sql/getAllocatorRetrievability.sql

@@ -0,0 +1,8 @@
+select
+    ceil(avg_weighted_retrievability_success_rate*20)*5 - 5 as "valueFromExclusive",
+    ceil(avg_weighted_retrievability_success_rate*20)*5 as "valueToInclusive",
+    count(*)::int as "count",
+    week
+from allocators_weekly
+group by 1, 2, week
+order by week;

prisma/sql/getProviderBiggestClientDistribution.sql

@@ -0,0 +1,12 @@
+with providers_with_ratio as (select provider,
+                                     max(total_deal_size) / sum(total_deal_size) biggest_to_total_ratio,
+                                     week
+                              from client_provider_distribution_weekly
+                              group by provider, week)
+select 100 * ceil(biggest_to_total_ratio::float8 * 20) / 20 - 5 as "valueFromExclusive",
+       100 * ceil(biggest_to_total_ratio::float8 * 20) / 20     as "valueToInclusive",
+       count(*)::int as "count",
+       week
+from providers_with_ratio
+group by "valueFromExclusive", "valueToInclusive", week
+order by week, 1;

prisma/sql/getProviderClientsWeekly.sql

@@ -0,0 +1,11 @@
+with clients_per_provider as (select count(distinct client) as clients_count,
+                                     week
+                              from client_provider_distribution_weekly
+                              group by provider, week)
+select (clients_count - 1)::float as "valueFromExclusive",
+       clients_count::float       as "valueToInclusive",
+       week                       as "week",
+       count(*)::int as "count"
+from clients_per_provider
+group by 1, 2, 3
+order by 3, 1;

prisma/sql/getProviderRetrievability.sql

@@ -0,0 +1,7 @@
+select 100 * ceil(avg_retrievability_success_rate * 20) / 20 - 5 as "valueFromExclusive",
+       100 * ceil(avg_retrievability_success_rate * 20) / 20     as "valueToInclusive",
+       count(*)::int                                             as "count",
+       week                                                      as "week"
+from providers_weekly
+group by 1, 2, 4
+order by 1;

src/aggregation/aggregation-tasks.service.ts

@@ -14,14 +14,23 @@ export class AggregationTasksService {
     if (!this.aggregationJobInProgress) {
       this.aggregationJobInProgress = true;
 
-      try {
-        this.logger.debug('Starting Aggregations');
+      let success = false;
+      let executionNumber = 0;
+      //retry up to 3 times in case of error
+      while (!success && executionNumber < 3) {
+        try {
+          this.logger.debug('Starting Aggregations');
 
-        await this.aggregationService.runAggregations();
+          await this.aggregationService.runAggregations();
+          success = true;
 
-        this.logger.debug('Finished Aggregations');
-      } catch (err) {
-        this.logger.error(`Error during Aggregations job: ${err}`);
+          this.logger.debug('Finished Aggregations');
+        } catch (err) {
+          this.logger.error(
+            `Error during Aggregations job, execution ${executionNumber}: ${err}`,
+          );
+          executionNumber++;
+        }
       }
 
       this.aggregationJobInProgress = false;

src/app.module.ts

@@ -1,6 +1,4 @@
 import { Module } from '@nestjs/common';
-import { AppController } from './app.controller';
-import { AppService } from './app.service';
 import { ConfigModule } from '@nestjs/config';
 import { ScheduleModule } from '@nestjs/schedule';
 import { AggregationTasksService } from './aggregation/aggregation-tasks.service';
@@ -21,16 +19,20 @@ import { UnifiedVerifiedDealRunner } from './aggregation/runners/unified-verifie
 import { HttpModule } from '@nestjs/axios';
 import { FilSparkService } from './filspark/filspark.service';
 import { ProviderRetrievabilityBackfillRunner } from './aggregation/runners/provider-retrievability-backfill.runner';
+import { ProvidersController } from './controller/stats/providers/providers.controller';
+import { ProviderService } from './service/provider/provider.service';
+import { AllocatorsController } from './controller/stats/allocators/allocators.controller';
+import { AllocatorService } from './service/allocator/allocator.service';
+import { HistogramHelper } from './helper/histogram.helper';
 
 @Module({
   imports: [
     ConfigModule.forRoot(),
     ScheduleModule.forRoot(),
     HttpModule.register({ timeout: 5000 }),
   ],
-  controllers: [AppController],
+  controllers: [ProvidersController, AllocatorsController],
   providers: [
-    AppService,
     AggregationService,
     AggregationTasksService,
     PrismaService,
@@ -47,6 +49,9 @@ import { ProviderRetrievabilityBackfillRunner } from './aggregation/runners/prov
     ProviderRetrievabilityBackfillRunner,
     ProvidersRunner,
     UnifiedVerifiedDealRunner,
+    ProviderService,
+    AllocatorService,
+    HistogramHelper,
     {
       provide: 'AggregationRunner',
       useFactory: (

src/controller/stats/allocators/allocators.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { AllocatorsController } from './allocators.controller';
+
+describe('AllocatorsController', () => {
+  let controller: AllocatorsController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [AllocatorsController],
+    }).compile();
+
+    controller = module.get<AllocatorsController>(AllocatorsController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

src/controller/stats/allocators/allocators.controller.ts

@@ -0,0 +1,22 @@
+import { Controller, Get } from '@nestjs/common';
+import { AllocatorService } from '../../../service/allocator/allocator.service';
+
+@Controller('stats/allocators')
+export class AllocatorsController {
+  constructor(private readonly allocatorService: AllocatorService) {}
+
+  @Get('retrievability')
+  async getAllocatorRetrievability() {
+    return await this.allocatorService.getAllocatorRetrievability();
+  }
+
+  @Get('biggest-client-distribution')
+  async getAllocatorBiggestClientDistribution() {
+    return await this.allocatorService.getAllocatorBiggestClientDistribution();
+  }
+
+  @Get('sps-compliance')
+  async getAllocatorSpsCompliance() {
+    return await this.allocatorService.getAllocatorSpsCompliance();
+  }
+}

src/controller/stats/providers/providers.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ProvidersController } from './providers.controller';
+
+describe('ProvidersController', () => {
+  let controller: ProvidersController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [ProvidersController],
+    }).compile();
+
+    controller = module.get<ProvidersController>(ProvidersController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

src/controller/stats/providers/providers.controller.ts

@@ -0,0 +1,22 @@
+import { Controller, Get } from '@nestjs/common';
+import { ProviderService } from '../../../service/provider/provider.service';
+
+@Controller('stats/providers')
+export class ProvidersController {
+  constructor(private readonly providerService: ProviderService) {}
+
+  @Get('clients')
+  async getProviderClients() {
+    return await this.providerService.getProviderClients();
+  }
+
+  @Get('biggest-client-distribution')
+  async getProviderBiggestClientDistribution() {
+    return await this.providerService.getProviderBiggestClientDistribution();
+  }
+
+  @Get('retrievability')
+  async getProviderRetrievability() {
+    return await this.providerService.getProviderRetrievability();
+  }
+}

src/helper/histogram.helper.ts

@@ -0,0 +1,90 @@
+import { Injectable } from '@nestjs/common';
+import { groupBy } from 'lodash';
+import { HistogramWeekDto } from '../types/histogramWeek.dto';
+import { HistogramDto } from '../types/histogram.dto';
+import { HistogramWeekResponseDto } from '../types/histogramWeek.response.dto';
+
+@Injectable()
+export class HistogramHelper {
+  async getWeeklyHistogramResult(
+    results: {
+      valueFromExclusive: number | null;
+      valueToInclusive: number | null;
+      count: number | null;
+      week: Date;
+    }[],
+    totalCount: number,
+  ): Promise<HistogramWeekResponseDto> {
+    const resultsByWeek = groupBy(results, (p) => p.week);
+
+    const histogramWeekDtos: HistogramWeekDto[] = [];
+    for (const key in resultsByWeek) {
+      const value = resultsByWeek[key];
+      const weekResponses = value.map((r) => {
+        return new HistogramDto(
+          r.valueFromExclusive,
+          r.valueToInclusive,
+          r.count,
+        );
+      });
+      histogramWeekDtos.push(
+        new HistogramWeekDto(
+          new Date(key),
+          weekResponses,
+          weekResponses.reduce(
+            (partialSum, response) => partialSum + response.count,
+            0,
+          ),
+        ),
+      );
+    }
+
+    // calculate missing histogram buckets
+    const { maxMinSpan, allBucketTopValues } =
+      this.getAllHistogramBucketTopValues(histogramWeekDtos);
+
+    for (const histogramWeekDto of histogramWeekDtos) {
+      const missingValues = allBucketTopValues.filter(
+        (topValue) =>
+          !histogramWeekDto.results.some(
+            (p) => p.valueToInclusive === topValue,
+          ),
+      );
+
+      if (missingValues.length > 0) {
+        histogramWeekDto.results.push(
+          ...missingValues.map((v) => new HistogramDto(v - maxMinSpan, v, 0)),
+        );
+
+        histogramWeekDto.results.sort(
+          (a, b) => a.valueToInclusive - b.valueToInclusive,
+        );
+      }
+    }
+
+    return new HistogramWeekResponseDto(totalCount, histogramWeekDtos);
+  }
+
+  private getAllHistogramBucketTopValues(
+    histogramWeekDtos: HistogramWeekDto[],
+  ) {
+    const maxRangeTopValue = Math.max(
+      ...histogramWeekDtos.flatMap((p) =>
+        p.results.map((r) => r.valueToInclusive),
+      ),
+    );
+
+    const maxHistogramEntry = histogramWeekDtos
+      .flatMap((p) => p.results)
+      .find((p) => p.valueToInclusive === maxRangeTopValue);
+
+    const maxMinSpan =
+      maxHistogramEntry.valueToInclusive - maxHistogramEntry.valueFromExclusive;
+
+    const allBucketTopValues: number[] = [];
+    for (let i = maxRangeTopValue; i > 0; i -= maxMinSpan) {
+      allBucketTopValues.push(i);
+    }
+    return { maxMinSpan, allBucketTopValues };
+  }
+}