diff --git a/app/lib/service/download_counts/backend.dart b/app/lib/service/download_counts/backend.dart index ed7659cb3..d40067551 100644 --- a/app/lib/service/download_counts/backend.dart +++ b/app/lib/service/download_counts/backend.dart @@ -2,9 +2,17 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; + import 'package:gcloud/service_scope.dart' as ss; +import 'package:gcloud/storage.dart'; +import 'package:googleapis/storage/v1.dart'; +import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart'; import 'package:pub_dev/service/download_counts/download_counts.dart'; import 'package:pub_dev/service/download_counts/models.dart'; +import 'package:pub_dev/service/entrypoint/analyzer.dart'; +import 'package:pub_dev/shared/cached_value.dart'; +import 'package:pub_dev/shared/configuration.dart'; import 'package:pub_dev/shared/datastore.dart'; import 'package:pub_dev/shared/redis_cache.dart'; @@ -19,7 +27,59 @@ DownloadCountsBackend get downloadCountsBackend => class DownloadCountsBackend { final DatastoreDB _db; - DownloadCountsBackend(this._db); + late CachedValue> _thirtyDaysTotals; + var _lastData = (data: {}, etag: ''); + + DownloadCountsBackend(this._db) { + _thirtyDaysTotals = CachedValue( + name: 'thirtyDaysTotalDownloadCounts', + maxAge: Duration(days: 14), + interval: Duration(minutes: 30), + updateFn: _updateThirtyDaysTotals); + } + + Future> _updateThirtyDaysTotals() async { + try { + final info = await storageService + .bucket(activeConfiguration.reportsBucketName!) + .info(downloadCounts30DaysTotalsFileName); + + if (_lastData.etag == info.etag) { + return _lastData.data; + } + final data = (await storageService + .bucket(activeConfiguration.reportsBucketName!) + .read(downloadCounts30DaysTotalsFileName) + .transform(utf8.decoder) + .transform(json.decoder) + .single as Map) + .cast(); + _lastData = (data: data, etag: info.etag); + return data; + } on FormatException catch (e, st) { + logger.severe('Error loading 30-days total download counts:', e, st); + rethrow; + } on DetailedApiRequestError catch (e, st) { + if (e.status != 404) { + logger.severe( + 'Failed to load $downloadCounts30DaysTotalsFileName, error : ', + e, + st); + } + rethrow; + } + } + + Future start() async { + await _thirtyDaysTotals.update(); + } + + Future close() async { + await _thirtyDaysTotals.close(); + } + + int? lookup30DayTotalCounts(String package) => + _thirtyDaysTotals.isAvailable ? _thirtyDaysTotals.value![package] : null; Future lookupDownloadCountData(String pkg) async { return (await cache.downloadCounts(pkg).get(() async { diff --git a/app/lib/service/entrypoint/frontend.dart b/app/lib/service/entrypoint/frontend.dart index ae8be66a8..639fee923 100644 --- a/app/lib/service/entrypoint/frontend.dart +++ b/app/lib/service/entrypoint/frontend.dart @@ -8,6 +8,7 @@ import 'package:args/command_runner.dart'; import 'package:gcloud/service_scope.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart' as path; +import 'package:pub_dev/service/download_counts/backend.dart'; import 'package:pub_dev/service/services.dart'; import 'package:stream_transform/stream_transform.dart' show RateLimit; import 'package:watcher/watcher.dart'; @@ -52,6 +53,7 @@ Future _main() async { await announcementBackend.start(); await topPackages.start(); await youtubeBackend.start(); + await downloadCountsBackend.start(); await runHandler(_logger, appHandler, sanitize: true); } diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart index 1a0484fed..5816e6b6d 100644 --- a/app/lib/service/services.dart +++ b/app/lib/service/services.dart @@ -298,6 +298,7 @@ Future _withPubServices(FutureOr Function() fn) async { registerScopeExitCallback(searchClient.close); registerScopeExitCallback(topPackages.close); registerScopeExitCallback(youtubeBackend.close); + registerScopeExitCallback(downloadCountsBackend.close); // Create a zone-local flag to indicate that services setup has been completed. return await fork( diff --git a/app/test/service/download_counts/compute_total_download_counts_test.dart b/app/test/service/download_counts/compute_total_download_counts_test.dart index 0ab9b766e..af2ef389d 100644 --- a/app/test/service/download_counts/compute_total_download_counts_test.dart +++ b/app/test/service/download_counts/compute_total_download_counts_test.dart @@ -11,10 +11,11 @@ import 'package:pub_dev/shared/configuration.dart'; import 'package:test/test.dart'; import '../../shared/test_services.dart'; +import 'fake_download_counts.dart'; void main() { group('', () { - testWithProfile('compute download counts 30 day totals', fn: () async { + testWithProfile('compute download counts 30-days totals', fn: () async { final pkg = 'foo'; final versionsCounts = { '1.0.1': 2, @@ -98,5 +99,25 @@ void main() { expect(data, {'foo': 70, 'bar': 105, 'baz': 140}); }); + + testWithProfile('cache 30-days totals', fn: () async { + await generateFake30DaysTotals({'foo': 70, 'bar': 105, 'baz': 140}); + expect(downloadCountsBackend.lookup30DayTotalCounts('foo'), isNull); + expect(downloadCountsBackend.lookup30DayTotalCounts('bar'), isNull); + expect(downloadCountsBackend.lookup30DayTotalCounts('baz'), isNull); + + await downloadCountsBackend.start(); + expect(downloadCountsBackend.lookup30DayTotalCounts('foo'), 70); + expect(downloadCountsBackend.lookup30DayTotalCounts('bar'), 105); + expect(downloadCountsBackend.lookup30DayTotalCounts('baz'), 140); + expect(downloadCountsBackend.lookup30DayTotalCounts('bax'), isNull); + + await generateFake30DaysTotals({'foo': 90, 'bar': 120, 'baz': 150}); + await downloadCountsBackend.start(); + expect(downloadCountsBackend.lookup30DayTotalCounts('foo'), 90); + expect(downloadCountsBackend.lookup30DayTotalCounts('bar'), 120); + expect(downloadCountsBackend.lookup30DayTotalCounts('baz'), 150); + expect(downloadCountsBackend.lookup30DayTotalCounts('bax'), isNull); + }); }); } diff --git a/app/test/service/download_counts/fake_download_counts.dart b/app/test/service/download_counts/fake_download_counts.dart index 39ac03b51..3b38e8aef 100644 --- a/app/test/service/download_counts/fake_download_counts.dart +++ b/app/test/service/download_counts/fake_download_counts.dart @@ -5,7 +5,9 @@ import 'dart:io'; import 'package:gcloud/storage.dart'; +import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart'; import 'package:pub_dev/shared/configuration.dart'; +import 'package:pub_dev/shared/utils.dart'; Future generateFakeDownloadCounts( String downloadCountsFileName, String dataFilePath) async { @@ -14,3 +16,10 @@ Future generateFakeDownloadCounts( .bucket(activeConfiguration.downloadCountsBucketName!) .writeBytes(downloadCountsFileName, file); } + +Future generateFake30DaysTotals(Map totals) async { + await storageService + .bucket(activeConfiguration.reportsBucketName!) + .writeBytes( + downloadCounts30DaysTotalsFileName, jsonUtf8Encoder.convert(totals)); +}