Fixing Upgrades fetch failure (#350)

* The github API has a default limit on records from the release query. Updated to include count and page.
* Updated Pricing Engine API pathing

closes: #349

CHANGELOG.md

@@ -57,6 +57,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
   * `/api/v2/smart_contract/code/{id}/contracts` - added filters on `creator`, `admin`
 * Added API to fetch list of unique non-UUID contract labels #339
   * `/api/v2/smart_contract/contract/labels`
+* Updated the Pricing Engine API pathing
 
 ### Bug Fixes
 * Updated how the service runs on an empty DB
@@ -66,6 +67,8 @@ Ref: https://keepachangelog.com/en/1.0.0/
   * Related to incorrect value mapping
   * Added sort to block retry so they process in height order
 * Fixed how the weighted vote percentage was being ingested
+* Fixed chain version fetch from github #349
+  * Due to a limit of count on records from teh Github API
 
 ### Data
 * Migration 1.58 - Add weight to proposal votes #323

docker/docker-compose.yml

@@ -37,7 +37,7 @@ services:
       - EXPLORER_HIDDEN_APIS=false
       - EXPLORER_SWAGGER_URL=localhost:8612
       - EXPLORER_SWAGGER_PROTOCOL=http
-      - EXPLORER_PRICING_URL=https://test.figure.tech/service-pricing-engine/service-pricing-engine
+      - EXPLORER_PRICING_URL=https://test.figure.tech/service-pricing-engine/external
     depends_on:
       - explorer-postgres
     links:

service/src/main/kotlin/io/provenance/explorer/service/ExplorerService.kt

@@ -204,7 +204,7 @@ class ExplorerService(
         val knownReleases =
             CacheUpdateRecord.fetchCacheByKey(CacheKeys.CHAIN_RELEASES.key)?.cacheValue?.let {
                 VANILLA_MAPPER.readValue<List<GithubReleaseData>>(it)
-            } ?: getChainReleases()
+            } ?: getAllChainReleases()
         val genesis = props.genesisVersionUrl.getLatestPatchVersion(
             knownReleases,
             props.upgradeVersionRegex,
@@ -241,9 +241,22 @@ class ExplorerService(
         return (listOf(genesis) + upgrades).sortedBy { it.upgradeHeight }
     }
 
+    fun getAllChainReleases(): MutableList<GithubReleaseData> {
+        var page = 1
+        val pageCount = 100
+        val records = getChainReleases(page, pageCount)
+        while (records.size == pageCount * page) {
+            page++
+            records.addAll(getChainReleases(page, pageCount))
+        }
+        if (records.isNotEmpty())
+            CacheUpdateRecord.updateCacheByKey(CacheKeys.CHAIN_RELEASES.key, VANILLA_MAPPER.writeValueAsString(records))
+        return records
+    }
+
     // Fetches and saves the ordered list of releases
-    fun getChainReleases(): MutableList<GithubReleaseData> = runBlocking {
-        val url = "https://api.github.com/repos/${props.upgradeGithubRepo}/releases"
+    fun getChainReleases(page: Int, count: Int): MutableList<GithubReleaseData> = runBlocking {
+        val url = "https://api.github.com/repos/${props.upgradeGithubRepo}/releases?per_page=$count&page=$page"
         val res = try {
             KTOR_CLIENT_JAVA.get<HttpResponse>(url)
         } catch (e: ResponseException) {
@@ -265,9 +278,6 @@ class ExplorerService(
                 mutableListOf<GithubReleaseData>().also { logger.error("Error: $e") }
             }
         } else mutableListOf<GithubReleaseData>().also { logger.error("Error reaching Pricing Engine: ${res.status.value}") }
-    }.also {
-        if (it.isNotEmpty())
-            CacheUpdateRecord.updateCacheByKey(CacheKeys.CHAIN_RELEASES.key, VANILLA_MAPPER.writeValueAsString(it))
     }
 
     fun String.getLatestPatchVersion(

service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt

@@ -193,7 +193,7 @@ class AsyncService(
     }
 
     @Scheduled(cron = "0 0/15 * * * ?") // Every 15 minutes
-    fun updateReleaseVersions() = explorerService.getChainReleases()
+    fun updateReleaseVersions() = explorerService.getAllChainReleases()
 
     @Scheduled(cron = "0 0 0/1 * * ?") // Every hour
     fun saveChainAum() = explorerService.saveChainAum()

service/src/main/resources/application-development.properties

@@ -20,13 +20,13 @@ explorer.swagger-protocol=http
 #explorer.mainnet=true
 #explorer.pb-url=grpcs://grpc.provenance.io:443
 #explorer.figment-url=https://pio-mainnet-1--lcd--archive.datahub.figment.io
-#explorer.pricing-url=https://figure.tech/service-pricing-engine/service-pricing-engine
+#explorer.pricing-url=https://figure.tech/service-pricing-engine/external
 #explorer.genesis-version-url=https://github.com/provenance-io/provenance/releases/download/v1.0.1/plan-v1.0.1.json
 
 
 #### TESTNET SETTINGS
 explorer.mainnet=false
-explorer.pricing-url=https://test.figure.tech/service-pricing-engine/service-pricing-engine
+explorer.pricing-url=https://test.figure.tech/service-pricing-engine/external
 explorer.figment-url=https://pio-testnet-1--lcd.datahub.figment.io
 explorer.pb-url=grpcs://grpc.test.provenance.io:443
 explorer.genesis-version-url=https://github.com/provenance-io/provenance/releases/download/v0.2.0/plan-v0.2.0.json