Merge pull request stellar#4489 from stellar/release-horizon-v2.19.0

horizon: merge branch 'release-horizon-v2.19.0' into 'master'

Makefile

@@ -35,7 +35,7 @@ gxdr/xdr_generated.go: $(XDRS)
 	go fmt $@
 
 xdr/%.x:
-	curl -Lsf -o $@ https://raw.githubusercontent.com/stellar/stellar-core/master/src/$@
+	curl -Lsf -o $@ https://raw.githubusercontent.com/stellar/stellar-core/master/src/protocol-curr/$@
 
 xdr/xdr_generated.go: $(XDRS)
 	docker run -it --rm -v $$PWD:/wd -w /wd ruby /bin/bash -c '\

ingest/ledgerbackend/stellar_core_runner.go

@@ -267,7 +267,7 @@ func (r *stellarCoreRunner) createCmd(params ...string) (cmdI, error) {
 		r.log.Debugf("captive core config file contents:\n%s", conf)
 	}
 
-	allParams := append([]string{"--conf", r.getConfFileName()}, params...)
+	allParams := append([]string{"--conf", r.getConfFileName(), "--console"}, params...)
 	cmd := r.systemCaller.command(r.executablePath, allParams...)
 	cmd.setDir(r.storagePath)
 	cmd.setStdout(r.getLogLineWriter())

ingest/ledgerbackend/stellar_core_runner_test.go

@@ -39,6 +39,7 @@ func TestCloseOffline(t *testing.T) {
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"catchup",
 		"200/101",
 		"--metadata-output-stream",
@@ -79,6 +80,7 @@ func TestCloseOnline(t *testing.T) {
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"run",
 		"--in-memory",
 		"--start-at-ledger",
@@ -121,6 +123,7 @@ func TestCloseOnlineWithError(t *testing.T) {
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"run",
 		"--in-memory",
 		"--start-at-ledger",
@@ -188,6 +191,7 @@ func TestRunFromUseDBLedgersMatch(t *testing.T) {
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"run",
 		"--metadata-output-stream",
 		"fd:3",
@@ -249,19 +253,22 @@ func TestRunFromUseDBLedgersNotMatch(t *testing.T) {
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"new-db",
 	).Return(newDBCmdMock)
 	scMock.On("command",
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"catchup",
 		"99/0",
 	).Return(catchupCmdMock)
 	scMock.On("command",
 		"/usr/bin/stellar-core",
 		"--conf",
 		mock.Anything,
+		"--console",
 		"run",
 		"--metadata-output-stream",
 		"fd:3",

services/horizon/CHANGELOG.md

@@ -4,15 +4,38 @@ All notable changes to this project will be documented in this
 file. This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Pending
-- Run postgres autovacuum on `history_trades_60000` table more frequently
-  ([4412](https://github.com/stellar/go/pull/4412)).
 - Added indexes by id for claimable balance and liquidity pool id's in the respective tx/ops tables ([4455](https://github.com/stellar/go/pull/4477))
 - Improve restart time of Captive-Core when started with `--captive-core-use-db` flag. The solution does not work on Windows. ([4471)](https://github.com/stellar/go/pull/4471))
 
+## 2.19.0
+
+**Upgrading to this version from <= v2.8.3 will trigger a state rebuild. During this process (which will take at least 10 minutes), Horizon will not ingest new ledgers.**
+
+### Breaking Changes
+
+* Update core version to 19.3.0 ([4485](https://github.com/stellar/go/pull/4485)).
+* Pass `--console` to captive core. This is due to a breaking change in stellar-core 19.3.0 ([4487](https://github.com/stellar/go/pull/4487)).
+
+### Changes
+
+* Run postgres autovacuum on `history_trades_60000` table more frequently. ([4412](https://github.com/stellar/go/pull/4412)).
+* Change `protocols/horizon.Transaction.AccountSequence` to `int64` from `string`. ([4409](https://github.com/stellar/go/pull/4409)).
+* Add missing signer key type names. ([4429](https://github.com/stellar/go/pull/4429)).
+* Update core version to 19.2.0. ([4441](https://github.com/stellar/go/pull/4441)).
+* Add `User-Agent` header to history archive HTTP requests. ([4463](https://github.com/stellar/go/pull/4463)).
+
+### DB Schema Migration
+
+The migration makes the following schema changes:
+
+  - tunes the `history_trades_60000` table to run autovacuum more frequently. Migration should be brief, does not incur any data/runtime processing. It does not need a reingestion.
+
 ## 2.18.1
+
 * Enabled txsub system to work if/when underlying horizon db connection is read only. ([4418](https://github.com/stellar/go/pull/4418))
 * Optimize the claimable balance SQL query for best index query plan performance([4398](https://github.com/stellar/go/pull/4398))
 
+
 ### DB Schema Migration
 
 The migration makes the following schema changes:

services/horizon/internal/ingest/filters/main.go

@@ -2,6 +2,7 @@ package filters
 
 import (
 	"context"
+	"sync"
 	"time"
 
 	"github.com/stellar/go/services/horizon/internal/db2/history"
@@ -13,9 +14,22 @@ var (
 
 	// the filter config cache will be checked against latest from db at most once per each of this interval.
 	//lint:ignore ST1011, don't need the linter warn on literal assignment
-	FilterConfigCheckIntervalSeconds time.Duration = 100
+	filterConfigCheckIntervalSeconds     time.Duration = 100
+	filterConfigCheckIntervalSecondsLock sync.RWMutex
 )
 
+func GetFilterConfigCheckIntervalSeconds() time.Duration {
+	filterConfigCheckIntervalSecondsLock.RLock()
+	defer filterConfigCheckIntervalSecondsLock.RUnlock()
+	return filterConfigCheckIntervalSeconds
+}
+
+func SetFilterConfigCheckIntervalSeconds(t time.Duration) {
+	filterConfigCheckIntervalSecondsLock.Lock()
+	defer filterConfigCheckIntervalSecondsLock.Unlock()
+	filterConfigCheckIntervalSeconds = t
+}
+
 var (
 	LOG = log.WithFields(log.F{
 		"filters": "load",
@@ -43,7 +57,7 @@ func NewFilters() Filters {
 // rebuild the list on expiration time interval. Method is NOT thread-safe.
 func (f *filtersCache) GetFilters(filterQ history.QFilter, ctx context.Context) []processors.LedgerTransactionFilterer {
 	// only attempt to refresh filter config cache state at configured interval limit
-	if time.Now().Unix() < (f.lastFilterConfigCheckUnixEpoch + int64(FilterConfigCheckIntervalSeconds.Seconds())) {
+	if time.Now().Unix() < (f.lastFilterConfigCheckUnixEpoch + int64(GetFilterConfigCheckIntervalSeconds().Seconds())) {
 		return f.convertCacheToList()
 	}
 

services/horizon/internal/integration/ingestion_filtering_test.go

@@ -48,7 +48,7 @@ func TestFilteringAccountWhiteList(t *testing.T) {
 	tt.NoError(err)
 
 	// Setup a whitelisted account rule, force refresh of filter configs to be quick
-	filters.FilterConfigCheckIntervalSeconds = 1
+	filters.SetFilterConfigCheckIntervalSeconds(1)
 
 	expectedAccountFilter := hProtocol.AccountFilterConfig{
 		Whitelist: []string{whitelistedAccount.GetAccountID()},
@@ -64,7 +64,7 @@ func TestFilteringAccountWhiteList(t *testing.T) {
 	tt.Equal(expectedAccountFilter.Enabled, accountFilter.Enabled)
 
 	// Ensure the latest filter configs are reloaded by the ingestion state machine processor
-	time.Sleep(time.Duration(filters.FilterConfigCheckIntervalSeconds) * time.Second)
+	time.Sleep(time.Duration(filters.GetFilterConfigCheckIntervalSeconds()) * time.Second)
 
 	// Make sure that when using a non-whitelisted account, the transaction is not stored
 	txResp = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(),
@@ -123,7 +123,7 @@ func TestFilteringAssetWhiteList(t *testing.T) {
 	tt.NoError(err)
 
 	// Setup a whitelisted asset rule, force refresh of filters to be quick
-	filters.FilterConfigCheckIntervalSeconds = 1
+	filters.SetFilterConfigCheckIntervalSeconds(1)
 
 	asset, err := whitelistedAsset.ToXDR()
 	tt.NoError(err)
@@ -141,7 +141,7 @@ func TestFilteringAssetWhiteList(t *testing.T) {
 	tt.Equal(expectedAssetFilter.Enabled, assetFilter.Enabled)
 
 	// Ensure the latest filter configs are reloaded by the ingestion state machine processor
-	time.Sleep(time.Duration(filters.FilterConfigCheckIntervalSeconds) * time.Second)
+	time.Sleep(time.Duration(filters.GetFilterConfigCheckIntervalSeconds()) * time.Second)
 
 	// Make sure that when using a non-whitelisted asset, the transaction is not stored
 	txResp = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(),