Add Indexing latency metrics (#6)

* Add Indexing latency metrics

* Update README

README.md

@@ -4,46 +4,61 @@ A Prometheus metrics exporter for The Open Network (TON) Node.
 TON Node does not provide any built-in metrics for monitoring, so this exporter was created to fill that gap.
 
 ## Prerequisites
+
 - Go 1.22 or later
 - Docker (optional)
 - Access to a TON Lite Server (Node) with a valid key.
 
 ## Installation
+
 ### From source
+
 To build and install the exporter from source, follow these steps:
+
 ```bash
 git clone https://github.com/Maxitosh/ton-node-exporter.git
 cd ton-node-exporter
 go build -o ton-node-exporter ./cmd/ton-node-exporter
 ```
 
 ### Docker
+
 To pull the Docker image, use the following command:
+
 ```bash
 docker pull ghcr.io/maxitosh/ton-node-exporter:latest
 ```
 
 ## Usage
+
 ### Configuration
+
 The exporter is configured using environment variables or a `.env` file. The following variables are available:
 
 - `EXPORTER_PORT`: The port on which the exporter will listen (default: `9100`).
 - `LITE_SERVER_ADDR`: The address of the TON Lite Server (Node) to monitor.
 - `LITE_SERVER_KEY`: The key to access the TON Lite Server.
 - `GLOBAL_CONFIG_URL`: The URL of the TON global lite server config.
 
-Check the .env.template file for an example and relevant environment variables. To create a .env file, copy the template:
+Check the .env.template file for an example and relevant environment variables. To create a .env file, copy the
+template:
+
 ```bash
 cp .env.template .env
 ```
 
 ## Running
+
 ### From source
+
 Assuming you set up the environment variables in the `.env` file, you can run the exporter with the following command:
+
 ```bash
 ./ton-node-exporter
 ```
+
 If you want to use environment variables instead of a `.env` file, you can run the exporter like this:
+
 ```bash
 export EXPORTER_PORT=9100
 export LITE_SERVER_ADDR=""
@@ -53,31 +68,42 @@ export GLOBAL_CONFIG_URL=""
 ```
 
 ### Docker
+
 To run the exporter using Docker:
+
 ```bash
 docker run -d --name ton-node-exporter --env-file .env -p 9100:9100 ghcr.io/maxitosh/ton-node-exporter:latest
 ```
 
 ## Metrics
+
 The exporter exposes the following metrics:
 
-| Metric name                        | Metric type | Description                                                                 | Labels/tags | Status |  
-|------------------------------------|-------------|-----------------------------------------------------------------------------|-------------|--------|  
-| ton_node_master_chain_block_number | Gauge       | The current master chain block number.                                      | env         | ✅      |
-| ton_node_head_lag                  | Gauge       | The lag between the current master chain block on the node and the network. | env         | ✅      |
+| Metric name                        | Metric type | Description                                                                    | Labels/tags | Status |  
+|------------------------------------|-------------|--------------------------------------------------------------------------------|-------------|--------|  
+| ton_node_master_chain_block_number | Gauge       | The current master chain block number.                                         | env         | ✅      |
+| ton_node_head_lag                  | Gauge       | The lag between the current master chain block on the node and the network.    |             | ✅      |
+| ton_node_indexing_latency          | Gauge       | Time lag in seconds between the last Elector transaction and the current time. |             | ✅      |
+| ton_node_last_elector_tx_time      | Gauge       | Last Elector transaction time in seconds (Unix time).                          |             | ✅      |
 
 ## Testing
+
 To run tests, execute the following command:
+
 ```bash
 go test -v ./...
 ```
 
 ## Examples
+
 To access the metrics, do curl request to the exporter:
+
 ```bash
 curl http://localhost:9100/metrics
 ```
+
 Response:
+
 ```plaintext
 ...
 # HELP ton_node_head_lag Head block lag
@@ -90,7 +116,9 @@ ton_node_master_chain_block_number{env="local"} 3.9033746e+07
 ```
 
 ## Contributing
+
 Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
 
 ## Author
+
 Created by [Maxitosh (Max Kureikin)](https://github.com/Maxitosh).

cmd/ton-node-exporter/main.go

@@ -44,7 +44,10 @@ func main() {
 		fetcher.NewADNLFetcher(localAPI),
 		fetcher.NewADNLFetcher(globalAPI),
 	)
-	prometheus.MustRegister(tonBlockNumberCollector)
+	tonIndexingLatencyCollector := collector.NewTonIndexingLatencyCollector(
+		fetcher.NewADNLFetcher(localAPI),
+	)
+	prometheus.MustRegister(tonBlockNumberCollector, tonIndexingLatencyCollector)
 
 	// Expose the registered metrics via HTTP.
 	http.Handle("/metrics", promhttp.Handler())

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/prometheus/client_golang v1.19.1
 	github.com/stretchr/testify v1.9.0
-	github.com/xssnick/tonutils-go v1.9.8
+	github.com/xssnick/tonutils-go v1.10.2
 	go.uber.org/mock v0.4.0
 )
 

go.sum

@@ -37,8 +37,8 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/xssnick/tonutils-go v1.9.8 h1:Sq382w8H63sjy5y+j13b9mytHPLf7H94LW+OmxZ4h/c=
-github.com/xssnick/tonutils-go v1.9.8/go.mod h1:p1l1Bxdv9sz6x2jfbuGQUGJn6g5cqg7xsTp8rBHFoJY=
+github.com/xssnick/tonutils-go v1.10.2 h1:1wgnQPrzbOt+5PtuNrlMSUyh1/y0pvWRi0zeRNRLEbw=
+github.com/xssnick/tonutils-go v1.10.2/go.mod h1:p1l1Bxdv9sz6x2jfbuGQUGJn6g5cqg7xsTp8rBHFoJY=
 go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
 go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=

internal/collector/indexing_latency.go

@@ -0,0 +1,58 @@
+package collector
+
+import (
+	"time"
+	"ton-node-exporter/internal/fetcher"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const ElectorAddress = "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"
+
+var (
+	tonLastElectorTxTimeDesc = prometheus.NewDesc(
+		"ton_node_last_elector_tx_time",
+		"Last Elector transaction time",
+		[]string{}, nil,
+	)
+	tonIndexingLatencyDesc = prometheus.NewDesc(
+		"ton_node_indexing_latency",
+		"Time lag between the last Elector transaction and the current time",
+		nil, nil,
+	)
+)
+
+var TimeNow = time.Now
+
+type TonIndexingLatencyCollector struct {
+	fetcher fetcher.Fetcher
+}
+
+func NewTonIndexingLatencyCollector(
+	fetcher fetcher.Fetcher,
+) *TonIndexingLatencyCollector {
+	return &TonIndexingLatencyCollector{
+		fetcher: fetcher,
+	}
+}
+
+func (collector *TonIndexingLatencyCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- tonLastElectorTxTimeDesc
+	ch <- tonIndexingLatencyDesc
+}
+
+func (collector *TonIndexingLatencyCollector) Collect(ch chan<- prometheus.Metric) {
+	lastElectorTxTime, err := collector.fetcher.FetchAddressLastTransactionTime(ElectorAddress)
+	if err == nil {
+		ch <- prometheus.MustNewConstMetric(
+			tonLastElectorTxTimeDesc,
+			prometheus.GaugeValue,
+			float64(lastElectorTxTime),
+		)
+		ch <- prometheus.MustNewConstMetric(
+			tonIndexingLatencyDesc,
+			prometheus.GaugeValue,
+			float64(TimeNow().Unix()-int64(lastElectorTxTime)),
+		)
+	}
+}

internal/collector/indexing_latency_test.go

@@ -0,0 +1,95 @@
+package collector
+
+import (
+	"strings"
+	"testing"
+	"time"
+	mock_fetcher "ton-node-exporter/internal/fetcher/mocks"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/mock/gomock"
+)
+
+func TestTonIndexingLatencyCollector_Describe(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	fetcher := mock_fetcher.NewMockFetcher(ctrl)
+	collector := NewTonIndexingLatencyCollector(fetcher)
+
+	ch := make(chan *prometheus.Desc, 2)
+	collector.Describe(ch)
+	close(ch)
+
+	var descriptions []*prometheus.Desc
+	for desc := range ch {
+		descriptions = append(descriptions, desc)
+	}
+
+	assert.Contains(t, descriptions, tonLastElectorTxTimeDesc)
+	assert.Contains(t, descriptions, tonIndexingLatencyDesc)
+}
+
+func TestTonIndexingLatencyCollector_Collect(t *testing.T) {
+	tests := []struct {
+		name                string
+		setupMocks          func(fetcher *mock_fetcher.MockFetcher)
+		expectedMetrics     string
+		expectedMetricCount int
+	}{
+		{
+			name: "Successful fetch",
+			setupMocks: func(fetcher *mock_fetcher.MockFetcher) {
+				fetcher.EXPECT().FetchAddressLastTransactionTime(ElectorAddress).Return(uint32(1727767355), nil).AnyTimes()
+			},
+			expectedMetrics: `
+# HELP ton_node_indexing_latency Time lag between the last Elector transaction and the current time
+# TYPE ton_node_indexing_latency gauge
+ton_node_indexing_latency 5
+# HELP ton_node_last_elector_tx_time Last Elector transaction time
+# TYPE ton_node_last_elector_tx_time gauge
+ton_node_last_elector_tx_time 1.727767355e+09
+`,
+			expectedMetricCount: 2,
+		},
+		{
+			name: "Error in Indexing latency fetcher",
+			setupMocks: func(fetcher *mock_fetcher.MockFetcher) {
+				fetcher.EXPECT().FetchAddressLastTransactionTime(ElectorAddress).Return(uint32(0), assert.AnError).AnyTimes()
+			},
+			expectedMetrics:     ``,
+			expectedMetricCount: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+			// 1727767355
+			fixedTime := time.Date(2024, 10, 1, 7, 22, 40, 0, time.UTC)
+			TimeNow = func() time.Time {
+				return fixedTime
+			}
+			defer func() { TimeNow = time.Now }() // Reset after the test.
+
+			fetcher := mock_fetcher.NewMockFetcher(ctrl)
+			tt.setupMocks(fetcher)
+
+			collector := NewTonIndexingLatencyCollector(fetcher)
+			reg := prometheus.NewPedanticRegistry()
+			reg.MustRegister(collector)
+
+			metrics, err := reg.Gather()
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expectedMetricCount, len(metrics))
+
+			if tt.expectedMetricCount > 0 {
+				err := testutil.GatherAndCompare(reg, strings.NewReader(tt.expectedMetrics))
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

internal/fetcher/adnl_fetcher.go

@@ -4,11 +4,21 @@ import (
 	"context"
 	"time"
 
+	"github.com/xssnick/tonutils-go/address"
+	"github.com/xssnick/tonutils-go/tlb"
 	"github.com/xssnick/tonutils-go/ton"
 )
 
 type TonAPIClient interface {
 	GetMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error)
+	GetAccount(ctx context.Context, block *ton.BlockIDExt, addr *address.Address) (*tlb.Account, error)
+	ListTransactions(
+		ctx context.Context,
+		addr *address.Address,
+		num uint32,
+		lt uint64,
+		txHash []byte,
+	) ([]*tlb.Transaction, error)
 }
 
 type ADNLFetcher struct {
@@ -32,3 +42,30 @@ func (fetcher *ADNLFetcher) FetchMasterChainBlockNumber() (float64, error) {
 
 	return float64(masterChainInfo.SeqNo), nil
 }
+
+func (fetcher *ADNLFetcher) FetchAddressLastTransactionTime(addr string) (uint32, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// we need fresh master block info to run get methods
+	masterChainInfo, err := fetcher.client.GetMasterchainInfo(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	// fetch account info to get last tx hash and lt
+	addrParsed := address.MustParseAddr(addr)
+	account, err := fetcher.client.GetAccount(ctx, masterChainInfo, addrParsed)
+	if err != nil {
+		return 0, err
+	}
+
+	// fetch last transaction
+	txs, err := fetcher.client.ListTransactions(ctx, address.MustParseAddr(addr), 1, account.LastTxLT, account.LastTxHash)
+	if err != nil {
+		return 0, err
+	}
+
+	electorTx := txs[0]
+	return electorTx.Now, nil
+}