ci(portal loop): Create exports from portal loop (#21)

.github/workflows/portal-loop-txs-exporter.yml

@@ -0,0 +1,46 @@
+name: Backup Portal Loop
+
+on:
+  # allow to run workflow manually
+  workflow_dispatch: { }
+
+  # Triggers the workflow every hour
+  schedule:
+    - cron: "0 * * * *"
+
+jobs:
+  backup:
+    name: "backup ${{ matrix.testnet }}"
+    runs-on: ubuntu-latest
+    timeout-minutes: 360 # very high; but it can take a while.
+
+    permissions:
+      contents: write
+
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        testnet:
+          - portal-loop
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22.x"
+
+      - name: Run backup script
+        run: |
+          cd portal-loop
+          bash export.sh
+
+      - name: Run stats script
+        run: make -C ${{ matrix.testnet }} stats-legacy
+
+      - name: Run extractor
+        run: make -C ${{ matrix.testnet }} extractor-legacy
+
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore: update ${{ matrix.testnet }} backup"

extractor-0.1.1/main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/gnolang/gno/gno.land/pkg/sdk/vm"
 	"github.com/gnolang/gno/tm2/pkg/amino"
+	"github.com/gnolang/gno/tm2/pkg/std"
 	"github.com/gnolang/tx-archive/types"
 	"github.com/peterbourgon/ff/v3/ffcli"
 )
@@ -37,6 +38,8 @@ type extractorCfg struct {
 	fileType   string
 	sourcePath string
 	outputDir  string
+
+	legacyMode bool
 }
 
 func main() {
@@ -88,6 +91,13 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) {
 		"./extracted",
 		"the output directory for the extracted Gno source code",
 	)
+
+	fs.BoolVar(
+		&c.legacyMode,
+		"legacy-mode",
+		false,
+		"flag indicating if the legacy tx sheet mode should be used",
+	)
 }
 
 // execExtract runs the extract service for Gno source code
@@ -129,40 +139,83 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error {
 		return errNoSourceFilesFound
 	}
 
-	for _, sourceFile := range sourceFiles {
-		sourceFile := sourceFile
+	var (
+		unwrapFn = func(data types.TxData) []std.Msg {
+			return data.Tx.Msgs
+		}
 
-		// Extract messages
-		msgs, processErr := extractAddMessages(sourceFile)
-		if processErr != nil {
-			return processErr
+		heightFn = func(data types.TxData) uint64 {
+			return data.BlockNum
 		}
 
-		// Process messages
-		for _, msg := range msgs {
-			outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/"))
+		unwrapLegacyFn = func(tx std.Tx) []std.Msg {
+			return tx.Msgs
+		}
 
-			if st, err := os.Stat(outputDir); err == nil && st.IsDir() {
-				outputDir += ":" + strconv.FormatUint(msg.Height, 10)
-			}
+		heightLegacyFn = func(_ std.Tx) uint64 {
+			return 0
+		}
+	)
 
-			// Write dir before writing files
-			if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil {
-				return fmt.Errorf("unable to write dir, %w", dirWriteErr)
+	for _, sourceFile := range sourceFiles {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			sourceFile := sourceFile
+
+			// Extract messages
+			var (
+				msgs       []AddPackage
+				processErr error
+			)
+
+			if !cfg.legacyMode {
+				msgs, processErr = extractAddMessages(
+					sourceFile,
+					unwrapFn,
+					heightFn,
+				)
+			} else {
+				msgs, processErr = extractAddMessages(
+					sourceFile,
+					unwrapLegacyFn,
+					heightLegacyFn,
+				)
 			}
 
-			// Write the package source code
-			if writeErr := writePackageFiles(msg, outputDir); writeErr != nil {
-				return writeErr
+			if processErr != nil {
+				return processErr
 			}
 
-			// Write the package metadata
-			if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil {
-				return writeErr
+			// Process messages
+			for _, msg := range msgs {
+				outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/"))
+
+				if !cfg.legacyMode {
+					if st, err := os.Stat(outputDir); err == nil && st.IsDir() {
+						outputDir += ":" + strconv.FormatUint(msg.Height, 10)
+					}
+				}
+
+				// Write dir before writing files
+				if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil {
+					return fmt.Errorf("unable to write dir, %w", dirWriteErr)
+				}
+
+				// Write the package source code
+				if writeErr := writePackageFiles(msg, outputDir); writeErr != nil {
+					return writeErr
+				}
+
+				// Write the package metadata
+				if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil {
+					return writeErr
+				}
 			}
 		}
-
 	}
+
 	return nil
 }
 
@@ -204,7 +257,12 @@ type AddPackage struct {
 	Height uint64
 }
 
-func extractAddMessages(filePath string) ([]AddPackage, error) {
+// extractAddMessages extracts the AddPackage messages
+func extractAddMessages[T std.Tx | types.TxData](
+	filePath string,
+	unwrapFn func(T) []std.Msg,
+	heightFn func(T) uint64,
+) ([]AddPackage, error) {
 	file, err := os.Open(filePath)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open file, %w", err)
@@ -226,7 +284,8 @@ func extractAddMessages(filePath string) ([]AddPackage, error) {
 	tempBuf := make([]byte, 0)
 
 	for {
-		var txData types.TxData
+		var txData T
+
 		line, isPrefix, err := reader.ReadLine()
 
 		// Exit if no more lines in file
@@ -262,7 +321,7 @@ func extractAddMessages(filePath string) ([]AddPackage, error) {
 			tempBuf = nil
 		}
 
-		for _, msg := range txData.Tx.Msgs {
+		for _, msg := range unwrapFn(txData) {
 			// Only MsgAddPkg should be parsed
 			if msg.Type() != "add_package" {
 				continue
@@ -279,7 +338,7 @@ func extractAddMessages(filePath string) ([]AddPackage, error) {
 
 			msgArr = append(msgArr, AddPackage{
 				MsgAddPackage: msgAddPkg,
-				Height:        txData.BlockNum,
+				Height:        heightFn(txData),
 			})
 		}
 	}

portal-loop/Makefile

@@ -0,0 +1,6 @@
+EXTRACTOR_DIR = extractor-0.1.1
+REMOTE = "https://rpc.gno.land"
+SHORTNAME = portal-loop
+LOOP_DURATION = 50000
+
+-include ../rules.mk

portal-loop/README.md

@@ -0,0 +1,129 @@
+# https://rpc.gno.land
+
+## TXs
+```
+    5542
+```
+
+## addpkgs
+```
+```
+
+## top realm calls
+```
+1920 "gno.land/r/portal/counter"
+ 220 "gno.land/r/gnoland/blog"
+ 108 "gno.land/r/demo/teritori/social_feeds"
+ 106 "gno.land/r/demo/memeland"
+  68 "gno.land/r/demo/postit/v1"
+  60 "gno.land/r/demo/users"
+  48 "gno.land/r/demo/userbook"
+  37 "gno.land/r/gnome/dao/v1dev1"
+  36 "gno.land/r/demo/wugnot"
+  32 "gno.land/r/demo/staging/memeland/v1"
+  30 "gno.land/r/demo/boards"
+  24 "gno.land/r/demo/event_emitter"
+  24 "gno.land/r/gnome/dao/v1dev0"
+  23 "gno.land/r/gnoland/events"
+  23 "gno.land/r/gnome/dao/pre1"
+  19 "gno.land/r/gnome/dao/v1pre2"
+  16 "gno.land/r/gnome/dao/v1pre3"
+  15 "gno.land/r/gc24/raffle"
+  15 "gno.land/r/gnome/dao/v1pre1"
+  15 "gno.land/r/jeronimoalbi/testpoll"
+  14 "gno.land/r/g1lyzcpa7duh69lk04nahxup484xrz4k6k2nqdun/gnome/dao/v1"
+  13 "gno.land/r/demo/staging/memeland/v2"
+  11 "gno.land/r/mikecito/gh_test_4"
+  11 "gno.land/r/varmeta/vmt721"
+  10 "gno.land/r/gnome/dao/v1dev01"
+  10 "gno.land/r/test321/hello"
+   9 "gno.land/r/albttx/home"
+   8 "gno.land/r/demo/postit"
+   8 "gno.land/r/gnome/dao/v1pre4"
+   8 "gno.land/r/leon/test/avltestv1"
+   7 "gno.land/r/leon/v2/raffle"
+   7 "gno.land/r/malek/todolist1"
+   6 "gno.land/r/gnome/dao/pre2"
+   6 "gno.land/r/leon/staging/v3/raffle"
+   6 "gno.land/r/mikecito/gh_test_3"
+   6 "gno.land/r/test2/hello"
+   5 "gno.land/r/demo/foo20"
+   5 "gno.land/r/demo/restest"
+   5 "gno.land/r/demo/test_event"
+   5 "gno.land/r/leon/test/test1/whitelistfactory"
+   5 "gno.land/r/leon/tokens/leongrc20"
+   5 "gno.land/r/mikecito/gh_test_2"
+   5 "gno.land/r/teritori/social_feeds"
+   4 "gno.land/r/michelle37/testpoll"
+   4 "gno.land/r/reza/rezablog"
+   4 "gno.land/r/test/counter"
+   3 "gno.land/r/boom/boom"
+   3 "gno.land/r/demo/banktest"
+   3 "gno.land/r/demo/hello001"
+   3 "gno.land/r/demo/mikecito/social_feeds"
+   3 "gno.land/r/demo/nft2"
+   3 "gno.land/r/gnoland/faucet"
+   3 "gno.land/r/leon/demo/poll/v2"
+   3 "gno.land/r/leon/grc/v1/leon200"
+   3 "gno.land/r/leon/staging/v4/raffle"
+   3 "gno.land/r/leon/test/avltest"
+   3 "gno.land/r/leon/testing/counter3"
+   3 "gno.land/r/leon/v5/memeland"
+   3 "gno.land/r/mikecito/gh_test_1"
+   2 "gno.land/r/demo/event"
+   2 "gno.land/r/demo/hi001"
+   2 "gno.land/r/demo/morgan/variadic"
+   2 "gno.land/r/gnome/tutorials/pre1"
+   2 "gno.land/r/hehe321/hehe321"
+   2 "gno.land/r/hehe321/hehe3211"
+   2 "gno.land/r/hehe321/hehe3212"
+   2 "gno.land/r/jeronimoalbi2/testpoll"
+   2 "gno.land/r/leon/counter/e/hello"
+   2 "gno.land/r/leon/demo/poll"
+   2 "gno.land/r/leon/testing/counter2"
+   2 "gno.land/r/leon/v1/raffle"
+   2 "gno.land/r/mikaelvallenet/mvc"
+   2 "gno.land/r/test3/hello"
+   2 "gno.land/r/test4/hello"
+   1 "gno.land/r/deelawn/tsrf"
+   1 "gno.land/r/demo/abcitest123"
+   1 "gno.land/r/demo/art/gnoface"
+   1 "gno.land/r/demo/counter"
+   1 "gno.land/r/demo/emit_main"
+   1 "gno.land/r/demo/faucet"
+   1 "gno.land/r/demo/grc20factory"
+   1 "gno.land/r/demo/hello"
+   1 "gno.land/r/demo/microblog"
+   1 "gno.land/r/demo/tamagotchi"
+   1 "gno.land/r/demo/test_event2"
+   1 "gno.land/r/demoyy/counter"
+   1 "gno.land/r/foo/event"
+   1 "gno.land/r/gnoland/ghverify"
+   1 "gno.land/r/gophercon/mood"
+   1 "gno.land/r/hehe3212/hehe3212"
+   1 "gno.land/r/hello/event"
+   1 "gno.land/r/hello/event_emitter"
+   1 "gno.land/r/iamkevin/kraffle5"
+   1 "gno.land/r/leon/demo/poll/v1"
+   1 "gno.land/r/leon/grc/v1/leon20"
+   1 "gno.land/r/leon/test/realm/hello"
+   1 "gno.land/r/leon/test/realms"
+   1 "gno.land/r/leon/test/realms2"
+   1 "gno.land/r/leon/test/todolist"
+   1 "gno.land/r/leon/v1/memeland"
+   1 "gno.land/r/manfred/home"
+   1 "gno.land/r/mikaelvallenet/path"
+   1 "gno.land/r/morgan/guestbook"
+   1 "gno.land/r/nebular24/guessbook"
+   1 "gno.land/r/stuyk/mood"
+   1 "gno.land/r/test/hello"
+   1 "gno.land/r/test10/poll"
+   1 "gno.land/r/test5/hello"
+   1 "gno.land/r/varmeta/demo/v1/domain/registrar"
+   1 "gno.land/r/whisky/goraffle"
+```
+
+## top faucet requesters
+```
+```
+