diff --git a/.github/workflows/portal-loop-txs-exporter.yml b/.github/workflows/portal-loop-txs-exporter.yml new file mode 100644 index 00000000..bae296fe --- /dev/null +++ b/.github/workflows/portal-loop-txs-exporter.yml @@ -0,0 +1,46 @@ +name: Backup Portal Loop + +on: + # allow to run workflow manually + workflow_dispatch: { } + + # Triggers the workflow every hour + schedule: + - cron: "0 * * * *" + +jobs: + backup: + name: "backup ${{ matrix.testnet }}" + runs-on: ubuntu-latest + timeout-minutes: 360 # very high; but it can take a while. + + permissions: + contents: write + + strategy: + fail-fast: false + max-parallel: 1 + matrix: + testnet: + - portal-loop + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.22.x" + + - name: Run backup script + run: | + cd portal-loop + bash export.sh + + - name: Run stats script + run: make -C ${{ matrix.testnet }} stats-legacy + + - name: Run extractor + run: make -C ${{ matrix.testnet }} extractor-legacy + + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "chore: update ${{ matrix.testnet }} backup" diff --git a/extractor-0.1.1/main.go b/extractor-0.1.1/main.go index cf46a6ed..70ce3740 100644 --- a/extractor-0.1.1/main.go +++ b/extractor-0.1.1/main.go @@ -16,6 +16,7 @@ import ( "github.com/gnolang/gno/gno.land/pkg/sdk/vm" "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/gnolang/gno/tm2/pkg/std" "github.com/gnolang/tx-archive/types" "github.com/peterbourgon/ff/v3/ffcli" ) @@ -37,6 +38,8 @@ type extractorCfg struct { fileType string sourcePath string outputDir string + + legacyMode bool } func main() { @@ -88,6 +91,13 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) { "./extracted", "the output directory for the extracted Gno source code", ) + + fs.BoolVar( + &c.legacyMode, + "legacy-mode", + false, + "flag indicating if the legacy tx sheet mode should be used", + ) } // execExtract runs the extract service for Gno source code @@ -129,40 +139,83 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error { return errNoSourceFilesFound } - for _, sourceFile := range sourceFiles { - sourceFile := sourceFile + var ( + unwrapFn = func(data types.TxData) []std.Msg { + return data.Tx.Msgs + } - // Extract messages - msgs, processErr := extractAddMessages(sourceFile) - if processErr != nil { - return processErr + heightFn = func(data types.TxData) uint64 { + return data.BlockNum } - // Process messages - for _, msg := range msgs { - outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + unwrapLegacyFn = func(tx std.Tx) []std.Msg { + return tx.Msgs + } - if st, err := os.Stat(outputDir); err == nil && st.IsDir() { - outputDir += ":" + strconv.FormatUint(msg.Height, 10) - } + heightLegacyFn = func(_ std.Tx) uint64 { + return 0 + } + ) - // Write dir before writing files - if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { - return fmt.Errorf("unable to write dir, %w", dirWriteErr) + for _, sourceFile := range sourceFiles { + select { + case <-ctx.Done(): + return ctx.Err() + default: + sourceFile := sourceFile + + // Extract messages + var ( + msgs []AddPackage + processErr error + ) + + if !cfg.legacyMode { + msgs, processErr = extractAddMessages( + sourceFile, + unwrapFn, + heightFn, + ) + } else { + msgs, processErr = extractAddMessages( + sourceFile, + unwrapLegacyFn, + heightLegacyFn, + ) } - // Write the package source code - if writeErr := writePackageFiles(msg, outputDir); writeErr != nil { - return writeErr + if processErr != nil { + return processErr } - // Write the package metadata - if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil { - return writeErr + // Process messages + for _, msg := range msgs { + outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/")) + + if !cfg.legacyMode { + if st, err := os.Stat(outputDir); err == nil && st.IsDir() { + outputDir += ":" + strconv.FormatUint(msg.Height, 10) + } + } + + // Write dir before writing files + if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil { + return fmt.Errorf("unable to write dir, %w", dirWriteErr) + } + + // Write the package source code + if writeErr := writePackageFiles(msg, outputDir); writeErr != nil { + return writeErr + } + + // Write the package metadata + if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil { + return writeErr + } } } - } + return nil } @@ -204,7 +257,12 @@ type AddPackage struct { Height uint64 } -func extractAddMessages(filePath string) ([]AddPackage, error) { +// extractAddMessages extracts the AddPackage messages +func extractAddMessages[T std.Tx | types.TxData]( + filePath string, + unwrapFn func(T) []std.Msg, + heightFn func(T) uint64, +) ([]AddPackage, error) { file, err := os.Open(filePath) if err != nil { return nil, fmt.Errorf("unable to open file, %w", err) @@ -226,7 +284,8 @@ func extractAddMessages(filePath string) ([]AddPackage, error) { tempBuf := make([]byte, 0) for { - var txData types.TxData + var txData T + line, isPrefix, err := reader.ReadLine() // Exit if no more lines in file @@ -262,7 +321,7 @@ func extractAddMessages(filePath string) ([]AddPackage, error) { tempBuf = nil } - for _, msg := range txData.Tx.Msgs { + for _, msg := range unwrapFn(txData) { // Only MsgAddPkg should be parsed if msg.Type() != "add_package" { continue @@ -279,7 +338,7 @@ func extractAddMessages(filePath string) ([]AddPackage, error) { msgArr = append(msgArr, AddPackage{ MsgAddPackage: msgAddPkg, - Height: txData.BlockNum, + Height: heightFn(txData), }) } } diff --git a/portal-loop/Makefile b/portal-loop/Makefile new file mode 100644 index 00000000..22a8e14c --- /dev/null +++ b/portal-loop/Makefile @@ -0,0 +1,6 @@ +EXTRACTOR_DIR = extractor-0.1.1 +REMOTE = "https://rpc.gno.land" +SHORTNAME = portal-loop +LOOP_DURATION = 50000 + +-include ../rules.mk diff --git a/portal-loop/README.md b/portal-loop/README.md new file mode 100644 index 00000000..72b8148c --- /dev/null +++ b/portal-loop/README.md @@ -0,0 +1,129 @@ +# https://rpc.gno.land + +## TXs +``` + 5542 +``` + +## addpkgs +``` +``` + +## top realm calls +``` +1920 "gno.land/r/portal/counter" + 220 "gno.land/r/gnoland/blog" + 108 "gno.land/r/demo/teritori/social_feeds" + 106 "gno.land/r/demo/memeland" + 68 "gno.land/r/demo/postit/v1" + 60 "gno.land/r/demo/users" + 48 "gno.land/r/demo/userbook" + 37 "gno.land/r/gnome/dao/v1dev1" + 36 "gno.land/r/demo/wugnot" + 32 "gno.land/r/demo/staging/memeland/v1" + 30 "gno.land/r/demo/boards" + 24 "gno.land/r/demo/event_emitter" + 24 "gno.land/r/gnome/dao/v1dev0" + 23 "gno.land/r/gnoland/events" + 23 "gno.land/r/gnome/dao/pre1" + 19 "gno.land/r/gnome/dao/v1pre2" + 16 "gno.land/r/gnome/dao/v1pre3" + 15 "gno.land/r/gc24/raffle" + 15 "gno.land/r/gnome/dao/v1pre1" + 15 "gno.land/r/jeronimoalbi/testpoll" + 14 "gno.land/r/g1lyzcpa7duh69lk04nahxup484xrz4k6k2nqdun/gnome/dao/v1" + 13 "gno.land/r/demo/staging/memeland/v2" + 11 "gno.land/r/mikecito/gh_test_4" + 11 "gno.land/r/varmeta/vmt721" + 10 "gno.land/r/gnome/dao/v1dev01" + 10 "gno.land/r/test321/hello" + 9 "gno.land/r/albttx/home" + 8 "gno.land/r/demo/postit" + 8 "gno.land/r/gnome/dao/v1pre4" + 8 "gno.land/r/leon/test/avltestv1" + 7 "gno.land/r/leon/v2/raffle" + 7 "gno.land/r/malek/todolist1" + 6 "gno.land/r/gnome/dao/pre2" + 6 "gno.land/r/leon/staging/v3/raffle" + 6 "gno.land/r/mikecito/gh_test_3" + 6 "gno.land/r/test2/hello" + 5 "gno.land/r/demo/foo20" + 5 "gno.land/r/demo/restest" + 5 "gno.land/r/demo/test_event" + 5 "gno.land/r/leon/test/test1/whitelistfactory" + 5 "gno.land/r/leon/tokens/leongrc20" + 5 "gno.land/r/mikecito/gh_test_2" + 5 "gno.land/r/teritori/social_feeds" + 4 "gno.land/r/michelle37/testpoll" + 4 "gno.land/r/reza/rezablog" + 4 "gno.land/r/test/counter" + 3 "gno.land/r/boom/boom" + 3 "gno.land/r/demo/banktest" + 3 "gno.land/r/demo/hello001" + 3 "gno.land/r/demo/mikecito/social_feeds" + 3 "gno.land/r/demo/nft2" + 3 "gno.land/r/gnoland/faucet" + 3 "gno.land/r/leon/demo/poll/v2" + 3 "gno.land/r/leon/grc/v1/leon200" + 3 "gno.land/r/leon/staging/v4/raffle" + 3 "gno.land/r/leon/test/avltest" + 3 "gno.land/r/leon/testing/counter3" + 3 "gno.land/r/leon/v5/memeland" + 3 "gno.land/r/mikecito/gh_test_1" + 2 "gno.land/r/demo/event" + 2 "gno.land/r/demo/hi001" + 2 "gno.land/r/demo/morgan/variadic" + 2 "gno.land/r/gnome/tutorials/pre1" + 2 "gno.land/r/hehe321/hehe321" + 2 "gno.land/r/hehe321/hehe3211" + 2 "gno.land/r/hehe321/hehe3212" + 2 "gno.land/r/jeronimoalbi2/testpoll" + 2 "gno.land/r/leon/counter/e/hello" + 2 "gno.land/r/leon/demo/poll" + 2 "gno.land/r/leon/testing/counter2" + 2 "gno.land/r/leon/v1/raffle" + 2 "gno.land/r/mikaelvallenet/mvc" + 2 "gno.land/r/test3/hello" + 2 "gno.land/r/test4/hello" + 1 "gno.land/r/deelawn/tsrf" + 1 "gno.land/r/demo/abcitest123" + 1 "gno.land/r/demo/art/gnoface" + 1 "gno.land/r/demo/counter" + 1 "gno.land/r/demo/emit_main" + 1 "gno.land/r/demo/faucet" + 1 "gno.land/r/demo/grc20factory" + 1 "gno.land/r/demo/hello" + 1 "gno.land/r/demo/microblog" + 1 "gno.land/r/demo/tamagotchi" + 1 "gno.land/r/demo/test_event2" + 1 "gno.land/r/demoyy/counter" + 1 "gno.land/r/foo/event" + 1 "gno.land/r/gnoland/ghverify" + 1 "gno.land/r/gophercon/mood" + 1 "gno.land/r/hehe3212/hehe3212" + 1 "gno.land/r/hello/event" + 1 "gno.land/r/hello/event_emitter" + 1 "gno.land/r/iamkevin/kraffle5" + 1 "gno.land/r/leon/demo/poll/v1" + 1 "gno.land/r/leon/grc/v1/leon20" + 1 "gno.land/r/leon/test/realm/hello" + 1 "gno.land/r/leon/test/realms" + 1 "gno.land/r/leon/test/realms2" + 1 "gno.land/r/leon/test/todolist" + 1 "gno.land/r/leon/v1/memeland" + 1 "gno.land/r/manfred/home" + 1 "gno.land/r/mikaelvallenet/path" + 1 "gno.land/r/morgan/guestbook" + 1 "gno.land/r/nebular24/guessbook" + 1 "gno.land/r/stuyk/mood" + 1 "gno.land/r/test/hello" + 1 "gno.land/r/test10/poll" + 1 "gno.land/r/test5/hello" + 1 "gno.land/r/varmeta/demo/v1/domain/registrar" + 1 "gno.land/r/whisky/goraffle" +``` + +## top faucet requesters +``` +``` + diff --git a/portal-loop/export.sh b/portal-loop/export.sh new file mode 100644 index 00000000..e820c88c --- /dev/null +++ b/portal-loop/export.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +set -xe + +# Define the constants +TMP_DIR=temp-gno # Temporary directory for the working data +GENESIS=genesis.json +WGET_OUTPUT=wget-genesis.json +BACKUP_NAME_TXS=backup_portal_loop_txs.jsonl +BACKUP_NAME_BALANCES=backup_portal_loop_balances.jsonl +# Latest backup is commited in the repo +LATEST_BACKUP_FILE_TXS=../"$BACKUP_NAME_TXS" + +# Make the generated backup files the reference ones stored into the repository +copyBackupFiles () { + cp "$BACKUP_NAME_TXS" "$LATEST_BACKUP_FILE_TXS" + cp "$BACKUP_NAME_BALANCES" ../"$BACKUP_NAME_BALANCES" +} + +# Create the temporary working dir +mkdir $TMP_DIR +cd $TMP_DIR || exit 1 + +# Grab the latest genesis.json +wget -O $WGET_OUTPUT https://rpc.gno.land/genesis + +# Extract the wget genesis response +jq ".result.genesis" $WGET_OUTPUT > $GENESIS + +# Install the gnoland binary +git clone https://github.com/gnolang/gno.git +cd gno/gno.land || exit 1 +make build.gnoland +cd ../.. # move back to the portal-loop directory + +# Extract the genesis transactions +./gno/gno.land/build/gnoland genesis txs export -genesis-path $GENESIS "$BACKUP_NAME_TXS" +# Extract the genesis balances +./gno/gno.land/build/gnoland genesis balances export -genesis-path $GENESIS "$BACKUP_NAME_BALANCES" + +# Clean up the downloaded genesis.json and the wget response +rm $GENESIS $WGET_OUTPUT + +# Check if there is an existing backup +if [[ ! -f "$LATEST_BACKUP_FILE_TXS" ]]; then + # Save the initial backup + echo "Saving initial backup to $BACKUP_NAME_TXS" + + # Make the backup files official + copyBackupFiles +else # Backup file exists! + LATEST_BACKUP_SORTED=latest_backup_sort.jsonl + DIFF_TXS=diff.jsonl + + # There is an existing backup already, check it + echo "Backup file already present in the repository" + + # Sort the latest backup file + sort "$LATEST_BACKUP_FILE_TXS" > "$LATEST_BACKUP_SORTED" + + # Sort the latest genesis tx sheet + sort "$BACKUP_NAME_TXS" > temp_"$BACKUP_NAME_TXS" + + # Compare existing and incoming txs backup files both sorted + # Use comm to find lines only in the incoming txs backup and write to an output file + comm -13 ./"$LATEST_BACKUP_SORTED" temp_"$BACKUP_NAME_TXS" > "$DIFF_TXS" + + # Notify if differences were found + if [[ -z $(grep '[^[:space:]]' "$DIFF_TXS") ]]; then + echo "No differences found. Exiting with no further activities." + else + echo "Differences found. Replacing backup files" + # Make the backup files official + copyBackupFiles + echo "Stored new backup files for balances and txs" + fi +fi + +cd .. || exit 1 + +# Clean up the temporary directory +rm -rf $TMP_DIR diff --git a/rules.mk b/rules.mk index 05bf8816..a3e87819 100644 --- a/rules.mk +++ b/rules.mk @@ -46,12 +46,47 @@ stats: echo '```' >> README.md echo >> README.md +stats-legacy: + echo "# $(REMOTE)" > README.md + echo >> README.md + + echo "## TXs" >> README.md + echo '```' >> README.md + cat backup_*.jsonl | wc -l >> README.md + echo '```' >> README.md + echo >> README.md + + echo "## addpkgs" >> README.md + echo '```' >> README.md + cat backup_*.jsonl | jq '.msg[].package.Path | select( . != null )' | sort | uniq -c | sort --stable -nr >> README.md + echo '```' >> README.md + echo >> README.md + + echo "## top realm calls" >> README.md + echo '```' >> README.md + cat backup_*.jsonl | jq '.msg[].pkg_path | select( . != null )' | sort | uniq -c | sort --stable -nr >> README.md + echo '```' >> README.md + echo >> README.md + + echo "## top faucet requesters" >> README.md + echo '```' >> README.md + cat backup_*.jsonl | jq -r '.msg[] | select(.["@type"]=="/bank.MsgSend") | select(.["from_address"]=="g127jydsh6cms3lrtdenydxsckh23a8d6emqcvfa") | .to_address + " " + .amount' | sed 's/ugnot$$//' | awk 'NR == 1 {next} {a[$$1] += $$2} {b[$$1] += 1} END {for (i in a) {if (a[i] >= 500000000){printf "%-15s\t%s\t%s\n", i, b[i], a[i]}}}' | sort -rnk2 >> README.md + echo '```' >> README.md + echo >> README.md + extractor: go run -C "../$(EXTRACTOR_DIR)" . \ -source-path "$(shell pwd)" \ -output-dir "$(shell pwd)/extracted" +extractor-legacy: + go run -C "../$(EXTRACTOR_DIR)" . \ + -source-path "$(shell pwd)" \ + -output-dir "$(shell pwd)/extracted" \ + --legacy-mode + + loop: while true; do \ ( \