From ec90b938b258171ff4059ea427d4dbda54fa39b5 Mon Sep 17 00:00:00 2001 From: giulio Date: Tue, 2 Jul 2024 10:20:26 +0200 Subject: [PATCH 01/11] Implement lsig size pooling --- cmd/goal/clerk.go | 13 +++++++++++-- config/consensus.go | 12 +++++++++--- data/transactions/verify/txn.go | 15 ++++++++++++++- protocol/tags.go | 2 +- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index a69ed5be98..cd6ea16c51 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -986,7 +986,7 @@ func assembleFileImpl(fname string, printWarnings bool) *logic.OpStream { reportErrorf(tealAppSize, fname, len(ops.Program), config.MaxAvailableAppProgramLen) } } else { - if uint64(len(ops.Program)) > params.LogicSigMaxSize { + if !params.EnableLogicSigSizePooling && uint64(len(ops.Program)) > params.LogicSigMaxSize { reportErrorf(tealLogicSigSize, fname, len(ops.Program), params.LogicSigMaxSize) } } @@ -1179,14 +1179,19 @@ var dryrunCmd = &cobra.Command{ if timeStamp <= 0 { timeStamp = time.Now().Unix() } + + lSigPooledSize := 0 for i, txn := range stxns { if txn.Lsig.Blank() { continue } - if uint64(txn.Lsig.Len()) > params.LogicSigMaxSize { + lsigLen := txn.Lsig.Len() + lSigPooledSize += lsigLen + if !params.EnableLogicSigSizePooling && uint64(lsigLen) > params.LogicSigMaxSize { reportErrorf("program size too large: %d > %d", len(txn.Lsig.Logic), params.LogicSigMaxSize) } ep := logic.NewSigEvalParams(stxns, ¶ms, logic.NoHeaderLedger{}) + err := logic.CheckSignature(i, ep) if err != nil { reportErrorf("program failed Check: %s", err) @@ -1204,6 +1209,10 @@ var dryrunCmd = &cobra.Command{ fmt.Fprintf(os.Stdout, "ERROR: %s\n", err.Error()) } } + lSigMaxPooledSize := len(stxns) * int(params.LogicSigMaxSize) + if params.EnableLogicSigSizePooling && lSigPooledSize > lSigMaxPooledSize { + reportErrorf("total lsigs size too large: %d > %d", lSigPooledSize, lSigMaxPooledSize) + } }, } diff --git a/config/consensus.go b/config/consensus.go index f86e45e831..8286d7ed46 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -113,9 +113,13 @@ type ConsensusParams struct { EnableAppCostPooling bool // EnableLogicSigCostPooling specifies LogicSig budgets are pooled across a - // group. The total available is len(group) * LogicSigMaxCost) + // group. The total available is len(group) * LogicSigMaxCost EnableLogicSigCostPooling bool + // EnableLogicSigSizePooling specifies LogicSig sizes are pooled across a + // group. The total available is len(group) * LogicSigMaxSize + EnableLogicSigSizePooling bool + // RewardUnit specifies the number of MicroAlgos corresponding to one reward // unit. // @@ -228,7 +232,7 @@ type ConsensusParams struct { // 0 for no support, otherwise highest version supported LogicSigVersion uint64 - // len(LogicSig.Logic) + len(LogicSig.Args[*]) must be less than this + // len(LogicSig.Logic) + len(LogicSig.Args[*]) must be less than this (unless pooling is enabled) LogicSigMaxSize uint64 // sum of estimated op cost must be less than this @@ -765,7 +769,7 @@ func checkSetAllocBounds(p ConsensusParams) { checkSetMax(p.MaxAppProgramLen, &MaxStateDeltaKeys) checkSetMax(p.MaxAppProgramLen, &MaxEvalDeltaAccounts) checkSetMax(p.MaxAppProgramLen, &MaxAppProgramLen) - checkSetMax(int(p.LogicSigMaxSize), &MaxLogicSigMaxSize) + checkSetMax((int(p.LogicSigMaxSize) * p.MaxTxGroupSize), &MaxLogicSigMaxSize) checkSetMax(p.MaxTxnNoteBytes, &MaxTxnNoteBytes) checkSetMax(p.MaxTxGroupSize, &MaxTxGroupSize) // MaxBytesKeyValueLen is max of MaxAppKeyLen and MaxAppBytesValueLen @@ -1512,6 +1516,8 @@ func initConsensusProtocols() { vFuture.LogicSigVersion = 11 // When moving this to a release, put a new higher LogicSigVersion here + vFuture.EnableLogicSigSizePooling = true + vFuture.Payouts.Enabled = true vFuture.Payouts.Percent = 75 vFuture.Payouts.GoOnlineFee = 2_000_000 // 2 algos diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a345c679af..4fdc848105 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -214,6 +214,7 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl minFeeCount := uint64(0) feesPaid := uint64(0) + lSigPooledSize := 0 for i, stxn := range stxs { prepErr := txnBatchPrep(i, groupCtx, verifier) if prepErr != nil { @@ -225,6 +226,18 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl minFeeCount++ } feesPaid = basics.AddSaturate(feesPaid, stxn.Txn.Fee.Raw) + lSigPooledSize += stxn.Lsig.Len() + } + if groupCtx.consensusParams.EnableLogicSigSizePooling { + lSigMaxPooledSize := len(stxs) * int(groupCtx.consensusParams.LogicSigMaxSize) + if lSigPooledSize > lSigMaxPooledSize { + errorMsg := fmt.Sprintf( + "txgroup had %d bytes of LogicSigs, more than the available pool of %d bytes", + lSigPooledSize, lSigMaxPooledSize, + ) + err = &TxGroupError{err: errors.New(errorMsg), GroupIndex: -1, Reason: TxGroupErrorReasonLogicSigFailed} + return nil, err + } } feeNeeded, overflow := basics.OMul(groupCtx.consensusParams.MinTxnFee, minFeeCount) if overflow { @@ -361,7 +374,7 @@ func logicSigSanityCheckBatchPrep(gi int, groupCtx *GroupContext, batchVerifier if version > groupCtx.consensusParams.LogicSigVersion { return errors.New("LogicSig.Logic version too new") } - if uint64(lsig.Len()) > groupCtx.consensusParams.LogicSigMaxSize { + if !groupCtx.consensusParams.EnableLogicSigSizePooling && uint64(lsig.Len()) > groupCtx.consensusParams.LogicSigMaxSize { return errors.New("LogicSig.Logic too long") } diff --git a/protocol/tags.go b/protocol/tags.go index 6cfcacd714..561ca491ab 100644 --- a/protocol/tags.go +++ b/protocol/tags.go @@ -87,7 +87,7 @@ const TopicMsgRespTagMaxSize = 6 * 1024 * 1024 // TxnTagMaxSize is the maximum size of a TxnTag message. This is equal to SignedTxnMaxSize() // which is size of just a single message containing maximum Stateproof. Since Stateproof // transactions can't be batched we don't need to multiply by MaxTxnBatchSize. -const TxnTagMaxSize = 4620031 +const TxnTagMaxSize = 8460031 // UniEnsBlockReqTagMaxSize is the maximum size of a UniEnsBlockReqTag message const UniEnsBlockReqTagMaxSize = 67 From bc74e17f61bcb097e2b383acbfb0aedc71d69d49 Mon Sep 17 00:00:00 2001 From: giulio Date: Sun, 21 Jul 2024 09:45:36 +0200 Subject: [PATCH 02/11] Implement requested changes --- cmd/goal/clerk.go | 2 +- data/transactions/verify/txn.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index cd6ea16c51..a4b78d4de0 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -986,7 +986,7 @@ func assembleFileImpl(fname string, printWarnings bool) *logic.OpStream { reportErrorf(tealAppSize, fname, len(ops.Program), config.MaxAvailableAppProgramLen) } } else { - if !params.EnableLogicSigSizePooling && uint64(len(ops.Program)) > params.LogicSigMaxSize { + if len(ops.Program) > config.MaxLogicSigMaxSize { reportErrorf(tealLogicSigSize, fname, len(ops.Program), params.LogicSigMaxSize) } } diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 4fdc848105..eb08ddec5e 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -93,7 +93,7 @@ type TxGroupErrorReason int const ( // TxGroupErrorReasonGeneric is a generic (not tracked) reason code TxGroupErrorReasonGeneric TxGroupErrorReason = iota - // TxGroupErrorReasonNotWellFormed is txn.WellFormed failure + // TxGroupErrorReasonNotWellFormed is txn.WellFormed failure or malformed logic signature TxGroupErrorReasonNotWellFormed // TxGroupErrorReasonInvalidFee is invalid fee pooling in transaction group TxGroupErrorReasonInvalidFee @@ -235,7 +235,7 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl "txgroup had %d bytes of LogicSigs, more than the available pool of %d bytes", lSigPooledSize, lSigMaxPooledSize, ) - err = &TxGroupError{err: errors.New(errorMsg), GroupIndex: -1, Reason: TxGroupErrorReasonLogicSigFailed} + err = &TxGroupError{err: errors.New(errorMsg), GroupIndex: -1, Reason: TxGroupErrorReasonNotWellFormed} return nil, err } } @@ -375,7 +375,7 @@ func logicSigSanityCheckBatchPrep(gi int, groupCtx *GroupContext, batchVerifier return errors.New("LogicSig.Logic version too new") } if !groupCtx.consensusParams.EnableLogicSigSizePooling && uint64(lsig.Len()) > groupCtx.consensusParams.LogicSigMaxSize { - return errors.New("LogicSig.Logic too long") + return errors.New("LogicSig too long") } err := logic.CheckSignature(gi, groupCtx.evalParams) From e6bcb26b9e9b1b558569eba481dbafc19bade78a Mon Sep 17 00:00:00 2001 From: giulio Date: Mon, 29 Jul 2024 13:53:45 +0200 Subject: [PATCH 03/11] Implement new max size for LogicSigMaxSize --- data/transactions/logicsig.go | 2 +- data/transactions/msgp_gen.go | 2 +- protocol/tags.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/transactions/logicsig.go b/data/transactions/logicsig.go index 31a14ea1dc..8874f11dbc 100644 --- a/data/transactions/logicsig.go +++ b/data/transactions/logicsig.go @@ -39,7 +39,7 @@ type LogicSig struct { Msig crypto.MultisigSig `codec:"msig"` // Args are not signed, but checked by Logic - Args [][]byte `codec:"arg,allocbound=EvalMaxArgs,allocbound=config.MaxLogicSigMaxSize"` + Args [][]byte `codec:"arg,allocbound=EvalMaxArgs,allocbound=config.MaxLogicSigMaxSize,maxtotalbytes=config.MaxLogicSigMaxSize"` } // Blank returns true if there is no content in this LogicSig diff --git a/data/transactions/msgp_gen.go b/data/transactions/msgp_gen.go index 7cc22db08a..f25d315a8e 100644 --- a/data/transactions/msgp_gen.go +++ b/data/transactions/msgp_gen.go @@ -3444,7 +3444,7 @@ func (z *LogicSig) MsgIsZero() bool { func LogicSigMaxSize() (s int) { s = 1 + 2 + msgp.BytesPrefixSize + config.MaxLogicSigMaxSize + 4 + crypto.SignatureMaxSize() + 5 + crypto.MultisigSigMaxSize() + 4 // Calculating size of slice: z.Args - s += msgp.ArrayHeaderSize + ((EvalMaxArgs) * (msgp.BytesPrefixSize + config.MaxLogicSigMaxSize)) + s += msgp.ArrayHeaderSize + config.MaxLogicSigMaxSize return } diff --git a/protocol/tags.go b/protocol/tags.go index 561ca491ab..1f545a6a70 100644 --- a/protocol/tags.go +++ b/protocol/tags.go @@ -87,7 +87,7 @@ const TopicMsgRespTagMaxSize = 6 * 1024 * 1024 // TxnTagMaxSize is the maximum size of a TxnTag message. This is equal to SignedTxnMaxSize() // which is size of just a single message containing maximum Stateproof. Since Stateproof // transactions can't be batched we don't need to multiply by MaxTxnBatchSize. -const TxnTagMaxSize = 8460031 +const TxnTagMaxSize = 4394756 // UniEnsBlockReqTagMaxSize is the maximum size of a UniEnsBlockReqTag message const UniEnsBlockReqTagMaxSize = 67 From 16793dc0a63953aa6cfc564ba7be6b40e7c4700b Mon Sep 17 00:00:00 2001 From: giulio Date: Wed, 28 Aug 2024 21:43:02 +0200 Subject: [PATCH 04/11] Make goal more permissive when compiling logicsigs --- cmd/goal/clerk.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index a4b78d4de0..42907927d1 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -980,16 +980,6 @@ func assembleFileImpl(fname string, printWarnings bool) *logic.OpStream { ops.ReportMultipleErrors(fname, os.Stderr) reportErrorf("%s: %s", fname, err) } - _, params := getProto(protoVersion) - if ops.HasStatefulOps { - if len(ops.Program) > config.MaxAvailableAppProgramLen { - reportErrorf(tealAppSize, fname, len(ops.Program), config.MaxAvailableAppProgramLen) - } - } else { - if len(ops.Program) > config.MaxLogicSigMaxSize { - reportErrorf(tealLogicSigSize, fname, len(ops.Program), params.LogicSigMaxSize) - } - } if printWarnings && len(ops.Warnings) != 0 { for _, warning := range ops.Warnings { From 16ef8991fae1ead79c293c0c6e0a24caafd484eb Mon Sep 17 00:00:00 2001 From: giulio Date: Wed, 28 Aug 2024 21:45:24 +0200 Subject: [PATCH 05/11] Implement max size for logicsig args of 4096 bytes --- data/transactions/logic/eval.go | 12 ++++++++++-- data/transactions/logicsig.go | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/data/transactions/logic/eval.go b/data/transactions/logic/eval.go index 3201877ed9..c5e73e8d35 100644 --- a/data/transactions/logic/eval.go +++ b/data/transactions/logic/eval.go @@ -1000,6 +1000,7 @@ func (pe panicError) Error() string { var errLogicSigNotSupported = errors.New("LogicSig not supported") var errTooManyArgs = errors.New("LogicSig has too many arguments") +var errLogicSigArgTooLarge = errors.New("LogicSig argument too large") // EvalError indicates AVM evaluation failure type EvalError struct { @@ -1305,8 +1306,15 @@ func eval(program []byte, cx *EvalContext) (pass bool, err error) { if (cx.EvalParams.Proto == nil) || (cx.EvalParams.Proto.LogicSigVersion == 0) { return false, errLogicSigNotSupported } - if cx.txn.Lsig.Args != nil && len(cx.txn.Lsig.Args) > transactions.EvalMaxArgs { - return false, errTooManyArgs + if cx.txn.Lsig.Args != nil { + if len(cx.txn.Lsig.Args) > transactions.EvalMaxArgs { + return false, errTooManyArgs + } + for _, arg := range cx.txn.Lsig.Args { + if len(arg) > transactions.MaxLogicSigArgSize { + return false, errLogicSigArgTooLarge + } + } } if verr != nil { return false, verr diff --git a/data/transactions/logicsig.go b/data/transactions/logicsig.go index 8874f11dbc..dbaf8c7799 100644 --- a/data/transactions/logicsig.go +++ b/data/transactions/logicsig.go @@ -25,6 +25,11 @@ import ( // EvalMaxArgs is the maximum number of arguments to an LSig const EvalMaxArgs = 255 +// MaxLsigArgSize is the maximum size of an argument to an LSig +// We use 4096 to match the maximum size of a TEAL value +// (as defined in `const maxStringSize` in package logic) +const MaxLogicSigArgSize = 4096 + // LogicSig contains logic for validating a transaction. // LogicSig is signed by an account, allowing delegation of operations. // OR @@ -39,7 +44,7 @@ type LogicSig struct { Msig crypto.MultisigSig `codec:"msig"` // Args are not signed, but checked by Logic - Args [][]byte `codec:"arg,allocbound=EvalMaxArgs,allocbound=config.MaxLogicSigMaxSize,maxtotalbytes=config.MaxLogicSigMaxSize"` + Args [][]byte `codec:"arg,allocbound=EvalMaxArgs,allocbound=MaxLogicSigArgSize,maxtotalbytes=config.MaxLogicSigMaxSize"` } // Blank returns true if there is no content in this LogicSig From 91d6e09d3d39805d7e5d637e76ab059cae525eff Mon Sep 17 00:00:00 2001 From: giulio Date: Wed, 28 Aug 2024 21:46:07 +0200 Subject: [PATCH 06/11] Make TxnTagMaxSize same as max block size --- data/transactions/msgp_gen.go | 8 ++++---- node/node_test.go | 3 +-- protocol/tags.go | 8 ++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/data/transactions/msgp_gen.go b/data/transactions/msgp_gen.go index f25d315a8e..15cd34ef2d 100644 --- a/data/transactions/msgp_gen.go +++ b/data/transactions/msgp_gen.go @@ -3306,8 +3306,8 @@ func (z *LogicSig) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o err = msgp.WrapError(err, "struct-from-array", "Args", zb0001) return } - if zb0007 > config.MaxLogicSigMaxSize { - err = msgp.ErrOverflow(uint64(zb0007), uint64(config.MaxLogicSigMaxSize)) + if zb0007 > MaxLogicSigArgSize { + err = msgp.ErrOverflow(uint64(zb0007), uint64(MaxLogicSigArgSize)) return } (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) @@ -3395,8 +3395,8 @@ func (z *LogicSig) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o err = msgp.WrapError(err, "Args", zb0001) return } - if zb0011 > config.MaxLogicSigMaxSize { - err = msgp.ErrOverflow(uint64(zb0011), uint64(config.MaxLogicSigMaxSize)) + if zb0011 > MaxLogicSigArgSize { + err = msgp.ErrOverflow(uint64(zb0011), uint64(MaxLogicSigArgSize)) return } (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) diff --git a/node/node_test.go b/node/node_test.go index 6b991751cb..b608eaa148 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -36,7 +36,6 @@ import ( "github.com/algorand/go-algorand/data/account" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/network/p2p" @@ -805,7 +804,7 @@ func TestMaxSizesCorrect(t *testing.T) { require.Equal(t, ppSize, protocol.ProposalPayloadTag.MaxMessageSize()) spSize := uint64(stateproof.SigFromAddrMaxSize()) require.Equal(t, spSize, protocol.StateProofSigTag.MaxMessageSize()) - txSize := uint64(transactions.SignedTxnMaxSize()) + txSize := uint64(config.MaxTxnBytesPerBlock) require.Equal(t, txSize, protocol.TxnTag.MaxMessageSize()) msSize := uint64(crypto.DigestMaxSize()) require.Equal(t, msSize, protocol.MsgDigestSkipTag.MaxMessageSize()) diff --git a/protocol/tags.go b/protocol/tags.go index 1f545a6a70..7403f0aae8 100644 --- a/protocol/tags.go +++ b/protocol/tags.go @@ -84,10 +84,10 @@ const StateProofSigTagMaxSize = 6378 // Matches current network.MaxMessageLength const TopicMsgRespTagMaxSize = 6 * 1024 * 1024 -// TxnTagMaxSize is the maximum size of a TxnTag message. This is equal to SignedTxnMaxSize() -// which is size of just a single message containing maximum Stateproof. Since Stateproof -// transactions can't be batched we don't need to multiply by MaxTxnBatchSize. -const TxnTagMaxSize = 4394756 +// TxnTagMaxSize is the maximum size of a TxnTag message. +// Matches current config.MaxTxnBytesPerBlock, the maximum lenght of a block, +// since a transaction or transaction group cannot be larger than a block. +const TxnTagMaxSize = 5 * 1024 * 1024 // UniEnsBlockReqTagMaxSize is the maximum size of a UniEnsBlockReqTag message const UniEnsBlockReqTagMaxSize = 67 From 5a1eeaf1beb4799aca60de4b69612072acf9e198 Mon Sep 17 00:00:00 2001 From: giulio Date: Mon, 9 Sep 2024 22:53:27 +0200 Subject: [PATCH 07/11] Revert "Make TxnTagMaxSize same as max block size" This reverts commit 91d6e09d3d39805d7e5d637e76ab059cae525eff which is not needed anymore after the merge of: AVM: Derive looser, but more principled, checks of txn max size #6114 --- data/transactions/msgp_gen.go | 8 ++++---- node/node_test.go | 3 ++- protocol/tags.go | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/data/transactions/msgp_gen.go b/data/transactions/msgp_gen.go index 15cd34ef2d..f25d315a8e 100644 --- a/data/transactions/msgp_gen.go +++ b/data/transactions/msgp_gen.go @@ -3306,8 +3306,8 @@ func (z *LogicSig) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o err = msgp.WrapError(err, "struct-from-array", "Args", zb0001) return } - if zb0007 > MaxLogicSigArgSize { - err = msgp.ErrOverflow(uint64(zb0007), uint64(MaxLogicSigArgSize)) + if zb0007 > config.MaxLogicSigMaxSize { + err = msgp.ErrOverflow(uint64(zb0007), uint64(config.MaxLogicSigMaxSize)) return } (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) @@ -3395,8 +3395,8 @@ func (z *LogicSig) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o err = msgp.WrapError(err, "Args", zb0001) return } - if zb0011 > MaxLogicSigArgSize { - err = msgp.ErrOverflow(uint64(zb0011), uint64(MaxLogicSigArgSize)) + if zb0011 > config.MaxLogicSigMaxSize { + err = msgp.ErrOverflow(uint64(zb0011), uint64(config.MaxLogicSigMaxSize)) return } (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) diff --git a/node/node_test.go b/node/node_test.go index b608eaa148..6b991751cb 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -36,6 +36,7 @@ import ( "github.com/algorand/go-algorand/data/account" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/network/p2p" @@ -804,7 +805,7 @@ func TestMaxSizesCorrect(t *testing.T) { require.Equal(t, ppSize, protocol.ProposalPayloadTag.MaxMessageSize()) spSize := uint64(stateproof.SigFromAddrMaxSize()) require.Equal(t, spSize, protocol.StateProofSigTag.MaxMessageSize()) - txSize := uint64(config.MaxTxnBytesPerBlock) + txSize := uint64(transactions.SignedTxnMaxSize()) require.Equal(t, txSize, protocol.TxnTag.MaxMessageSize()) msSize := uint64(crypto.DigestMaxSize()) require.Equal(t, msSize, protocol.MsgDigestSkipTag.MaxMessageSize()) diff --git a/protocol/tags.go b/protocol/tags.go index 7403f0aae8..1f545a6a70 100644 --- a/protocol/tags.go +++ b/protocol/tags.go @@ -84,10 +84,10 @@ const StateProofSigTagMaxSize = 6378 // Matches current network.MaxMessageLength const TopicMsgRespTagMaxSize = 6 * 1024 * 1024 -// TxnTagMaxSize is the maximum size of a TxnTag message. -// Matches current config.MaxTxnBytesPerBlock, the maximum lenght of a block, -// since a transaction or transaction group cannot be larger than a block. -const TxnTagMaxSize = 5 * 1024 * 1024 +// TxnTagMaxSize is the maximum size of a TxnTag message. This is equal to SignedTxnMaxSize() +// which is size of just a single message containing maximum Stateproof. Since Stateproof +// transactions can't be batched we don't need to multiply by MaxTxnBatchSize. +const TxnTagMaxSize = 4394756 // UniEnsBlockReqTagMaxSize is the maximum size of a UniEnsBlockReqTag message const UniEnsBlockReqTagMaxSize = 67 From 39600e5c3987bdebc408d3bc310ae729660f53f1 Mon Sep 17 00:00:00 2001 From: giulio Date: Mon, 9 Sep 2024 23:15:04 +0200 Subject: [PATCH 08/11] Update TestMaxSizesCorrect with logicsig pooling --- data/transactions/msgp_gen.go | 8 ++++---- node/node_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/data/transactions/msgp_gen.go b/data/transactions/msgp_gen.go index f25d315a8e..15cd34ef2d 100644 --- a/data/transactions/msgp_gen.go +++ b/data/transactions/msgp_gen.go @@ -3306,8 +3306,8 @@ func (z *LogicSig) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o err = msgp.WrapError(err, "struct-from-array", "Args", zb0001) return } - if zb0007 > config.MaxLogicSigMaxSize { - err = msgp.ErrOverflow(uint64(zb0007), uint64(config.MaxLogicSigMaxSize)) + if zb0007 > MaxLogicSigArgSize { + err = msgp.ErrOverflow(uint64(zb0007), uint64(MaxLogicSigArgSize)) return } (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) @@ -3395,8 +3395,8 @@ func (z *LogicSig) UnmarshalMsgWithState(bts []byte, st msgp.UnmarshalState) (o err = msgp.WrapError(err, "Args", zb0001) return } - if zb0011 > config.MaxLogicSigMaxSize { - err = msgp.ErrOverflow(uint64(zb0011), uint64(config.MaxLogicSigMaxSize)) + if zb0011 > MaxLogicSigArgSize { + err = msgp.ErrOverflow(uint64(zb0011), uint64(MaxLogicSigArgSize)) return } (*z).Args[zb0001], bts, err = msgp.ReadBytesBytes(bts, (*z).Args[zb0001]) diff --git a/node/node_test.go b/node/node_test.go index 19463177df..550bea3049 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -824,10 +824,10 @@ func TestMaxSizesCorrect(t *testing.T) { maxCombinedTxnSize := uint64(transactions.SignedTxnMaxSize()) // subtract out the two smaller signature sizes (logicsig is biggest, it can *contain* the others) maxCombinedTxnSize -= uint64(crypto.SignatureMaxSize() + crypto.MultisigSigMaxSize()) - // the logicsig size is *also* an overestimate, because it thinks each - // logicsig arg can be big, but really the sum of the args and the program - // has a max size. - maxCombinedTxnSize -= uint64(transactions.EvalMaxArgs * config.MaxLogicSigMaxSize) + // the logicsig size is *also* an overestimate, because it thinks that the logicsig and + // the logicsig args can both be up to to MaxLogicSigMaxSize, but that's the max for + // them combined, so it double counts and we have to subtract one. + maxCombinedTxnSize -= uint64(config.MaxLogicSigMaxSize) // maxCombinedTxnSize is still an overestimate because it assumes all txn // type fields can be in the same txn. That's not true, but it provides an From 4de5dd0d3470ac252cc35343438c7ddb21c805ea Mon Sep 17 00:00:00 2001 From: giulio Date: Tue, 10 Sep 2024 11:26:09 +0200 Subject: [PATCH 09/11] Don't test app/lsig max size in goal clerk compile We now let goal clerk compile app program and lsig of any sizes, even it they will be rejected by the nodes. --- .../cli/goal/expect/tealConsensusTest.exp | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/test/e2e-go/cli/goal/expect/tealConsensusTest.exp b/test/e2e-go/cli/goal/expect/tealConsensusTest.exp index a4231acd96..a6abb862d4 100644 --- a/test/e2e-go/cli/goal/expect/tealConsensusTest.exp +++ b/test/e2e-go/cli/goal/expect/tealConsensusTest.exp @@ -54,14 +54,17 @@ if { [catch { "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } } - teal "$TEST_ROOT_DIR/big-sig.teal" 2 1001 1 - spawn goal clerk compile "$TEST_ROOT_DIR/big-sig.teal" - expect { - -re {[A-Z2-9]{58}} { ::AlgorandGoal::Abort "hash" } - -re {.*logicsig program size too large} { puts "bigsigcheck: pass" } - eof { ::AlgorandGoal::Abort $expect_out(buffer) } - "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } - } + # we do not test anymore for a max lsig size because we let goal compile any program size, + # even if they will be rejected by the nodes + + #teal "$TEST_ROOT_DIR/big-sig.teal" 2 16001 1 + #spawn goal clerk compile "$TEST_ROOT_DIR/big-sig.teal" + #expect { + # -re {[A-Z2-9]{58}} { ::AlgorandGoal::Abort "hash" } + # -re {.*logicsig program size too large} { puts "bigsigcheck: pass" } + # eof { ::AlgorandGoal::Abort $expect_out(buffer) } + # "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } + #} teal "$TEST_ROOT_DIR/barely-fits-app.teal" 2 4090 1 "int 0\nbalance\npop\n" spawn goal clerk compile "$TEST_ROOT_DIR/barely-fits-app.teal" @@ -71,15 +74,18 @@ if { [catch { "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } } + # we do not test anymore for a max app program size because we let goal compile any program size, + # even if they will be rejected by the nodes + # MaxAppProgramLen = 2K * (1 + 3 pages max) - teal "$TEST_ROOT_DIR/big-app.teal" 2 8192 1 "int 0\nbalance\npop\n" - spawn goal clerk compile "$TEST_ROOT_DIR/big-app.teal" - expect { - -re {[A-Z2-9]{58}} { ::AlgorandGoal::Abort "hash" } - -re {.*app program size too large} { puts "bigappcheck: pass" } - eof { ::AlgorandGoal::Abort $expect_out(buffer) } - "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } - } + #teal "$TEST_ROOT_DIR/big-app.teal" 2 8192 1 "int 0\nbalance\npop\n" + #spawn goal clerk compile "$TEST_ROOT_DIR/big-app.teal" + #expect { + # -re {[A-Z2-9]{58}} { ::AlgorandGoal::Abort "hash" } + # -re {.*app program size too large} { puts "bigappcheck: pass" } + # eof { ::AlgorandGoal::Abort $expect_out(buffer) } + # "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } + #} # Test cost limits during dryrun exec goal clerk send -F "$TEST_ROOT_DIR/small-sig.teal" -t GXBNLU4AXQABPLHXJDMTG2YXSDT4EWUZACT7KTPFXDQW52XPTIUS5OZ5HQ -a 100 -d $TEST_PRIMARY_NODE_DIR -o $TEST_ROOT_DIR/small-sig.tx From 795386251f4943894f9118cf0bc169c1501536b7 Mon Sep 17 00:00:00 2001 From: giulio Date: Sat, 14 Sep 2024 09:35:19 +0200 Subject: [PATCH 10/11] Add integration test for lsig size pooling --- data/transactions/verify/txn.go | 5 +- .../cli/goal/expect/tealConsensusTest.exp | 25 --- .../features/transactions/logicsig_test.go | 167 ++++++++++++++++++ .../nettemplates/TwoNodes50EachV18.json | 36 ++++ 4 files changed, 205 insertions(+), 28 deletions(-) create mode 100644 test/e2e-go/features/transactions/logicsig_test.go create mode 100644 test/testdata/nettemplates/TwoNodes50EachV18.json diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index eb08ddec5e..24f7cc7793 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -231,12 +231,11 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl if groupCtx.consensusParams.EnableLogicSigSizePooling { lSigMaxPooledSize := len(stxs) * int(groupCtx.consensusParams.LogicSigMaxSize) if lSigPooledSize > lSigMaxPooledSize { - errorMsg := fmt.Sprintf( + errorMsg := fmt.Errorf( "txgroup had %d bytes of LogicSigs, more than the available pool of %d bytes", lSigPooledSize, lSigMaxPooledSize, ) - err = &TxGroupError{err: errors.New(errorMsg), GroupIndex: -1, Reason: TxGroupErrorReasonNotWellFormed} - return nil, err + return nil, &TxGroupError{err: errorMsg, GroupIndex: -1, Reason: TxGroupErrorReasonNotWellFormed} } } feeNeeded, overflow := basics.OMul(groupCtx.consensusParams.MinTxnFee, minFeeCount) diff --git a/test/e2e-go/cli/goal/expect/tealConsensusTest.exp b/test/e2e-go/cli/goal/expect/tealConsensusTest.exp index a6abb862d4..9df3d4abe9 100644 --- a/test/e2e-go/cli/goal/expect/tealConsensusTest.exp +++ b/test/e2e-go/cli/goal/expect/tealConsensusTest.exp @@ -54,18 +54,6 @@ if { [catch { "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } } - # we do not test anymore for a max lsig size because we let goal compile any program size, - # even if they will be rejected by the nodes - - #teal "$TEST_ROOT_DIR/big-sig.teal" 2 16001 1 - #spawn goal clerk compile "$TEST_ROOT_DIR/big-sig.teal" - #expect { - # -re {[A-Z2-9]{58}} { ::AlgorandGoal::Abort "hash" } - # -re {.*logicsig program size too large} { puts "bigsigcheck: pass" } - # eof { ::AlgorandGoal::Abort $expect_out(buffer) } - # "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } - #} - teal "$TEST_ROOT_DIR/barely-fits-app.teal" 2 4090 1 "int 0\nbalance\npop\n" spawn goal clerk compile "$TEST_ROOT_DIR/barely-fits-app.teal" expect { @@ -74,19 +62,6 @@ if { [catch { "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } } - # we do not test anymore for a max app program size because we let goal compile any program size, - # even if they will be rejected by the nodes - - # MaxAppProgramLen = 2K * (1 + 3 pages max) - #teal "$TEST_ROOT_DIR/big-app.teal" 2 8192 1 "int 0\nbalance\npop\n" - #spawn goal clerk compile "$TEST_ROOT_DIR/big-app.teal" - #expect { - # -re {[A-Z2-9]{58}} { ::AlgorandGoal::Abort "hash" } - # -re {.*app program size too large} { puts "bigappcheck: pass" } - # eof { ::AlgorandGoal::Abort $expect_out(buffer) } - # "\n" { ::AlgorandGoal::Abort $expect_out(buffer) } - #} - # Test cost limits during dryrun exec goal clerk send -F "$TEST_ROOT_DIR/small-sig.teal" -t GXBNLU4AXQABPLHXJDMTG2YXSDT4EWUZACT7KTPFXDQW52XPTIUS5OZ5HQ -a 100 -d $TEST_PRIMARY_NODE_DIR -o $TEST_ROOT_DIR/small-sig.tx spawn goal clerk dryrun -t $TEST_ROOT_DIR/small-sig.tx diff --git a/test/e2e-go/features/transactions/logicsig_test.go b/test/e2e-go/features/transactions/logicsig_test.go new file mode 100644 index 0000000000..087fed198d --- /dev/null +++ b/test/e2e-go/features/transactions/logicsig_test.go @@ -0,0 +1,167 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package transactions + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/data/transactions/logic" + "github.com/algorand/go-algorand/test/framework/fixtures" + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/require" +) + +// CreateTealOfSize return a TEAL bytecode of `size` bytes which always succeeds. +// `size` must be at least 9 bytes +func CreateTealOfSize(size uint, pragma uint) ([]byte, error) { + if size < 9 { + return nil, fmt.Errorf("size must be at least 9 bytes; got %d", size) + } + ls := fmt.Sprintf("#pragma version %d\n", pragma) + if size%2 == 0 { + ls += "int 10\npop\nint 1\npop\n" + } else { + ls += "int 1\npop\nint 1\npop\n" + } + for i := uint(11); i <= size; i += 2 { + ls = ls + "int 1\npop\n" + } + ls = ls + "int 1" + code, err := logic.AssembleString(ls) + if err != nil { + return nil, err + } + // panic if the function is not working as expected and needs to be updated + if len(code.Program) != int(size) { + panic(fmt.Sprintf("wanted to create a program of size %d but got a program of size %d", + size, len(code.Program))) + } + return code.Program, nil +} + +func TestLogicSigSizeBeforePooling(t *testing.T) { + partitiontest.PartitionTest(t) + defer fixtures.ShutdownSynchronizedTest(t) + a := require.New(fixtures.SynchronizedTest(t)) + + // From consensus version 18, we have lsigs with a maximum size of 1000 bytes. + // We need to use pragma 1 for teal in v18 + pragma := uint(1) + tealOK, err := CreateTealOfSize(1000, pragma) + a.NoError(err) + tealTooLong, err := CreateTealOfSize(1001, pragma) + a.NoError(err) + + testLogicSize(t, tealOK, tealTooLong, filepath.Join("nettemplates", "TwoNodes50EachV18.json")) +} + +func TestLogicSigSizeAfterPooling(t *testing.T) { + partitiontest.PartitionTest(t) + defer fixtures.ShutdownSynchronizedTest(t) + a := require.New(fixtures.SynchronizedTest(t)) + + pragma := uint(1) + tealOK, err := CreateTealOfSize(2000, pragma) + a.NoError(err) + tealTooLong, err := CreateTealOfSize(2001, pragma) + a.NoError(err) + + // TODO: Update this when lsig pooling graduates from vFuture + testLogicSize(t, tealOK, tealTooLong, filepath.Join("nettemplates", "TwoNodes50EachFuture.json")) +} + +// testLogicSize takes two TEAL programs, one expected to be ok and one expected to be too long +// and thus to be rejected, and tests that's indeed the case. +func testLogicSize(t *testing.T, tealOK, tealTooLong []byte, + networkTemplate string) { + + t.Parallel() + a := require.New(fixtures.SynchronizedTest(t)) + + var fixture fixtures.RestClientFixture + fixture.Setup(t, networkTemplate) + defer fixture.Shutdown() + + client := fixture.LibGoalClient + accountList, err := fixture.GetWalletsSortedByBalance() + a.NoError(err) + baseAcct := accountList[0].Address + + walletHandler, err := client.GetUnencryptedWalletHandle() + a.NoError(err) + + signatureOK, err := client.SignProgramWithWallet(walletHandler, nil, baseAcct, tealOK) + a.NoError(err) + lsigOK := transactions.LogicSig{Logic: tealOK, Sig: signatureOK} + + signatureTooLong, err := client.SignProgramWithWallet(walletHandler, nil, baseAcct, tealTooLong) + a.NoError(err) + lsigTooLong := transactions.LogicSig{Logic: tealTooLong, Sig: signatureTooLong} + + // We build two transaction groups of two transactions each. + // The first txn will be either signed by an ok lsig or a too long one. + // The second is a vanilla payment transaction to complete the group. + var txn1Success, txn2Success, txn1Fail, txn2Fail transactions.Transaction + for i, txn := range []*transactions.Transaction{&txn1Success, &txn2Success, &txn1Fail, &txn2Fail} { + *txn, err = client.ConstructPayment(baseAcct, baseAcct, 0, uint64(i), nil, "", [32]byte{}, 0, 0) + a.NoError(err) + } + + // success group + gidSuccess, err := client.GroupID([]transactions.Transaction{txn1Success, txn2Success}) + a.NoError(err) + + txn1Success.Group = gidSuccess + stxn1Success := transactions.SignedTxn{Txn: txn1Success, Lsig: lsigOK} + + txn2Success.Group = gidSuccess + stxn2Success, err := client.SignTransactionWithWallet(walletHandler, nil, txn2Success) + a.NoError(err) + + err = client.BroadcastTransactionGroup([]transactions.SignedTxn{stxn1Success, stxn2Success}) + a.NoError(err) + + // fail group + gidFail, err := client.GroupID([]transactions.Transaction{txn1Fail, txn2Fail}) + a.NoError(err) + + txn1Fail.Group = gidFail + stxn1Fail := transactions.SignedTxn{Txn: txn1Fail, Lsig: lsigTooLong} + + txn2Fail.Group = gidFail + stxn2Fail, err := client.SignTransactionWithWallet(walletHandler, nil, txn2Fail) + a.NoError(err) + + cp, err := client.ConsensusParams(0) + a.NoError(err) + + err = client.BroadcastTransactionGroup([]transactions.SignedTxn{stxn1Fail, stxn2Fail}) + if cp.EnableLogicSigSizePooling { + a.Contains(err.Error(), "more than the available pool") + } else { + a.Contains(err.Error(), "LogicSig too long") + } + + // wait for the second transaction in the successful group to confirm + txn2SuccessId := txn2Success.ID().String() + _, curRound := fixture.GetBalanceAndRound(baseAcct) + confirmed := fixture.WaitForTxnConfirmation(curRound+5, txn2SuccessId) + a.True(confirmed) +} diff --git a/test/testdata/nettemplates/TwoNodes50EachV18.json b/test/testdata/nettemplates/TwoNodes50EachV18.json new file mode 100644 index 0000000000..198a4f0f99 --- /dev/null +++ b/test/testdata/nettemplates/TwoNodes50EachV18.json @@ -0,0 +1,36 @@ +{ + "Genesis": { + "NetworkName": "tbd", + "ConsensusProtocol": "https://github.com/algorandfoundation/specs/tree/6c6bd668be0ab14098e51b37e806c509f7b7e31f", + "LastPartKeyRound": 3000, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 50, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 50, + "Online": true + } + ] + }, + "Nodes": [ + { + "Name": "Primary", + "IsRelay": true, + "Wallets": [ + { "Name": "Wallet1", + "ParticipationOnly": false } + ] + }, + { + "Name": "Node", + "Wallets": [ + { "Name": "Wallet2", + "ParticipationOnly": false } + ] + } + ] +} From 2265d1733f2d4da9736ac8793fedbc6481a0d0af Mon Sep 17 00:00:00 2001 From: giulio Date: Wed, 25 Sep 2024 22:56:15 +0200 Subject: [PATCH 11/11] Refactor --- .../features/transactions/logicsig_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e-go/features/transactions/logicsig_test.go b/test/e2e-go/features/transactions/logicsig_test.go index 087fed198d..aca4db77e3 100644 --- a/test/e2e-go/features/transactions/logicsig_test.go +++ b/test/e2e-go/features/transactions/logicsig_test.go @@ -28,9 +28,9 @@ import ( "github.com/stretchr/testify/require" ) -// CreateTealOfSize return a TEAL bytecode of `size` bytes which always succeeds. +// CreateProgramOfSize return a TEAL bytecode of `size` bytes which always succeeds. // `size` must be at least 9 bytes -func CreateTealOfSize(size uint, pragma uint) ([]byte, error) { +func CreateProgramOfSize(size uint, pragma uint) ([]byte, error) { if size < 9 { return nil, fmt.Errorf("size must be at least 9 bytes; got %d", size) } @@ -64,9 +64,9 @@ func TestLogicSigSizeBeforePooling(t *testing.T) { // From consensus version 18, we have lsigs with a maximum size of 1000 bytes. // We need to use pragma 1 for teal in v18 pragma := uint(1) - tealOK, err := CreateTealOfSize(1000, pragma) + tealOK, err := CreateProgramOfSize(1000, pragma) a.NoError(err) - tealTooLong, err := CreateTealOfSize(1001, pragma) + tealTooLong, err := CreateProgramOfSize(1001, pragma) a.NoError(err) testLogicSize(t, tealOK, tealTooLong, filepath.Join("nettemplates", "TwoNodes50EachV18.json")) @@ -78,9 +78,9 @@ func TestLogicSigSizeAfterPooling(t *testing.T) { a := require.New(fixtures.SynchronizedTest(t)) pragma := uint(1) - tealOK, err := CreateTealOfSize(2000, pragma) + tealOK, err := CreateProgramOfSize(2000, pragma) a.NoError(err) - tealTooLong, err := CreateTealOfSize(2001, pragma) + tealTooLong, err := CreateProgramOfSize(2001, pragma) a.NoError(err) // TODO: Update this when lsig pooling graduates from vFuture @@ -154,9 +154,9 @@ func testLogicSize(t *testing.T, tealOK, tealTooLong []byte, err = client.BroadcastTransactionGroup([]transactions.SignedTxn{stxn1Fail, stxn2Fail}) if cp.EnableLogicSigSizePooling { - a.Contains(err.Error(), "more than the available pool") + a.ErrorContains(err, "more than the available pool") } else { - a.Contains(err.Error(), "LogicSig too long") + a.ErrorContains(err, "LogicSig too long") } // wait for the second transaction in the successful group to confirm