From 997e693082103cebeb128afb047e00f4949f27f2 Mon Sep 17 00:00:00 2001 From: karminski Date: Wed, 10 Jan 2024 20:47:34 +0800 Subject: [PATCH] ADD template parser for restapi body. --- src/actionruntime/restapi/service.go | 10 + src/actionruntime/restapi/types.go | 25 --- src/utils/parser/template/parser.go | 259 +++++++++++++++++++++++ src/utils/parser/template/parser_test.go | 65 ++++++ 4 files changed, 334 insertions(+), 25 deletions(-) create mode 100644 src/utils/parser/template/parser.go create mode 100644 src/utils/parser/template/parser_test.go diff --git a/src/actionruntime/restapi/service.go b/src/actionruntime/restapi/service.go index c7f66788..a2b35f93 100644 --- a/src/actionruntime/restapi/service.go +++ b/src/actionruntime/restapi/service.go @@ -27,6 +27,7 @@ import ( "github.com/go-resty/resty/v2" "github.com/icholy/digest" "github.com/illacloud/builder-backend/src/actionruntime/common" + parser_template "github.com/illacloud/builder-backend/src/utils/parser/template" "github.com/mitchellh/mapstructure" ) @@ -100,6 +101,10 @@ func (r *RESTAPIConnector) Run(resourceOptions map[string]interface{}, actionOpt } var err error + // process context + r.Action.SetRawQueryAndContext(rawActionOptions) + fmt.Printf("[DUMP] r.Action.Context: %+v\n", r.Action.Context) + fmt.Printf("[DUMP] RESTAPIConnector.Resource: %+v, r.Resource.BaseURL: %+v\n", r.Resource, r.Resource.BaseURL) uriParsed, err := url.ParseRequestURI(r.Resource.BaseURL) fmt.Printf("[DUMP] ParseRequestURI: uriParsed:%+v, err: %+v\n", uriParsed, err) @@ -205,6 +210,11 @@ func (r *RESTAPIConnector) Run(resourceOptions map[string]interface{}, actionOpt fmt.Printf("[DUMP] restapi r.Action: %+v\n", r.Action) b := r.Action.ReflectBodyToRaw() + var errInAssembleBodyContent error + b.Content, errInAssembleBodyContent = parser_template.AssembleTemplateWithVariable(b.Content, r.Action.Context) + if errInAssembleBodyContent != nil { + return res, errInAssembleBodyContent + } fmt.Printf("[DUMP] b := r.Action.ReflectBodyToRaw(): %+v\n", b) rawBody, contentType := b.UnmarshalRawBody() diff --git a/src/actionruntime/restapi/types.go b/src/actionruntime/restapi/types.go index fd2432c7..570392ff 100644 --- a/src/actionruntime/restapi/types.go +++ b/src/actionruntime/restapi/types.go @@ -262,31 +262,6 @@ func (q *RESTTemplate) DoesContextValied(rawTemplate map[string]interface{}) boo } func (q *RESTTemplate) SetRawQueryAndContext(rawTemplate map[string]interface{}) error { - assertStringField := func(rawTemplate map[string]interface{}, field string) (string, error) { - // set template - fieldRaw, hit := rawTemplate[field] - if !hit { - return "", fmt.Errorf("missing query field \"%s\" for SetRawQueryAndContext() in query", FIELD_URL) - } - fieldAsserted, assertPass := fieldRaw.(string) - if !assertPass { - return "", fmt.Errorf("query field \"%s\" assert failed in SetRawQueryAndContext() method", FIELD_URL) - } - return fieldAsserted, nil - } - - // set string field - var errInAssert error - if q.URL, errInAssert = assertStringField(rawTemplate, FIELD_URL); errInAssert != nil { - return errInAssert - } - if q.Method, errInAssert = assertStringField(rawTemplate, FIELD_METHOD); errInAssert != nil { - return errInAssert - } - if q.BodyType, errInAssert = assertStringField(rawTemplate, FIELD_BODY_TYPE); errInAssert != nil { - return errInAssert - } - // set context contextRaw, hit := rawTemplate[FIELD_CONTEXT] if !hit { diff --git a/src/utils/parser/template/parser.go b/src/utils/parser/template/parser.go new file mode 100644 index 00000000..d4fe5bc1 --- /dev/null +++ b/src/utils/parser/template/parser.go @@ -0,0 +1,259 @@ +package parser_template + +import ( + "encoding/json" + "errors" + "strconv" +) + +type JSONNumberConvertor struct { + Payload json.Number `json:"payload"` +} + +func (c *JSONNumberConvertor) ExportNumberInString() string { + return string(c.Payload) +} + +func ExportFloat64ToNumberInString(payload float64) string { + dummyJSON := map[string]interface{}{ + "payload": payload, + } + dummyJSONInByte, _ := json.Marshal(dummyJSON) + jsonNumberConvertor := &JSONNumberConvertor{} + json.Unmarshal(dummyJSONInByte, &jsonNumberConvertor) + return jsonNumberConvertor.ExportNumberInString() +} + +func ExtractVariableNameConst(template string) []string { + variableNames := make([]string, 0) + + variableLT := make(map[string]string, 0) + processesPrompt := "" + variable := "" + escapedBracketWithVariable := "" + leftBraketCounter := 0 + rightBraketCounter := 0 + leftBracketPlus := func() { + leftBraketCounter++ + escapedBracketWithVariable += "{" + } + rightBracketPlus := func() { + rightBraketCounter++ + escapedBracketWithVariable += "}" + } + escapeIllegalLeftBracket := func() { + leftBraketCounter = 0 + processesPrompt += escapedBracketWithVariable + "{" + escapedBracketWithVariable = "" + } + escapeIllegalRightBracket := func() { + rightBraketCounter = 0 + processesPrompt += escapedBracketWithVariable + "}" + escapedBracketWithVariable = "" + } + isIgnoredCharacter := func(c rune) bool { + switch c { + case '\t', '\n', '\v', '\f', '\r', ' ': + return true + } + return false + } + for _, c := range template { + + // process bracket + // '' + '{' or '{' + '{' + if c == '{' && leftBraketCounter <= 1 { + leftBracketPlus() + continue + } + // '{{...' + '{' + if c == '{' && leftBraketCounter > 1 { + escapeIllegalLeftBracket() + continue + } + // '}...' + '{' + if c == '{' && rightBraketCounter > 0 { + escapeIllegalRightBracket() + continue + } + // '' + '}' or '{' + '}' + if c == '}' && leftBraketCounter != 2 && rightBraketCounter == 0 { + escapeIllegalRightBracket() + continue + } + // '{{' + '}' + if c == '}' && leftBraketCounter == 2 && rightBraketCounter == 0 { + rightBracketPlus() + continue + } + // '{{' + '}}', hit! + if c == '}' && leftBraketCounter == 2 && rightBraketCounter == 1 { + rightBraketCounter++ + escapedBracketWithVariable += "}" + // collect variable name + variableNames = append(variableNames, variable) + // process varibale signal + + variableMappedValue, hitVariable := variableLT[variable] + if !hitVariable { + processesPrompt += escapedBracketWithVariable + } else { + processesPrompt += variableMappedValue + } + escapedBracketWithVariable = "" + variable = "" + continue + } + // process bracker inner (record variable name) + if leftBraketCounter == 2 && rightBraketCounter == 0 { + // filter escape character + if isIgnoredCharacter(c) { + continue + } + // collect variable name + variable += string(c) + escapedBracketWithVariable += string(c) + continue + } + // process other utf-8 character + leftBraketCounter = 0 + rightBraketCounter = 0 + processesPrompt += escapedBracketWithVariable + string(c) + escapedBracketWithVariable = "" + variable = "" + continue + } + + return variableNames +} + +func AssembleTemplateWithVariable(template string, variableLT map[string]interface{}) (string, error) { + processesPrompt := "" + variable := "" + escapedBracketWithVariable := "" + leftBraketCounter := 0 + rightBraketCounter := 0 + leftBracketPlus := func() { + leftBraketCounter++ + escapedBracketWithVariable += "{" + } + rightBracketPlus := func() { + rightBraketCounter++ + escapedBracketWithVariable += "}" + } + escapeIllegalLeftBracket := func() { + leftBraketCounter = 0 + processesPrompt += escapedBracketWithVariable + "{" + escapedBracketWithVariable = "" + } + escapeIllegalRightBracket := func() { + rightBraketCounter = 0 + processesPrompt += escapedBracketWithVariable + "}" + escapedBracketWithVariable = "" + } + isIgnoredCharacter := func(c rune) bool { + switch c { + case '\t', '\n', '\v', '\f', '\r', ' ': + return true + } + return false + } + assertDataAndConvertToString := func(data interface{}) (string, error) { + switch data.(type) { + case int: + dataInInt := data.(int) + return strconv.Itoa(dataInInt), nil + case int64: + dataInInt64 := data.(int64) + return strconv.FormatInt(dataInInt64, 10), nil + case float32: + case float64: + dataInFloat64 := data.(float64) + return ExportFloat64ToNumberInString(dataInFloat64), nil + case string: + return data.(string), nil + case bool: + dataInBool := data.(bool) + if dataInBool { + return "true", nil + } + return "false", nil + default: + // treat other types as json + dataInJsonByte, errInMarshal := json.Marshal(data) + if errInMarshal != nil { + return "", errInMarshal + } + return string(dataInJsonByte), nil + } + return "", errors.New("can not convert target data into string") + } + for _, c := range template { + + // process bracket + // '' + '{' or '{' + '{' + if c == '{' && leftBraketCounter <= 1 { + leftBracketPlus() + continue + } + // '{{...' + '{' + if c == '{' && leftBraketCounter > 1 { + escapeIllegalLeftBracket() + continue + } + // '}...' + '{' + if c == '{' && rightBraketCounter > 0 { + escapeIllegalRightBracket() + continue + } + // '' + '}' or '{' + '}' + if c == '}' && leftBraketCounter != 2 && rightBraketCounter == 0 { + escapeIllegalRightBracket() + continue + } + // '{{' + '}' + if c == '}' && leftBraketCounter == 2 && rightBraketCounter == 0 { + rightBracketPlus() + continue + } + // '{{' + '}}', hit! + if c == '}' && leftBraketCounter == 2 && rightBraketCounter == 1 { + rightBraketCounter++ + escapedBracketWithVariable += "}" + // process varibale signal + + variableMappedValue, hitVariable := variableLT[variable] + if !hitVariable { + processesPrompt += escapedBracketWithVariable + } else { + valueInString, errInConvertData := assertDataAndConvertToString(variableMappedValue) + if errInConvertData != nil { + return "", errInConvertData + } + processesPrompt += valueInString + } + escapedBracketWithVariable = "" + variable = "" + continue + } + // process bracker inner (record variable name) + if leftBraketCounter == 2 && rightBraketCounter == 0 { + // filter escape character + if isIgnoredCharacter(c) { + continue + } + // collect variable name + variable += string(c) + escapedBracketWithVariable += string(c) + continue + } + // process other utf-8 character + leftBraketCounter = 0 + rightBraketCounter = 0 + processesPrompt += escapedBracketWithVariable + string(c) + escapedBracketWithVariable = "" + variable = "" + continue + } + return processesPrompt, nil +} diff --git a/src/utils/parser/template/parser_test.go b/src/utils/parser/template/parser_test.go new file mode 100644 index 00000000..470695f1 --- /dev/null +++ b/src/utils/parser/template/parser_test.go @@ -0,0 +1,65 @@ +package parser_template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSample(t *testing.T) { + assert.Nil(t, nil) +} + +func TestGetAllVariableNameConstFromActionTemplate1(t *testing.T) { + actionTemplate := `{"mode": "sql", "query": "select * \nfrom users\njoin orders\non users.id = orders.id\nwhere {{!input1.value}} or lower(users.name) like '%{{input1.value.toLowerCase()}}%'"}` + variableNames := ExtractVariableNameConst(actionTemplate) + + assert.Equal(t, "!input1.value", variableNames[0], "it should be string \"yes\" ") + assert.Equal(t, "input1.value.toLowerCase()", variableNames[1], "it should be string \"yes\" ") + +} + +func TestGetAllVariableNameConstFromActionTemplate2(t *testing.T) { + actionTemplate := `{"mode": "sql-safe", "query": "select count(distinct email) from users\nwhere DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE 'GMT+8') BETWEEN '{{date1.value}}'::date AND '{{date2.value}}'::date"}` + variableNames := ExtractVariableNameConst(actionTemplate) + + assert.Equal(t, "date1.value", variableNames[0], "it should be string \"yes\" ") + assert.Equal(t, "date2.value", variableNames[1], "it should be string \"yes\" ") + +} + +func TestAssembleTemplateWithVariable_case1_BoolAndString(t *testing.T) { + actionTemplate := `{"mode": "sql", "query": "select * \nfrom users\njoin orders\non users.id = orders.id\nwhere {{!input1.value}} or lower(users.name) like '%{{input1.value.toLowerCase()}}%'"}` + dataLT := map[string]interface{}{ + "!input1.value": false, + "input1.value.toLowerCase()": "jackmall", + } + finalTemplate, errInAssemble := AssembleTemplateWithVariable(actionTemplate, dataLT) + assert.Nil(t, errInAssemble) + assert.Equal(t, `{"mode": "sql", "query": "select * \nfrom users\njoin orders\non users.id = orders.id\nwhere false or lower(users.name) like '%jackmall%'"}`, finalTemplate, "it should be equal ") + +} + +func TestAssembleTemplateWithVariable_case2_integerAndFloat(t *testing.T) { + actionTemplate := `{"mode": "sql-safe", "query": "select count(distinct email) from users\nwhere DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE 'GMT+8') BETWEEN '{{date1.value}}'::date AND '{{date2.value}}'::date"}` + dataLT := map[string]interface{}{ + "date1.value": 99811111111231220, + "date2.value": 14.90000002, + } + finalTemplate, errInAssemble := AssembleTemplateWithVariable(actionTemplate, dataLT) + assert.Nil(t, errInAssemble) + assert.Equal(t, `{"mode": "sql-safe", "query": "select count(distinct email) from users\nwhere DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE 'GMT+8') BETWEEN '99811111111231220'::date AND '14.90000002'::date"}`, finalTemplate, "it should be equal ") + +} + +func TestAssembleTemplateWithVariable_case3_WarppedString(t *testing.T) { + actionTemplate := `{"mode": "sql", "query": "select * \nfrom users\njoin orders\non users.id = orders.id\nwhere {{!input1.value}} or lower(users.name) like '%{{input1.value.toLowerCase()}}%'"}` + dataLT := map[string]interface{}{ + "!input1.value": "\"BIG APPLE\"", + "input1.value.toLowerCase()": "[A\nAA]", + } + finalTemplate, errInAssemble := AssembleTemplateWithVariable(actionTemplate, dataLT) + assert.Nil(t, errInAssemble) + assert.Equal(t, `{"mode": "sql", "query": "select * \nfrom users\njoin orders\non users.id = orders.id\nwhere \"BIG APPLE\" or lower(users.name) like '%[A\nAA]%'"}`, finalTemplate, "it should be equal ") + +}