diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go
index fc69e7894..05f4514d0 100644
--- a/yb-voyager/cmd/analyzeSchema.go
+++ b/yb-voyager/cmd/analyzeSchema.go
@@ -581,7 +581,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I
1. normal index on column with these types
2. expression index with casting of unsupported column to supported types [No handling as such just to test as colName will not be there]
3. expression index with casting to unsupported types
- 4. normal index on column with UDTs
+ 4. normal index on column with UDTs
5. these type of indexes on different access method like gin etc.. [TODO to explore more, for now not reporting the indexes on anyother access method than btree]
*/
colName := param.GetIndexElem().GetName()
@@ -615,7 +615,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I
summaryMap["INDEX"].invalidCount[displayObjName] = true
reason := fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, castTypeName)
if slices.Contains(compositeTypes, fullCastTypeName) {
- reason = fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "user_defined_type")
+ reason = fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "user_defined_type")
}
reportCase(fpath, reason, "https://github.com/yugabyte/yugabyte-db/issues/9698",
"Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt,
@@ -1001,7 +1001,9 @@ func reportGeneratedStoredColumnTables(createTableNode *pg_query.Node_CreateStmt
if len(generatedColumns) > 0 {
summaryMap["TABLE"].invalidCount[sqlStmtInfo.objName] = true
reportCase(fpath, STORED_GENERATED_COLUMN_ISSUE_REASON+fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")),
- "https://github.com/yugabyte/yugabyte-db/issues/10695", "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GENERATED_STORED_COLUMN_DOC_LINK)
+ "https://github.com/yugabyte/yugabyte-db/issues/10695",
+ "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.",
+ TABLE, fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GENERATED_STORED_COLUMN_DOC_LINK)
}
}
@@ -1799,17 +1801,17 @@ func packAndSendAnalyzeSchemaPayload(status string) {
issue.SqlStatement = "" // Obfuscate sensitive information before sending to callhome cluster
issue.ObjectName = "XXX" // Redacting object name before sending
/*
- Removing Reason and Suggestion completely for now as there can be sensitive information in some of the cases
- so will enable it later with proper understanding
- some of the examples -
- Reason:
- Stored generated columns are not supported. [columns]
- Unsupported datatype - xml on [column]
- Unsupported datatype - xid on [column]
- Unsupported PG syntax - [error msg from parser]
- Policy require roles to be created. [role names]
- Suggestion:
- Foreign Table issue mentions Server name to be created.
+ Removing Reason and Suggestion completely for now as there can be sensitive information in some of the cases
+ so will enable it later with proper understanding
+ some of the examples -
+ Reason:
+ Stored generated columns are not supported. [columns]
+ Unsupported datatype - xml on [column]
+ Unsupported datatype - xid on [column]
+ Unsupported PG syntax - [error msg from parser]
+ Policy require roles to be created. [role names]
+ Suggestion:
+ Foreign Table issue mentions Server name to be created.
*/
issue.Reason = "XXX"
issue.Suggestion = "XXX"
diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go
index 2aaf1447e..ddfde30fb 100644
--- a/yb-voyager/cmd/assessMigrationCommand.go
+++ b/yb-voyager/cmd/assessMigrationCommand.go
@@ -38,6 +38,7 @@ import (
"github.com/yugabyte/yb-voyager/yb-voyager/src/cp"
"github.com/yugabyte/yb-voyager/yb-voyager/src/metadb"
"github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment"
+ "github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser"
"github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb"
"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
)
@@ -723,6 +724,12 @@ func generateAssessmentReport() (err error) {
}
assessmentReport.UnsupportedFeatures = append(assessmentReport.UnsupportedFeatures, unsupportedFeatures...)
+ unsupportedQueries, err := fetchUnsupportedQueryConstructs()
+ if err != nil {
+ return fmt.Errorf("failed to fetch unsupported queries on YugabyteDB: %w", err)
+ }
+ assessmentReport.UnsupportedQueryConstructs = unsupportedQueries
+
unsupportedDataTypes, unsupportedDataTypesForLiveMigration, err := fetchColumnsWithUnsupportedDataTypes()
if err != nil {
return fmt.Errorf("failed to fetch columns with unsupported data types: %w", err)
@@ -833,7 +840,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem
DisplayDDL: false,
Objects: []ObjectInfo{},
}
- unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type
+ unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type
unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type.
for _, unsupportedType := range unsupportedIndexDatatypes {
indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), schemaAnalysisReport, false, "")
@@ -906,6 +913,66 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) {
return unsupportedFeatures, nil
}
+func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error) {
+ query := fmt.Sprintf("SELECT DISTINCT query from %s", migassessment.DB_QUERIES_SUMMARY)
+ rows, err := assessmentDB.Query(query)
+ if err != nil {
+ return nil, fmt.Errorf("error querying=%s on assessmentDB: %w", query, err)
+ }
+ defer func() {
+ closeErr := rows.Close()
+ if closeErr != nil {
+ log.Warnf("error closing rows while fetching database queries summary metadata: %v", err)
+ }
+ }()
+
+ var executedQueries []string
+ for rows.Next() {
+ var executedQuery string
+ err := rows.Scan(&executedQuery)
+ if err != nil {
+ return nil, fmt.Errorf("error scanning rows: %w", err)
+ }
+ executedQueries = append(executedQueries, executedQuery)
+ }
+
+ var result []utils.UnsupportedQueryConstruct
+ for i := 0; i < len(executedQueries); i++ {
+ query := executedQueries[i]
+
+ // Check if the query starts with CREATE, INSERT, UPDATE, or DELETE
+ upperQuery := strings.ToUpper(strings.TrimSpace(query))
+ if strings.HasPrefix(upperQuery, "CREATE") || strings.HasPrefix(upperQuery, "INSERT") ||
+ strings.HasPrefix(upperQuery, "UPDATE") || strings.HasPrefix(upperQuery, "DELETE") {
+ continue
+ }
+
+ queryParser := queryparser.New(query)
+ err := queryParser.Parse()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse query-%s: %w", query, err)
+ }
+
+ // Check for unsupported constructs in the parsed query
+ unsupportedConstructType, err := queryParser.CheckUnsupportedQueryConstruct()
+ if err != nil {
+ log.Warnf("failed while trying to parse the query: %s", err.Error())
+ }
+ if unsupportedConstructType != "" {
+ fmt.Printf("Unsupported query: %s, Type: %s\n", query, unsupportedConstructType)
+ result = append(result, utils.UnsupportedQueryConstruct{
+ ConstructType: unsupportedConstructType,
+ Query: query,
+ })
+ }
+ }
+ if len(result) != 0 {
+ utils.PrintAndLog("Found YB unsupported queries in source DB, for more details please refer migration assessment report")
+ }
+ // TODO: sort the slice for better readability
+ return result, nil
+}
+
func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []utils.TableColumnsDataTypes, error) {
var unsupportedDataTypes []utils.TableColumnsDataTypes
var unsupportedDataTypesForLiveMigration []utils.TableColumnsDataTypes
@@ -914,7 +981,7 @@ func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []ut
migassessment.TABLE_COLUMNS_DATA_TYPES)
rows, err := assessmentDB.Query(query)
if err != nil {
- return nil, nil, fmt.Errorf("error querying-%s: %w", query, err)
+ return nil, nil, fmt.Errorf("error querying-%s on assessmentDB: %w", query, err)
}
defer func() {
closeErr := rows.Close()
diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go
index 11ce70643..e890efbc5 100644
--- a/yb-voyager/cmd/common.go
+++ b/yb-voyager/cmd/common.go
@@ -1117,6 +1117,7 @@ type AssessmentReport struct {
TableIndexStats *[]migassessment.TableIndexStats `json:"TableIndexStats"`
Notes []string `json:"Notes"`
MigrationCaveats []UnsupportedFeature `json:"MigrationCaveats"`
+ UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct `json:"UnsupportedQueryConstructs"`
}
type AssessmentDetail struct {
diff --git a/yb-voyager/cmd/templates/assessmentReport.template b/yb-voyager/cmd/templates/assessmentReport.template
index 364e7fe63..28911d15b 100644
--- a/yb-voyager/cmd/templates/assessmentReport.template
+++ b/yb-voyager/cmd/templates/assessmentReport.template
@@ -67,7 +67,7 @@
Migration Assessment Report
Database Name: {{.SchemaSummary.DBName}}
{{ if .SchemaSummary.SchemaNames}}
- Schema Name:
+
Schema Name:
{{range $i, $a := .SchemaSummary.SchemaNames}}
{{$a}}
{{end}}
@@ -213,7 +213,6 @@
No unsupported features were present among the ones assessed.
{{end}}
-
{{if .Notes}}
@@ -227,6 +226,41 @@
{{end}}
+ Unsupported Query Constructs
+ Below are source database queries not supported in YugabyteDB, identified by scanning system tables.:
+
+
+ | Construct Type |
+ Queries |
+
+ {{ $currentType := "" }}
+ {{ range $i, $construct := .UnsupportedQueryConstructs }}
+ {{ if ne $construct.ConstructType $currentType }}
+ {{ if ne $currentType "" }}
+
+
+
+
+ {{ end }}
+
+ | {{ $construct.ConstructType }} |
+
+
+ |
+
+ {{ end }}
+
+
+
{{ if .MigrationCaveats}}
Migration caveats
diff --git a/yb-voyager/src/migassessment/assessmentDB.go b/yb-voyager/src/migassessment/assessmentDB.go
index de7b58b88..e31421a63 100644
--- a/yb-voyager/src/migassessment/assessmentDB.go
+++ b/yb-voyager/src/migassessment/assessmentDB.go
@@ -38,6 +38,7 @@ const (
OBJECT_TYPE_MAPPING = "object_type_mapping"
TABLE_COLUMNS_DATA_TYPES = "table_columns_data_types"
TABLE_INDEX_STATS = "table_index_stats"
+ DB_QUERIES_SUMMARY = "db_queries_summary"
PARTITIONED_TABLE_OBJECT_TYPE = "partitioned table"
PARTITIONED_INDEX_OBJECT_TYPE = "partitioned index"
@@ -132,6 +133,16 @@ func InitAssessmentDB() error {
parent_table_name TEXT,
size_in_bytes INTEGER,
PRIMARY KEY(schema_name, object_name));`, TABLE_INDEX_STATS),
+ fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
+ queryid BIGINT,
+ query TEXT,
+ calls BIGINT,
+ total_exec_time REAL,
+ mean_exec_time REAL,
+ min_exec_time REAL,
+ max_exec_time REAL,
+ rows BIGINT,
+ PRIMARY KEY(queryid));`, DB_QUERIES_SUMMARY),
}
for _, cmd := range cmds {
diff --git a/yb-voyager/src/queryparser/queryParser.go b/yb-voyager/src/queryparser/queryParser.go
new file mode 100644
index 000000000..409918da2
--- /dev/null
+++ b/yb-voyager/src/queryparser/queryParser.go
@@ -0,0 +1,33 @@
+package queryparser
+
+import (
+ pg_query "github.com/pganalyze/pg_query_go/v5"
+)
+
+type QueryParser struct {
+ QueryString string
+ ParseTree *pg_query.ParseResult
+}
+
+func New(query string) *QueryParser {
+ return &QueryParser{
+ QueryString: query,
+ }
+}
+
+func (qp *QueryParser) Parse() error {
+ tree, err := pg_query.Parse(qp.QueryString)
+ if err != nil {
+ return err
+ }
+ qp.ParseTree = tree
+ return nil
+}
+
+func (qp *QueryParser) CheckUnsupportedQueryConstruct() (string, error) {
+ if qp.containsAdvisoryLocks() {
+ return ADVISORY_LOCKS, nil
+ }
+ // TODO: Add checks for unsupported constructs - system columns, XML functions
+ return "", nil
+}
diff --git a/yb-voyager/src/queryparser/unsupportedConstructs.go b/yb-voyager/src/queryparser/unsupportedConstructs.go
new file mode 100644
index 000000000..4379028dd
--- /dev/null
+++ b/yb-voyager/src/queryparser/unsupportedConstructs.go
@@ -0,0 +1,126 @@
+package queryparser
+
+import (
+ "slices"
+
+ pg_query "github.com/pganalyze/pg_query_go/v5"
+)
+
+const (
+ ADVISORY_LOCKS = "Advisory Locks"
+ SYSTEM_COLUMNS = "System Columns"
+ XML_FUNCTIONS = "XML Functions"
+)
+
+// NOTE: pg parser converts the func names in parse tree to lower case by default
+var advisoryLockFunctions = []string{
+ "pg_advisory_lock", "pg_try_advisory_lock", "pg_advisory_xact_lock",
+ "pg_advisory_unlock", "pg_advisory_unlock_all", "pg_try_advisory_xact_lock",
+}
+
+func (qp *QueryParser) containsAdvisoryLocks() bool {
+ if qp.ParseTree == nil {
+ return false
+ }
+
+ selectStmtNode, isSelectStmt := qp.ParseTree.Stmts[0].Stmt.Node.(*pg_query.Node_SelectStmt)
+ if !isSelectStmt {
+ return false
+ }
+
+ // Check advisory locks in the target list
+ if containsAdvisoryLocksInTargetList(selectStmtNode.SelectStmt.TargetList) {
+ return true
+ }
+
+ // Check advisory locks in FROM clause
+ if containsAdvisoryLocksInFromClause(selectStmtNode.SelectStmt.FromClause) {
+ return true
+ }
+
+ // Check advisory locks in WHERE clause
+ if containsAdvisoryLocksInWhereClause(selectStmtNode.SelectStmt.WhereClause) {
+ return true
+ }
+ return false
+}
+
+// Checks for advisory lock functions in the main query's target list.
+// This includes direct function calls like:
+// Example: SELECT pg_advisory_lock(1);
+// Example: SELECT col1, pg_try_advisory_lock(2) FROM my_table;
+func containsAdvisoryLocksInTargetList(targetList []*pg_query.Node) bool {
+ for _, target := range targetList {
+ if resTarget := target.GetResTarget(); resTarget != nil {
+ if funcCallNode, isFuncCall := resTarget.Val.Node.(*pg_query.Node_FuncCall); isFuncCall {
+ funcList := funcCallNode.FuncCall.Funcname
+ functionName := funcList[0].GetString_().Sval
+ if slices.Contains(advisoryLockFunctions, functionName) {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+// Recursively checks the FROM clause for subqueries containing advisory locks.
+// This covers advisory locks embedded in subqueries such as:
+// Example: SELECT * FROM (SELECT pg_advisory_lock(1)) AS lock_query;
+// Example: SELECT * FROM my_table JOIN LATERAL (SELECT pg_try_advisory_xact_lock(3)) AS lock_check ON true;
+func containsAdvisoryLocksInFromClause(fromClause []*pg_query.Node) bool {
+ for _, fromItem := range fromClause {
+ if subselectNode, isSubselect := fromItem.Node.(*pg_query.Node_RangeSubselect); isSubselect {
+ subSelectStmt := subselectNode.RangeSubselect.Subquery.GetSelectStmt()
+ if subSelectStmt != nil {
+ // Recursively check for advisory locks in the subquery's target list
+ if containsAdvisoryLocksInTargetList(subSelectStmt.TargetList) {
+ return true
+ }
+ // Recursively check for advisory locks in the subquery's FROM clause
+ if containsAdvisoryLocksInFromClause(subSelectStmt.FromClause) {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+// Recursively checks the WHERE clause for advisory lock functions.
+// This allows for advisory locks embedded within conditions like:
+// Example: SELECT * FROM my_table WHERE pg_advisory_lock(1) = true;
+func containsAdvisoryLocksInWhereClause(whereClause *pg_query.Node) bool {
+ if whereClause == nil {
+ return false
+ }
+
+ if funcCallNode := whereClause.GetFuncCall(); funcCallNode != nil {
+ funcList := funcCallNode.Funcname
+ functionName := funcList[0].GetString_().Sval
+ if slices.Contains(advisoryLockFunctions, functionName) {
+ return true
+ }
+ }
+
+ // Recursively check for advisory locks in nested expressions
+ switch n := whereClause.Node.(type) {
+ case *pg_query.Node_BoolExpr:
+ return containsAdvisoryLocksInNodeList(n.BoolExpr.Args)
+ case *pg_query.Node_SubLink:
+ return containsAdvisoryLocksInWhereClause(n.SubLink.Subselect)
+ // Add more cases for other types of expressions as needed
+ }
+
+ return false
+}
+
+// Helper function to check advisory locks within a node list (for WHERE clause and nested conditions).
+func containsAdvisoryLocksInNodeList(nodes []*pg_query.Node) bool {
+ for _, node := range nodes {
+ if containsAdvisoryLocksInWhereClause(node) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz b/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz
index 7be29d652..b69b3d7f7 100644
Binary files a/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz and b/yb-voyager/src/srcdb/data/gather-assessment-metadata.tar.gz differ
diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db_queries_summary.psql b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db_queries_summary.psql
new file mode 100644
index 000000000..26de12265
--- /dev/null
+++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/postgresql/db_queries_summary.psql
@@ -0,0 +1,20 @@
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+
+CREATE TEMP TABLE temp_table AS
+SELECT
+ queryid,
+ query,
+ calls,
+ total_exec_time,
+ mean_exec_time,
+ min_exec_time,
+ max_exec_time,
+ rows
+FROM
+ pg_stat_statements
+WHERE
+ dbid = (SELECT oid FROM pg_database WHERE datname = current_database());
+
+\copy temp_table to 'db_queries_summary.csv' WITH CSV HEADER;
+
+DROP TABLE temp_table;
\ No newline at end of file
diff --git a/yb-voyager/src/utils/commonVariables.go b/yb-voyager/src/utils/commonVariables.go
index a024c5952..c8f36d8f9 100644
--- a/yb-voyager/src/utils/commonVariables.go
+++ b/yb-voyager/src/utils/commonVariables.go
@@ -121,6 +121,11 @@ type TableColumnsDataTypes struct {
DataType string `json:"DataType"`
}
+type UnsupportedQueryConstruct struct {
+ ConstructType string
+ Query string
+}
+
// ================== Segment ==============================
type Segment struct {
Num int