[DB-13024] Implement Detailed Reporting for Unsupported Query Constructs

yb-voyager/cmd/analyzeSchema.go

@@ -581,7 +581,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I
 				1. normal index on column with these types
 				2. expression index with  casting of unsupported column to supported types [No handling as such just to test as colName will not be there]
 				3. expression index with  casting to unsupported types
-				4. normal index on column with UDTs 
+				4. normal index on column with UDTs
 				5. these type of indexes on different access method like gin etc.. [TODO to explore more, for now not reporting the indexes on anyother access method than btree]
 		*/
 		colName := param.GetIndexElem().GetName()
@@ -615,7 +615,7 @@ func reportUnsupportedIndexesOnComplexDatatypes(createIndexNode *pg_query.Node_I
 			summaryMap["INDEX"].invalidCount[displayObjName] = true
 			reason := fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, castTypeName)
 			if slices.Contains(compositeTypes, fullCastTypeName) {
-				reason = fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "user_defined_type") 
+				reason = fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, "user_defined_type")
 			}
 			reportCase(fpath, reason, "https://github.com/yugabyte/yugabyte-db/issues/9698",
 				"Refer to the docs link for the workaround", "INDEX", displayObjName, sqlStmtInfo.formattedStmt,
@@ -1001,7 +1001,9 @@ func reportGeneratedStoredColumnTables(createTableNode *pg_query.Node_CreateStmt
 	if len(generatedColumns) > 0 {
 		summaryMap["TABLE"].invalidCount[sqlStmtInfo.objName] = true
 		reportCase(fpath, STORED_GENERATED_COLUMN_ISSUE_REASON+fmt.Sprintf(" Generated Columns: (%s)", strings.Join(generatedColumns, ",")),
-			"https://github.com/yugabyte/yugabyte-db/issues/10695", "Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.", "TABLE", fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GENERATED_STORED_COLUMN_DOC_LINK)
+			"https://github.com/yugabyte/yugabyte-db/issues/10695",
+			"Using Triggers to update the generated columns is one way to work around this issue, refer docs link for more details.",
+			TABLE, fullyQualifiedName, sqlStmtInfo.formattedStmt, UNSUPPORTED_FEATURES, GENERATED_STORED_COLUMN_DOC_LINK)
 	}
 }
 
@@ -1799,17 +1801,17 @@ func packAndSendAnalyzeSchemaPayload(status string) {
 		issue.SqlStatement = ""  // Obfuscate sensitive information before sending to callhome cluster
 		issue.ObjectName = "XXX" // Redacting object name before sending
 		/*
-		Removing Reason and Suggestion completely for now as there can be sensitive information in some of the cases 
-		so will enable it later with proper understanding
-		some of the examples -
-		Reason:
-			Stored generated columns are not supported. [columns]
-			Unsupported datatype - xml on [column]
-			Unsupported datatype - xid on [column]
-			Unsupported PG syntax - [error msg from parser]
-			Policy require roles to be created. [role names]
-		Suggestion:
-			Foreign Table issue mentions Server name to be created.
+			Removing Reason and Suggestion completely for now as there can be sensitive information in some of the cases
+			so will enable it later with proper understanding
+			some of the examples -
+			Reason:
+				Stored generated columns are not supported. [columns]
+				Unsupported datatype - xml on [column]
+				Unsupported datatype - xid on [column]
+				Unsupported PG syntax - [error msg from parser]
+				Policy require roles to be created. [role names]
+			Suggestion:
+				Foreign Table issue mentions Server name to be created.
 		*/
 		issue.Reason = "XXX"
 		issue.Suggestion = "XXX"

yb-voyager/cmd/assessMigrationCommand.go

@@ -38,6 +38,7 @@ import (
 	"github.com/yugabyte/yb-voyager/yb-voyager/src/cp"
 	"github.com/yugabyte/yb-voyager/yb-voyager/src/metadb"
 	"github.com/yugabyte/yb-voyager/yb-voyager/src/migassessment"
+	"github.com/yugabyte/yb-voyager/yb-voyager/src/queryparser"
 	"github.com/yugabyte/yb-voyager/yb-voyager/src/srcdb"
 	"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
 )
@@ -723,6 +724,12 @@ func generateAssessmentReport() (err error) {
 	}
 	assessmentReport.UnsupportedFeatures = append(assessmentReport.UnsupportedFeatures, unsupportedFeatures...)
 
+	unsupportedQueries, err := fetchUnsupportedQueryConstructs()
+	if err != nil {
+		return fmt.Errorf("failed to fetch unsupported queries on YugabyteDB: %w", err)
+	}
+	assessmentReport.UnsupportedQueryConstructs = unsupportedQueries
+
 	unsupportedDataTypes, unsupportedDataTypesForLiveMigration, err := fetchColumnsWithUnsupportedDataTypes()
 	if err != nil {
 		return fmt.Errorf("failed to fetch columns with unsupported data types: %w", err)
@@ -833,7 +840,7 @@ func getIndexesOnComplexTypeUnsupportedFeature(schemaAnalysisiReport utils.Schem
 		DisplayDDL:  false,
 		Objects:     []ObjectInfo{},
 	}
-	unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array") // adding it here only as we know issue form analyze will come with type
+	unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "array")             // adding it here only as we know issue form analyze will come with type
 	unsupportedIndexDatatypes = append(unsupportedIndexDatatypes, "user_defined_type") // adding it here as we UDTs will come with this type.
 	for _, unsupportedType := range unsupportedIndexDatatypes {
 		indexes := getUnsupportedFeaturesFromSchemaAnalysisReport(fmt.Sprintf("%s indexes", unsupportedType), fmt.Sprintf(ISSUE_INDEX_WITH_COMPLEX_DATATYPES, unsupportedType), schemaAnalysisReport, false, "")
@@ -906,6 +913,66 @@ func fetchUnsupportedObjectTypes() ([]UnsupportedFeature, error) {
 	return unsupportedFeatures, nil
 }
 
+func fetchUnsupportedQueryConstructs() ([]utils.UnsupportedQueryConstruct, error) {
+	query := fmt.Sprintf("SELECT DISTINCT query from %s", migassessment.DB_QUERIES_SUMMARY)
+	rows, err := assessmentDB.Query(query)
+	if err != nil {
+		return nil, fmt.Errorf("error querying=%s on assessmentDB: %w", query, err)
+	}
+	defer func() {
+		closeErr := rows.Close()
+		if closeErr != nil {
+			log.Warnf("error closing rows while fetching database queries summary metadata: %v", err)
+		}
+	}()
+
+	var executedQueries []string
+	for rows.Next() {
+		var executedQuery string
+		err := rows.Scan(&executedQuery)
+		if err != nil {
+			return nil, fmt.Errorf("error scanning rows: %w", err)
+		}
+		executedQueries = append(executedQueries, executedQuery)
+	}
+
+	var result []utils.UnsupportedQueryConstruct
+	for i := 0; i < len(executedQueries); i++ {
+		query := executedQueries[i]
+
+		// Check if the query starts with CREATE, INSERT, UPDATE, or DELETE
+		upperQuery := strings.ToUpper(strings.TrimSpace(query))
+		if strings.HasPrefix(upperQuery, "CREATE") || strings.HasPrefix(upperQuery, "INSERT") ||
+			strings.HasPrefix(upperQuery, "UPDATE") || strings.HasPrefix(upperQuery, "DELETE") {
+			continue
+		}
+
+		queryParser := queryparser.New(query)
+		err := queryParser.Parse()
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse query-%s: %w", query, err)
+		}
+
+		// Check for unsupported constructs in the parsed query
+		unsupportedConstructType, err := queryParser.CheckUnsupportedQueryConstruct()
+		if err != nil {
+			log.Warnf("failed while trying to parse the query: %s", err.Error())
+		}
+		if unsupportedConstructType != "" {
+			fmt.Printf("Unsupported query: %s, Type: %s\n", query, unsupportedConstructType)
+			result = append(result, utils.UnsupportedQueryConstruct{
+				ConstructType: unsupportedConstructType,
+				Query:         query,
+			})
+		}
+	}
+	if len(result) != 0 {
+		utils.PrintAndLog("Found YB unsupported queries in source DB, for more details please refer migration assessment report")
+	}
+	// TODO: sort the slice for better readability
+	return result, nil
+}
+
 func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []utils.TableColumnsDataTypes, error) {
 	var unsupportedDataTypes []utils.TableColumnsDataTypes
 	var unsupportedDataTypesForLiveMigration []utils.TableColumnsDataTypes
@@ -914,7 +981,7 @@ func fetchColumnsWithUnsupportedDataTypes() ([]utils.TableColumnsDataTypes, []ut
 		migassessment.TABLE_COLUMNS_DATA_TYPES)
 	rows, err := assessmentDB.Query(query)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error querying-%s: %w", query, err)
+		return nil, nil, fmt.Errorf("error querying-%s on assessmentDB: %w", query, err)
 	}
 	defer func() {
 		closeErr := rows.Close()

yb-voyager/cmd/common.go

@@ -1117,6 +1117,7 @@ type AssessmentReport struct {
 	TableIndexStats            *[]migassessment.TableIndexStats      `json:"TableIndexStats"`
 	Notes                      []string                              `json:"Notes"`
 	MigrationCaveats           []UnsupportedFeature                  `json:"MigrationCaveats"`
+	UnsupportedQueryConstructs []utils.UnsupportedQueryConstruct     `json:"UnsupportedQueryConstructs"`
 }
 
 type AssessmentDetail struct {

yb-voyager/cmd/templates/assessmentReport.template

@@ -67,7 +67,7 @@
         <h1>Migration Assessment Report</h1>
         <p><strong>Database Name:</strong> {{.SchemaSummary.DBName}}</p>
         {{ if .SchemaSummary.SchemaNames}}
-            <p><strong>Schema Name:</strong> 
+            <p><strong>Schema Name:</strong>
             {{range $i, $a := .SchemaSummary.SchemaNames}}
                 {{$a}}&nbsp;
             {{end}}
@@ -213,7 +213,6 @@
             <p>No unsupported features were present among the ones assessed.</p>
         {{end}}
 
-
         {{if .Notes}}
             <br>
             <hr>
@@ -227,6 +226,41 @@
             </div>
         {{end}}
 
+        <h2>Unsupported  Query Constructs</h2>
+        <p>Below are source database queries not supported in YugabyteDB, identified by scanning system tables.:</p>
+        <table>
+            <tr>
+                <th>Construct Type</th>
+                <th>Queries</th>
+            </tr>
+            {{ $currentType := "" }}
+            {{ range $i, $construct := .UnsupportedQueryConstructs }}
+                {{ if ne $construct.ConstructType $currentType }}
+                    {{ if ne $currentType "" }}
+                            </ul>
+                        </div>
+                    </td>
+                    </tr>
+                    {{ end }}
+                    <tr>
+                        <td><strong>{{ $construct.ConstructType }}</strong></td>
+                        <td>
+                            <div class="scrollable-div">
+                                <ul>
+                {{ $currentType = $construct.ConstructType }}
+                {{ end }}
+                <li class="list_item">{{ $construct.Query }}</li>
+            {{ end }}
+            <!-- Close the last construct type -->
+            {{ if ne $currentType "" }}
+                        </ul>
+                    </div>
+                </td>
+                </tr>
+            {{ end }}
+        </table>
+
+
         {{ if .MigrationCaveats}}
             <h2>Migration caveats</h2>
             <p></p>

yb-voyager/src/migassessment/assessmentDB.go

@@ -38,6 +38,7 @@ const (
 	OBJECT_TYPE_MAPPING      = "object_type_mapping"
 	TABLE_COLUMNS_DATA_TYPES = "table_columns_data_types"
 	TABLE_INDEX_STATS        = "table_index_stats"
+	DB_QUERIES_SUMMARY       = "db_queries_summary"
 
 	PARTITIONED_TABLE_OBJECT_TYPE = "partitioned table"
 	PARTITIONED_INDEX_OBJECT_TYPE = "partitioned index"
@@ -132,6 +133,16 @@ func InitAssessmentDB() error {
 			parent_table_name   TEXT,
 			size_in_bytes       INTEGER,
 			PRIMARY KEY(schema_name, object_name));`, TABLE_INDEX_STATS),
+		fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
+			queryid			BIGINT,
+			query		TEXT,
+			calls		BIGINT,
+			total_exec_time REAL,
+			mean_exec_time REAL,
+			min_exec_time REAL,
+			max_exec_time REAL,
+			rows		BIGINT,
+			PRIMARY KEY(queryid));`, DB_QUERIES_SUMMARY),
 	}
 
 	for _, cmd := range cmds {

yb-voyager/src/queryparser/queryParser.go

@@ -0,0 +1,33 @@
+package queryparser
+
+import (
+	pg_query "github.com/pganalyze/pg_query_go/v5"
+)
+
+type QueryParser struct {
+	QueryString string
+	ParseTree   *pg_query.ParseResult
+}
+
+func New(query string) *QueryParser {
+	return &QueryParser{
+		QueryString: query,
+	}
+}
+
+func (qp *QueryParser) Parse() error {
+	tree, err := pg_query.Parse(qp.QueryString)
+	if err != nil {
+		return err
+	}
+	qp.ParseTree = tree
+	return nil
+}
+
+func (qp *QueryParser) CheckUnsupportedQueryConstruct() (string, error) {
+	if qp.containsAdvisoryLocks() {
+		return ADVISORY_LOCKS, nil
+	}
+	// TODO: Add checks for unsupported constructs - system columns, XML functions
+	return "", nil
+}