Merge pull request #163 from GridProtectionAlliance/feature/v5.1.0

Feature/v5.1.0

CHANGELOG.md

@@ -45,3 +45,16 @@
 - Changed the query editor layout
 - Support Grafana version 11
 - Drop support for Grafana 8.x and 9.x
+
+## 5.1.0
+
+- Add units and description to new format - issue #154
+- Fixed digital state - issue #159
+- Fixed summary data - issue #160
+- Fixed an error in recorded max number of points - issue #162
+
+- Updated the query editor layout
+- Added boundary type support in recorded values
+- Recognize partial usage of variables in elements
+- Added configuration to hide API errors in panel
+- Truncate time from grafana date time picker to seconds

dist/plugin.json

@@ -39,8 +39,8 @@
       {"name": "Datasource Configuration", "path": "img/configuration.png"},
       {"name": "Annotations Editor", "path": "img/annotations.png"}
     ],
-    "version": "5.0.0",
-    "updated": "2024-06-14"
+    "version": "5.1.0",
+    "updated": "2024-09-19"
   },
   "dependencies": {
     "grafanaDependency": ">=10.1.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "grid-protection-alliance-osisoftpi-grafana",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "description": "OSISoft PI Grafana Plugin",
   "scripts": {
     "build": "webpack -c ./.config/webpack/webpack.config.ts --env production",

pkg/plugin/annotation_query.go

@@ -80,7 +80,8 @@ func (d *Datasource) processAnnotationQuery(ctx context.Context, query backend.D
 }
 
 func (q PiProcessedAnnotationQuery) getTimeRangeURIComponent() string {
-	return "&startTime=" + q.TimeRange.From.UTC().Format(time.RFC3339) + "&endTime=" + q.TimeRange.To.UTC().Format(time.RFC3339)
+	return "&startTime=" + q.TimeRange.From.UTC().Truncate(time.Second).Format(time.RFC3339) +
+		"&endTime=" + q.TimeRange.To.UTC().Truncate(time.Second).Format(time.RFC3339)
 }
 
 // getEventFrameQueryURL returns the URI for the event frame query

pkg/plugin/datasource.go

@@ -183,8 +183,8 @@ func (d *Datasource) QueryAnnotations(ctx context.Context, req *backend.QueryDat
 			trace.WithAttributes(
 				attribute.String("query.ref_id", q.RefID),
 				attribute.String("query.type", q.QueryType),
-				attribute.Int64("query.time_range.from", q.TimeRange.From.Unix()),
-				attribute.Int64("query.time_range.to", q.TimeRange.To.Unix()),
+				attribute.Int64("query.time_range.from", q.TimeRange.From.Truncate(time.Second).Unix()),
+				attribute.Int64("query.time_range.to", q.TimeRange.To.Truncate(time.Second).Unix()),
 			),
 		)
 
@@ -232,7 +232,7 @@ func (d *Datasource) QueryAnnotations(ctx context.Context, req *backend.QueryDat
 	return response, nil
 }
 
-// This function provides a way to proxy requests to the PI Web API. It is used to limit access fromt he frontend to the PI Web API.
+// This function provides a way to proxy requests to the PI Web API. It is used to limit access from the frontend to the PI Web API.
 // These endpoints are called by the front end while configuring the datasource, query, and annotations.
 func (d *Datasource) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
 	// Create spans for this function.
@@ -275,7 +275,10 @@ func (d *Datasource) CallResource(ctx context.Context, req *backend.CallResource
 
 	r, err := apiGet(ctx, d, req.URL)
 	if err != nil {
-		return err
+		return sender.Send(&backend.CallResourceResponse{
+			Status: http.StatusNotFound,
+			Body:   []byte(`{}`),
+		})
 	}
 	return sender.Send(&backend.CallResourceResponse{
 		Status: http.StatusOK,

pkg/plugin/helpers.go

@@ -352,7 +352,8 @@ func compatible(actual reflect.Type, expected reflect.Type) bool {
 		a == reflect.Float64)
 }
 
-func getDataLabels(useNewFormat bool, q *PiProcessedQuery, pointType string, summaryLabel string) map[string]string {
+func getDataLabels(useNewFormat bool, q *PiProcessedQuery, pointType string, description string,
+	units string, summaryLabel string) map[string]string {
 	var frameLabel map[string]string
 	summaryNewFormat := ""
 
@@ -373,26 +374,31 @@ func getDataLabels(useNewFormat bool, q *PiProcessedQuery, pointType string, sum
 		} else {
 			label = targetParts[len(targetParts)-1]
 		}
+		label += summaryLabel
 	}
 
 	if q.IsPIPoint {
 		// New format returns the full path with metadata
 		// PiPoint {element="PISERVER", name="Attribute", type="Float32"}
 		targetParts := strings.Split(q.FullTargetPath, `\`)
 		frameLabel = map[string]string{
-			"element": targetParts[0],
-			"name":    label,
-			"type":    pointType + summaryNewFormat,
+			"element":     targetParts[0],
+			"name":        label,
+			"type":        pointType + summaryNewFormat,
+			"description": description,
+			"units":       units,
 		}
 	} else {
 		// New format returns the full path with metadata
 		// Element|Attribute {element="Element", name="Attribute", type="Single"}
 		targetParts := strings.Split(q.FullTargetPath, `\`)
 		labelParts := strings.SplitN(targetParts[len(targetParts)-1], "|", 2)
 		frameLabel = map[string]string{
-			"element": labelParts[0],
-			"name":    label,
-			"type":    pointType + summaryNewFormat,
+			"element":     labelParts[0],
+			"name":        label,
+			"type":        pointType + summaryNewFormat,
+			"description": description,
+			"units":       units,
 		}
 	}
 
@@ -413,7 +419,7 @@ func convertItemsToDataFrame(processedQuery *PiProcessedQuery, d *Datasource, Su
 	webID := processedQuery.WebID
 	includeMetaData := processedQuery.UseUnit
 	digitalStates := processedQuery.DigitalStates
-	noDataReplace := processedQuery.getSummaryNoDataReplace()
+	noDataReplace := processedQuery.getNoDataReplace()
 
 	digitalStateValues := make([]string, 0)
 	sliceType := d.getTypeForWebID(webID)
@@ -427,7 +433,8 @@ func convertItemsToDataFrame(processedQuery *PiProcessedQuery, d *Datasource, Su
 	}
 
 	// get frame name
-	frameLabel := getDataLabels(d.isUsingNewFormat(), processedQuery, d.getPointTypeForWebID(webID), SummaryType)
+	frameLabel := getDataLabels(d.isUsingNewFormat(), processedQuery, d.getPointTypeForWebID(webID),
+		d.getDescriptionForWebID(webID), d.getUnitsForWebID(webID), SummaryType)
 
 	var labels map[string]string
 	var digitalState = d.getDigitalStateForWebID(webID)
@@ -469,13 +476,10 @@ func convertItemsToDataFrame(processedQuery *PiProcessedQuery, d *Datasource, Su
 			}
 		}
 
-		// if the value isn't good, or is not the same type as the slice,
-		// add it to the list of bad values and nullify later
-		//TODO we should make this pattern match the query options
-		_, digitalState := item.Value.(map[string]interface{})
+		_, digitalState = item.Value.(map[string]interface{})
 		if !item.isGood() {
 			fP = updateBadData(i, fP, item.Timestamp, noDataReplace)
-		} else if digitalState {
+		} else if digitalState { // digital state
 			var pds PointDigitalState
 			if b, err := json.Marshal(item.Value); err == nil {
 				if err := json.Unmarshal(b, &pds); err == nil {
@@ -495,7 +499,7 @@ func convertItemsToDataFrame(processedQuery *PiProcessedQuery, d *Datasource, Su
 				backend.Logger.Error("Error unmarshalling digital state", err)
 				fP = updateBadData(i, fP, item.Timestamp, noDataReplace)
 			}
-		} else if fP.val.Type().Kind() != fP.sliceType.Elem().Kind() {
+		} else if fP.val.Type().Kind() != fP.sliceType.Elem().Kind() { // mismatch - try conversion
 			backend.Logger.Warn("Mismatch type", "ValKind", fP.val.Type().String(), "Val", fP.val.Interface(),
 				"SliceKind", fP.sliceType.Elem().String(), "item", item)
 			if compatible(fP.val.Type(), fP.sliceType.Elem()) { // try to convert if numeric values
@@ -505,7 +509,7 @@ func convertItemsToDataFrame(processedQuery *PiProcessedQuery, d *Datasource, Su
 			} else {
 				fP = updateBadData(i, fP, item.Timestamp, noDataReplace)
 			}
-		} else {
+		} else { // normal
 			fP.timestamps = append(fP.timestamps, item.Timestamp)
 			fP.values = reflect.Append(reflect.ValueOf(fP.values), fP.val).Interface()
 			fP.prevVal = fP.val
@@ -518,7 +522,7 @@ func convertItemsToDataFrame(processedQuery *PiProcessedQuery, d *Datasource, Su
 	// in the slice type, or values that are not "good"
 	valuepointers := convertSliceToPointers(fP.values, fP.badValues)
 
-	timeField := data.NewField("time", nil, fP.timestamps)
+	timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, fP.timestamps)
 	if !digitalState || !digitalStates {
 		valueField := data.NewField(frameLabel["name"], labels, valuepointers)
 		frame.Fields = append(frame.Fields,

pkg/plugin/timeseries_query.go

@@ -91,13 +91,15 @@ func (d *Datasource) processQuery(query backend.DataQuery, datasourceUID string)
 				UID:                 datasourceUID,
 				IntervalNanoSeconds: PiQuery.Interval,
 				IsPIPoint:           PiQuery.Pi.IsPiPoint,
+				HideError:           PiQuery.Pi.HideError,
 				Streamable:          PiQuery.isStreamable() && *d.dataSourceOptions.UseExperimental && *d.dataSourceOptions.UseStreaming,
 				FullTargetPath:      fullTargetPath,
 				TargetPath:          targetBasePath,
 				UseUnit:             UseUnits,
 				DigitalStates:       DigitalStates,
 				Display:             PiQuery.Pi.Display,
 				Regex:               PiQuery.Pi.Regex,
+				Nodata:              PiQuery.Pi.Nodata,
 				Summary:             PiQuery.Pi.Summary,
 				Variable:            PiQuery.Pi.getVariable(i),
 				Index:               (j + 1) + 100*(i+1),
@@ -268,8 +270,10 @@ func (d *Datasource) processBatchtoFrames(processedQuery map[string][]PiProcesse
 		for _, q := range query {
 			// if there is an error in the query, we set the error in the subresponse and break out of the loop returning the error.
 			if q.Error != nil {
-				backend.Logger.Error("Error processing query", "RefID", RefID, "query", q)
-				subResponse.Error = q.Error
+				backend.Logger.Error("Error processing query", "RefID", RefID, "query", q, "hide", q.HideError)
+				if !q.HideError && strings.Contains(q.Error.Error(), "api error") {
+					subResponse.Error = q.Error
+				}
 				break
 			}
 
@@ -318,10 +322,10 @@ func (q *PIWebAPIQuery) isSummary() bool {
 	if q.Summary == nil {
 		return false
 	}
-	if q.Summary.Types == nil {
+	if q.Summary.Enable == nil {
 		return false
 	}
-	return *q.Summary.Basis != "" && len(*q.Summary.Types) > 0
+	return *q.Summary.Enable && *q.Summary.Basis != "" && len(*q.Summary.Types) > 0
 }
 
 // PiProcessedQuery isRegex returns true if the query is a regex query and is enabled
@@ -361,17 +365,28 @@ func (q *PiProcessedQuery) isRegexQuery() bool {
 // A default of 30s is returned if the summary duration is not provided by the frontend
 // or if the format is invalid
 func (q *PIWebAPIQuery) getSummaryDuration() string {
+	backend.Logger.Debug("Summary duration", "summary", q.Summary)
 	// Return the default value if the summary is not provided by the frontend
-	if q.Summary == nil || *q.Summary.Interval == "" {
+	if q.Summary == nil || q.Summary.Duration == nil || *q.Summary.Duration == "" {
 		return "30s"
 	}
+	return _getDurationBase(*q.Summary.Duration)
+}
 
-	// If the summary duration is provided, then validate the format piwebapi expects
+func (q *PIWebAPIQuery) getSampleInterval() string {
+	// Return the default value if the summary is not provided by the frontend
+	if q.Summary == nil || q.Summary.SampleInterval == nil || *q.Summary.SampleInterval == "" {
+		return "30s"
+	}
+	return _getDurationBase(*q.Summary.SampleInterval)
+}
 
+func _getDurationBase(duration string) string {
+	// If the summary duration is provided, then validate the format piwebapi expects
 	// Regular expression to match the format: <number><short_name>
 	pattern := `^(\d+(\.\d+)?)\s*(ms|s|m|h|d|mo|w|wd|yd)$`
 	re := regexp.MustCompile(pattern)
-	matches := re.FindStringSubmatch(*q.Summary.Interval)
+	matches := re.FindStringSubmatch(duration)
 
 	if len(matches) != 4 {
 		return "30s" // Return the default value if the format is invalid
@@ -391,11 +406,11 @@ func (q *PIWebAPIQuery) getSummaryDuration() string {
 	switch shortName {
 	case "ms", "s", "m", "h":
 		// Fractions allowed for millisecond, second, minute, and hour
-		return *q.Summary.Interval
+		return duration
 	case "d", "mo", "w", "wd", "yd":
 		// No fractions allowed for day, month, week, weekday, yearday
 		if numericPart == float64(int64(numericPart)) {
-			return *q.Summary.Interval
+			return duration
 		}
 	default:
 		return "30s" // Return the default value if the short name or fractions are not allowed
@@ -410,7 +425,13 @@ func (q *PIWebAPIQuery) getSummaryURIComponent() string {
 		uri += "&summaryType=" + t.Value.Value
 	}
 	uri += "&summaryBasis=" + *q.Summary.Basis
-	uri += "&summaryDuration=" + q.getSummaryDuration()
+	if q.Summary.Duration != nil && *q.Summary.Duration != "" {
+		uri += "&summaryDuration=" + q.getSummaryDuration()
+	}
+	if q.Summary.SampleTypeInterval != nil && *q.Summary.SampleTypeInterval &&
+		q.Summary.SampleInterval != nil && *q.Summary.SampleInterval != "" {
+		uri += "&sampleType=Interval&sampleInterval=" + q.getSampleInterval()
+	}
 	return uri
 }
 
@@ -554,6 +575,13 @@ func (q *Query) getMaxDataPoints() int {
 	return q.MaxDataPoints
 }
 
+func (q *Query) getBoundaryType() string {
+	if q.Pi.RecordedValues.BoundaryType != nil {
+		return *q.Pi.RecordedValues.BoundaryType
+	}
+	return "Inside"
+}
+
 func (q Query) getQueryBaseURL() string {
 	var uri string
 	if q.Pi.isExpression() {
@@ -562,10 +590,7 @@ func (q Query) getQueryBaseURL() string {
 			uri += "/times?time=" + q.getTimeRangeURIToComponent()
 		} else {
 			if q.Pi.isSummary() {
-				uri += "/summary" + q.getTimeRangeURIComponent()
-				if q.Pi.isInterpolated() {
-					uri += fmt.Sprintf("&sampleType=Interval&sampleInterval=%s", q.getIntervalTime())
-				}
+				uri += "/summary" + q.getTimeRangeURIComponent() + q.Pi.getSummaryURIComponent()
 			} else if q.Pi.isInterpolated() {
 				uri += "/intervals" + q.getTimeRangeURIComponent()
 				uri += fmt.Sprintf("&sampleInterval=%s", q.getIntervalTime())
@@ -587,12 +612,11 @@ func (q Query) getQueryBaseURL() string {
 			}
 		} else {
 			if q.Pi.isSummary() {
-				uri += "/summary" + q.getTimeRangeURIComponent() + fmt.Sprintf("&intervals=%d", q.getMaxDataPoints())
-				uri += q.Pi.getSummaryURIComponent()
+				uri += "/summary" + q.getTimeRangeURIComponent() + q.Pi.getSummaryURIComponent()
 			} else if q.Pi.isInterpolated() {
 				uri += "/interpolated" + q.getTimeRangeURIComponent() + fmt.Sprintf("&interval=%s", q.getIntervalTime())
 			} else if q.Pi.isRecordedValues() {
-				uri += "/recorded" + q.getTimeRangeURIComponent() + fmt.Sprintf("&maxCount=%d", q.getMaxDataPoints()) + "&boundaryType=Interpolated"
+				uri += "/recorded" + q.getTimeRangeURIComponent() + fmt.Sprintf("&maxCount=%d", q.getMaxDataPoints()) + "&boundaryType=" + q.getBoundaryType()
 			} else {
 				uri += "/plot" + q.getTimeRangeURIComponent() + fmt.Sprintf("&intervals=%d", q.getMaxDataPoints())
 			}