Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: renaming merge commands to profile #3630

Merged
merged 5 commits into from
Oct 17, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions cmd/profilecli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func main() {
adminCmd := app.Command("admin", "Administrative tasks for Pyroscope cluster operators.")

blocksCmd := adminCmd.Command("blocks", "Operate on Grafana Pyroscope's blocks.")
blocksCmd.Flag("path", "Path to blocks directory").Default("./data/local").StringVar(&cfg.blocks.path)
blocksCmd.Flag("path", "Path to blocks directory").Default("./data/anonymous/local").StringVar(&cfg.blocks.path)

blocksListCmd := blocksCmd.Command("list", "List blocks.")
blocksListCmd.Flag("restore-missing-meta", "").Default("false").BoolVar(&cfg.blocks.restoreMissingMeta)
Expand All @@ -56,6 +56,12 @@ func main() {
blocksCompactCmd.Arg("dest", "The destination where compacted blocks should be stored.").Required().StringVar(&cfg.blocks.compact.dst)
blocksCompactCmd.Flag("shards", "The amount of shards to split output blocks into.").Default("0").IntVar(&cfg.blocks.compact.shards)

blocksQueryCmd := blocksCmd.Command("query", "Query on local/remote blocks.")
blocksQuerySeriesCmd := blocksQueryCmd.Command("series", "Request series labels on local/remote blocks.")
blocksQuerySeriesParams := addBlocksQuerySeriesParams(blocksQuerySeriesCmd)
blocksQueryProfileCmd := blocksQueryCmd.Command("profile", "Request merged profile on local/remote block.").Alias("merge")
blocksQueryProfileParams := addBlocksQueryProfileParams(blocksQueryProfileCmd)

parquetCmd := adminCmd.Command("parquet", "Operate on a Parquet file.")
parquetInspectCmd := parquetCmd.Command("inspect", "Inspect a parquet file's structure.")
parquetInspectFiles := parquetInspectCmd.Arg("file", "parquet file path").Required().ExistingFiles()
Expand All @@ -65,9 +71,9 @@ func main() {
tsdbSeriesFiles := tsdbSeriesCmd.Arg("file", "tsdb file path").Required().ExistingFiles()

queryCmd := app.Command("query", "Query profile store.")
queryMergeCmd := queryCmd.Command("merge", "Request merged profile.")
queryMergeOutput := queryMergeCmd.Flag("output", "How to output the result, examples: console, raw, pprof=./my.pprof").Default("console").String()
queryMergeParams := addQueryMergeParams(queryMergeCmd)
queryProfileCmd := queryCmd.Command("profile", "Request merged profile.").Alias("merge")
queryProfileOutput := queryProfileCmd.Flag("output", "How to output the result, examples: console, raw, pprof=./my.pprof").Default("console").String()
queryProfileParams := addQueryProfileParams(queryProfileCmd)
queryGoPGOCmd := queryCmd.Command("go-pgo", "Request profile for Go PGO.")
queryGoPGOOutput := queryGoPGOCmd.Flag("output", "How to output the result, examples: console, raw, pprof=./my.pprof").Default("pprof=./default.pgo").String()
queryGoPGOParams := addQueryGoPGOParams(queryGoPGOCmd)
Expand All @@ -79,12 +85,6 @@ func main() {
queryTracerCmd := app.Command("query-tracer", "Analyze query traces.")
queryTracerParams := addQueryTracerParams(queryTracerCmd)

queryBlocksCmd := app.Command("query-blocks", "Query on local/remote blocks")
queryBlocksSeriesCmd := queryBlocksCmd.Command("series", "Request series labels on local/remote blocks")
queryBlocksSeriesParams := addQueryBlocksSeriesParams(queryBlocksSeriesCmd)
queryBlocksMergeCmd := queryBlocksCmd.Command("merge", "Request merged profile.")
queryBlocksMergeParams := addQueryBlocksMergeParams(queryBlocksMergeCmd)

uploadCmd := app.Command("upload", "Upload profile(s).")
uploadParams := addUploadParams(uploadCmd)

Expand Down Expand Up @@ -125,8 +125,8 @@ func main() {
os.Exit(checkError(err))
}
}
case queryMergeCmd.FullCommand():
if err := queryMerge(ctx, queryMergeParams, *queryMergeOutput); err != nil {
case queryProfileCmd.FullCommand():
if err := queryProfile(ctx, queryProfileParams, *queryProfileOutput); err != nil {
os.Exit(checkError(err))
}
case queryGoPGOCmd.FullCommand():
Expand All @@ -138,12 +138,12 @@ func main() {
os.Exit(checkError(err))
}

case queryBlocksSeriesCmd.FullCommand():
if err := queryBlocksSeries(ctx, queryBlocksSeriesParams); err != nil {
case blocksQuerySeriesCmd.FullCommand():
if err := blocksQuerySeries(ctx, blocksQuerySeriesParams); err != nil {
os.Exit(checkError(err))
}
case queryBlocksMergeCmd.FullCommand():
if err := queryBlocksMerge(ctx, queryBlocksMergeParams); err != nil {
case blocksQueryProfileCmd.FullCommand():
if err := blocksQueryProfile(ctx, blocksQueryProfileParams); err != nil {
os.Exit(checkError(err))
}

Expand Down
56 changes: 27 additions & 29 deletions cmd/profilecli/query-blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,60 +19,58 @@ import (
"github.com/grafana/pyroscope/pkg/phlaredb"
)

type queryBlocksParams struct {
LocalPath string
type blocksQueryParams struct {
BucketName string
BlockIds []string
TenantID string
ObjectStoreType string
Query string
}

type queryBlocksMergeParams struct {
*queryBlocksParams
type blocksQueryProfileParams struct {
*blocksQueryParams
Output string
ProfileType string
StacktraceSelector []string
}

type queryBlocksSeriesParams struct {
*queryBlocksParams
type blocksQuerySeriesParams struct {
*blocksQueryParams
LabelNames []string
}

func addQueryBlocksParams(queryCmd commander) *queryBlocksParams {
params := new(queryBlocksParams)
queryCmd.Flag("local-path", "Path to blocks directory.").Default("./data/anonymous/local").StringVar(&params.LocalPath)
func addBlocksQueryParams(queryCmd commander) *blocksQueryParams {
params := new(blocksQueryParams)
queryCmd.Flag("bucket-name", "The name of the object storage bucket.").StringVar(&params.BucketName)
queryCmd.Flag("object-store-type", "The type of the object storage (e.g., gcs).").Default("gcs").StringVar(&params.ObjectStoreType)
queryCmd.Flag("block-ids", "List of blocks ids to query on").StringsVar(&params.BlockIds)
queryCmd.Flag("block", "Block ids to query on (accepts multiples)").StringsVar(&params.BlockIds)
queryCmd.Flag("tenant-id", "Tenant id of the queried block for remote bucket").StringVar(&params.TenantID)
queryCmd.Flag("query", "Label selector to query.").Default("{}").StringVar(&params.Query)
return params
}

func addQueryBlocksMergeParams(queryCmd commander) *queryBlocksMergeParams {
params := new(queryBlocksMergeParams)
params.queryBlocksParams = addQueryBlocksParams(queryCmd)
func addBlocksQueryProfileParams(queryCmd commander) *blocksQueryProfileParams {
params := new(blocksQueryProfileParams)
params.blocksQueryParams = addBlocksQueryParams(queryCmd)
queryCmd.Flag("output", "How to output the result, examples: console, raw, pprof=./my.pprof").Default("console").StringVar(&params.Output)
queryCmd.Flag("profile-type", "Profile type to query.").Default("process_cpu:cpu:nanoseconds:cpu:nanoseconds").StringVar(&params.ProfileType)
queryCmd.Flag("stacktrace-selector", "Only query locations with those symbols. Provide multiple times starting with the root").StringsVar(&params.StacktraceSelector)
return params
}

func addQueryBlocksSeriesParams(queryCmd commander) *queryBlocksSeriesParams {
params := new(queryBlocksSeriesParams)
params.queryBlocksParams = addQueryBlocksParams(queryCmd)
func addBlocksQuerySeriesParams(queryCmd commander) *blocksQuerySeriesParams {
params := new(blocksQuerySeriesParams)
params.blocksQueryParams = addBlocksQueryParams(queryCmd)
queryCmd.Flag("label-names", "Filter returned labels to the supplied label names. Without any filter all labels are returned.").StringsVar(&params.LabelNames)
return params
}

func queryBlocksMerge(ctx context.Context, params *queryBlocksMergeParams) error {
level.Info(logger).Log("msg", "query-block merge", "blockIds", fmt.Sprintf("%v", params.BlockIds), "localPath",
params.LocalPath, "bucketName", params.BucketName, "tenantId", params.TenantID, "query", params.Query, "type", params.ProfileType)
func blocksQueryProfile(ctx context.Context, params *blocksQueryProfileParams) error {
level.Info(logger).Log("msg", "blocks query profile", "blockIds", fmt.Sprintf("%v", params.BlockIds), "path",
cfg.blocks.path, "bucketName", params.BucketName, "tenantId", params.TenantID, "query", params.Query, "type", params.ProfileType)

if len(params.BlockIds) > 1 {
return errors.New("query merge is limited to a single block")
return errors.New("query profile is limited to a single block")
}

profileType, err := model.ParseProfileTypeSelector(params.ProfileType)
Expand All @@ -94,7 +92,7 @@ func queryBlocksMerge(ctx context.Context, params *queryBlocksMergeParams) error
level.Info(logger).Log("msg", "selecting with stackstrace selector", "call-site", fmt.Sprintf("%#+v", params.StacktraceSelector))
}

bucket, err := getBucket(ctx, params.queryBlocksParams)
bucket, err := getBucket(ctx, params.blocksQueryParams)
if err != nil {
return err
}
Expand Down Expand Up @@ -122,11 +120,11 @@ func queryBlocksMerge(ctx context.Context, params *queryBlocksMergeParams) error
return outputMergeProfile(ctx, params.Output, resp)
}

func queryBlocksSeries(ctx context.Context, params *queryBlocksSeriesParams) error {
level.Info(logger).Log("msg", "query-block series", "labelNames", fmt.Sprintf("%v", params.LabelNames),
"blockIds", fmt.Sprintf("%v", params.BlockIds), "localPath", params.LocalPath, "bucketName", params.BucketName, "tenantId", params.TenantID)
func blocksQuerySeries(ctx context.Context, params *blocksQuerySeriesParams) error {
level.Info(logger).Log("msg", "blocks query series", "labelNames", fmt.Sprintf("%v", params.LabelNames),
"blockIds", fmt.Sprintf("%v", params.BlockIds), "path", cfg.blocks.path, "bucketName", params.BucketName, "tenantId", params.TenantID)

bucket, err := getBucket(ctx, params.queryBlocksParams)
bucket, err := getBucket(ctx, params.blocksQueryParams)
if err != nil {
return err
}
Expand All @@ -136,7 +134,7 @@ func queryBlocksSeries(ctx context.Context, params *queryBlocksSeriesParams) err
var from, to int64
from, to = math.MaxInt64, math.MinInt64
var targetBlockQueriers phlaredb.Queriers
for _, blockId := range params.queryBlocksParams.BlockIds {
for _, blockId := range params.blocksQueryParams.BlockIds {
meta, err := blockQuerier.BlockMeta(ctx, blockId)
if err != nil {
return err
Expand All @@ -161,15 +159,15 @@ func queryBlocksSeries(ctx context.Context, params *queryBlocksSeriesParams) err
return outputSeries(response.Msg.LabelsSet)
}

func getBucket(ctx context.Context, params *queryBlocksParams) (objstore.Bucket, error) {
func getBucket(ctx context.Context, params *blocksQueryParams) (objstore.Bucket, error) {
if params.BucketName != "" {
return getRemoteBucket(ctx, params)
} else {
return filesystem.NewBucket(params.LocalPath)
return filesystem.NewBucket(cfg.blocks.path)
}
}

func getRemoteBucket(ctx context.Context, params *queryBlocksParams) (objstore.Bucket, error) {
func getRemoteBucket(ctx context.Context, params *blocksQueryParams) (objstore.Bucket, error) {
if params.TenantID == "" {
return nil, errors.New("specify tenant id for remote bucket")
}
Expand Down
12 changes: 6 additions & 6 deletions cmd/profilecli/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,21 +90,21 @@ func addQueryParams(queryCmd commander) *queryParams {
return params
}

type queryMergeParams struct {
type queryProfileParams struct {
*queryParams
ProfileType string
StacktraceSelector []string
}

func addQueryMergeParams(queryCmd commander) *queryMergeParams {
params := new(queryMergeParams)
func addQueryProfileParams(queryCmd commander) *queryProfileParams {
params := new(queryProfileParams)
params.queryParams = addQueryParams(queryCmd)
queryCmd.Flag("profile-type", "Profile type to query.").Default("process_cpu:cpu:nanoseconds:cpu:nanoseconds").StringVar(&params.ProfileType)
queryCmd.Flag("stacktrace-selector", "Only query locations with those symbols. Provide multiple times starting with the root").StringsVar(&params.StacktraceSelector)
return params
}

func queryMerge(ctx context.Context, params *queryMergeParams, outputFlag string) (err error) {
func queryProfile(ctx context.Context, params *queryProfileParams, outputFlag string) (err error) {
from, to, err := params.parseFromTo()
if err != nil {
return err
Expand Down Expand Up @@ -145,14 +145,14 @@ func selectMergeProfile(ctx context.Context, client *phlareClient, outputFlag st
}

type queryGoPGOParams struct {
*queryMergeParams
*queryProfileParams
KeepLocations uint32
AggregateCallees bool
}

func addQueryGoPGOParams(queryCmd commander) *queryGoPGOParams {
params := new(queryGoPGOParams)
params.queryMergeParams = addQueryMergeParams(queryCmd)
params.queryProfileParams = addQueryProfileParams(queryCmd)
queryCmd.Flag("keep-locations", "Number of leaf locations to keep.").Default("5").Uint32Var(&params.KeepLocations)
queryCmd.Flag("aggregate-callees", "Aggregate samples for the same callee by ignoring the line numbers in the leaf locations.").Default("true").BoolVar(&params.AggregateCallees)
return params
Expand Down
12 changes: 6 additions & 6 deletions docs/sources/view-and-analyze-profile-data/profile-cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,27 +224,27 @@ You can narrow the results down with the `--query` flag. See `profilecli help qu

### Reading a raw profile from a Pyroscope server

You can use the `profilecli query merge` command to retrieve a merged (aggregated) profile from a Pyroscope server.
You can use the `profilecli query profile` command to retrieve a merged (aggregated) profile from a Pyroscope server.
The command merges all samples found in the profile store for the specified query and time range.
By default it looks for samples within the last hour, though this can be controlled with the `--from` and `--to` flags. The source data can be narrowed down with the `--query` flag in the same way as with the `series` command.

#### Query merge steps
#### Query profile steps

1. Specify optional flags.

- You can provide a label selector using the `--query` flag, for example, `--query='{service_name="my_application_name"}'`.
- You can provide a custom time range using the `--from` and `--to` flags, for example, `--from="now-3h" --to="now"`.
- You can specify the profile type via the `--profile-type` flag. The available profile types are listed in the output of the `profilecli query series` command.

2. Construct and execute the Query Merge command.
2. Construct and execute the Query Profile command.
alsoba13 marked this conversation as resolved.
Show resolved Hide resolved

- Here's a basic command template:
```bash
export PROFILECLI_URL=<pyroscope_server_url>
export PROFILECLI_USERNAME=<username>
export PROFILECLI_PASSWORD=<password>

profilecli query merge \
profilecli query profile \
--profile-type=<profile_type> \
--query='{<label_name>="<label_value>"' \
--from="<from>" --to="<to>"
Expand All @@ -256,7 +256,7 @@ By default it looks for samples within the last hour, though this can be control
export PROFILECLI_USERNAME=my_username
export PROFILECLI_PASSWORD=my_password

profilecli query merge \
profilecli query profile \
--profile-type=memory:inuse_space:bytes:space:bytes \
--query='{service_name="my_application_name"}' \
--from="now-1h" --to="now"
Expand All @@ -278,7 +278,7 @@ By default it looks for samples within the last hour, though this can be control
### Exporting a profile for Go PGO

You can use the `profilecli query go-pgo` command to retrieve an aggregated profile from a Pyroscope server for use with Go PGO.
Profiles retrieved with `profilecli query merge` include all samples found in the profile store, resulting in a large profile size.
Profiles retrieved with `profilecli query profile` include all samples found in the profile store, resulting in a large profile size.
The profile size may cause issues with network transfer and slow down the PGO process.
In contrast, profiles retrieved with `profilecli query go-pgo` include only the information used in Go PGO, making them significantly smaller and more efficient to handle.
By default, it looks for samples within the last hour, though this can be controlled with the `--from` and `--to` flags. The source data can be narrowed down with the `--query` flag in the same way as with the `query` command.
Expand Down
Loading