diff --git a/cmd/api/src/api/registration/v2.go b/cmd/api/src/api/registration/v2.go index 488690cce..379eed9bb 100644 --- a/cmd/api/src/api/registration/v2.go +++ b/cmd/api/src/api/registration/v2.go @@ -286,6 +286,7 @@ func NewV2API(cfg config.Configuration, resources v2.Resources, routerInst *rout // Datapipe API routerInst.GET("/api/v2/datapipe/status", resources.GetDatapipeStatus).RequireAuth(), //TODO: Update the permission on this once we get something more concrete + routerInst.GET("/api/v2/analysis/status", resources.GetAnalysisRequest).RequirePermissions(permissions.GraphDBRead), routerInst.PUT("/api/v2/analysis", resources.RequestAnalysis).RequirePermissions(permissions.GraphDBWrite), ) } diff --git a/cmd/api/src/api/v2/agi.go b/cmd/api/src/api/v2/agi.go index 976abef46..ab1f9ec18 100644 --- a/cmd/api/src/api/v2/agi.go +++ b/cmd/api/src/api/v2/agi.go @@ -34,6 +34,7 @@ import ( "github.com/specterops/bloodhound/headers" "github.com/specterops/bloodhound/log" "github.com/specterops/bloodhound/src/api" + "github.com/specterops/bloodhound/src/auth" "github.com/specterops/bloodhound/src/ctx" "github.com/specterops/bloodhound/src/model" "github.com/specterops/bloodhound/src/utils" @@ -265,7 +266,18 @@ func (s Resources) UpdateAssetGroupSelectors(response http.ResponseWriter, reque if assetGroup.Tag == model.TierZeroAssetGroupTag { // When T0 asset group selectors are modified, entire analysis must be re-run - s.TaskNotifier.RequestAnalysis() + var userId string + if user, isUser := auth.GetUserFromAuthCtx(ctx.FromRequest(request).AuthCtx); !isUser { + log.Warnf("encountered request analysis for unknown user, this shouldn't happen") + userId = "unknown-user-update-asset-group-selectors" + } else { + userId = user.ID.String() + } + + if err := s.DB.RequestAnalysis(request.Context(), userId); err != nil { + api.HandleDatabaseError(request, response, err) + return + } } api.WriteBasicResponse(request.Context(), result, http.StatusCreated, response) diff --git a/cmd/api/src/api/v2/agi_test.go b/cmd/api/src/api/v2/agi_test.go index 89689c7fa..e83270888 100644 --- a/cmd/api/src/api/v2/agi_test.go +++ b/cmd/api/src/api/v2/agi_test.go @@ -25,25 +25,24 @@ import ( "net/url" "testing" - "github.com/specterops/bloodhound/headers" - "github.com/specterops/bloodhound/mediatypes" - "github.com/specterops/bloodhound/src/auth" - "github.com/specterops/bloodhound/src/test/must" - + "github.com/gofrs/uuid" "github.com/gorilla/mux" "github.com/specterops/bloodhound/dawgs/graph" "github.com/specterops/bloodhound/errors" "github.com/specterops/bloodhound/graphschema/ad" "github.com/specterops/bloodhound/graphschema/azure" "github.com/specterops/bloodhound/graphschema/common" + "github.com/specterops/bloodhound/headers" + "github.com/specterops/bloodhound/mediatypes" "github.com/specterops/bloodhound/src/api" v2 "github.com/specterops/bloodhound/src/api/v2" "github.com/specterops/bloodhound/src/api/v2/apitest" + "github.com/specterops/bloodhound/src/auth" "github.com/specterops/bloodhound/src/ctx" - datapipeMocks "github.com/specterops/bloodhound/src/daemons/datapipe/mocks" dbmocks "github.com/specterops/bloodhound/src/database/mocks" "github.com/specterops/bloodhound/src/model" queriesMocks "github.com/specterops/bloodhound/src/queries/mocks" + "github.com/specterops/bloodhound/src/test/must" "github.com/specterops/bloodhound/src/utils/test" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -592,16 +591,11 @@ func TestResources_UpdateAssetGroupSelectors_SuccessT0(t *testing.T) { mockDB.EXPECT().UpdateAssetGroupSelectors(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedResult, nil) mockGraph.EXPECT().UpdateSelectorTags(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - mockTasker := datapipeMocks.NewMockTasker(mockCtrl) - // MockTasker should receive a call to RequestAnalysis() since this is a Tier Zero Asset group. + // Should receive a call to RequestAnalysis() since this is a Tier Zero Asset group. // Analysis must be run upon updating a T0 AG - mockTasker.EXPECT().RequestAnalysis() + mockDB.EXPECT().RequestAnalysis(gomock.Any(), uuid.UUID{}.String()) - handlers := v2.Resources{ - DB: mockDB, - TaskNotifier: mockTasker, - GraphQuery: mockGraph, - } + handlers := v2.Resources{DB: mockDB, GraphQuery: mockGraph} response := httptest.NewRecorder() handler := http.HandlerFunc(handlers.UpdateAssetGroupSelectors) @@ -687,15 +681,11 @@ func TestResources_UpdateAssetGroupSelectors_SuccessOwned(t *testing.T) { mockGraph.EXPECT().UpdateSelectorTags(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - mockTasker := datapipeMocks.NewMockTasker(mockCtrl) - // NOTE MockTasker should NOT receive a call to RequestAnalysis() since this is not a Tier Zero Asset group. + // NOTE should NOT receive a call to RequestAnalysis() since this is not a Tier Zero Asset group. // Analysis should not be re-run when a non T0 AG is updated + mockDB.EXPECT().RequestAnalysis(gomock.Any(), uuid.UUID{}.String()).Times(0) - handlers := v2.Resources{ - DB: mockDB, - TaskNotifier: mockTasker, - GraphQuery: mockGraph, - } + handlers := v2.Resources{DB: mockDB, GraphQuery: mockGraph} response := httptest.NewRecorder() handler := http.HandlerFunc(handlers.UpdateAssetGroupSelectors) diff --git a/cmd/api/src/api/v2/analysisrequest.go b/cmd/api/src/api/v2/analysisrequest.go new file mode 100644 index 000000000..19551e50a --- /dev/null +++ b/cmd/api/src/api/v2/analysisrequest.go @@ -0,0 +1,57 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package v2 + +import ( + "database/sql" + "net/http" + + "github.com/specterops/bloodhound/errors" + "github.com/specterops/bloodhound/log" + "github.com/specterops/bloodhound/src/api" + "github.com/specterops/bloodhound/src/auth" + "github.com/specterops/bloodhound/src/ctx" +) + +func (s Resources) GetAnalysisRequest(response http.ResponseWriter, request *http.Request) { + if analRequest, err := s.DB.GetAnalysisRequest(request.Context()); err != nil && !errors.Is(err, sql.ErrNoRows) { + api.HandleDatabaseError(request, response, err) + } else if errors.Is(err, sql.ErrNoRows) { + api.WriteErrorResponse(request.Context(), api.BuildErrorResponse(http.StatusNotFound, api.ErrorResponseDetailsResourceNotFound, request), response) + } else { + api.WriteBasicResponse(request.Context(), analRequest, http.StatusOK, response) + } +} + +func (s Resources) RequestAnalysis(response http.ResponseWriter, request *http.Request) { + defer log.Measure(log.LevelDebug, "Requesting analysis")() + + var userId string + if user, isUser := auth.GetUserFromAuthCtx(ctx.FromRequest(request).AuthCtx); !isUser { + log.Warnf("encountered request analysis for unknown user, this shouldn't happen") + userId = "unknown-user" + } else { + userId = user.ID.String() + } + + if err := s.DB.RequestAnalysis(request.Context(), userId); err != nil { + api.HandleDatabaseError(request, response, err) + return + } + + response.WriteHeader(http.StatusAccepted) +} diff --git a/cmd/api/src/api/v2/analysisrequest_test.go b/cmd/api/src/api/v2/analysisrequest_test.go new file mode 100644 index 000000000..4beba7a12 --- /dev/null +++ b/cmd/api/src/api/v2/analysisrequest_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package v2_test + +import ( + "testing" + + "github.com/specterops/bloodhound/src/api/v2/integration" + "github.com/specterops/bloodhound/src/model" + "github.com/stretchr/testify/require" +) + +func TestRequestAnalysis(t *testing.T) { + testCtx := integration.NewFOSSContext(t) + + err := testCtx.AdminClient().RequestAnalysis() + require.Nil(t, err) + + analReq, err := testCtx.AdminClient().GetAnalysisRequest() + require.Nil(t, err) + require.Equal(t, analReq.RequestType, model.AnalysisRequestAnalysis) +} diff --git a/cmd/api/src/api/v2/apiclient/analysisrequest.go b/cmd/api/src/api/v2/apiclient/analysisrequest.go new file mode 100644 index 000000000..2430fb33c --- /dev/null +++ b/cmd/api/src/api/v2/apiclient/analysisrequest.go @@ -0,0 +1,54 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package apiclient + +import ( + "net/http" + + "github.com/specterops/bloodhound/src/api" + "github.com/specterops/bloodhound/src/model" +) + +func (s Client) RequestAnalysis() error { + if response, err := s.Request(http.MethodPut, "api/v2/analysis", nil, nil); err != nil { + return err + } else { + defer response.Body.Close() + + if api.IsErrorResponse(response) { + return ReadAPIError(response) + } + + return nil + } +} + +func (s Client) GetAnalysisRequest() (model.AnalysisRequest, error) { + var analReq model.AnalysisRequest + + if response, err := s.Request(http.MethodGet, "api/v2/analysis/status", nil, nil); err != nil { + return analReq, err + } else { + defer response.Body.Close() + + if api.IsErrorResponse(response) { + return analReq, ReadAPIError(response) + } + + return analReq, api.ReadAPIV2ResponsePayload(&analReq, response) + } +} diff --git a/cmd/api/src/api/v2/database_wipe.go b/cmd/api/src/api/v2/database_wipe.go index ba4040ce6..e3769fc98 100644 --- a/cmd/api/src/api/v2/database_wipe.go +++ b/cmd/api/src/api/v2/database_wipe.go @@ -24,6 +24,8 @@ import ( "github.com/specterops/bloodhound/log" "github.com/specterops/bloodhound/src/api" + "github.com/specterops/bloodhound/src/auth" + "github.com/specterops/bloodhound/src/ctx" "github.com/specterops/bloodhound/src/model" "github.com/specterops/bloodhound/src/model/appcfg" ) @@ -108,7 +110,18 @@ func (s Resources) HandleDatabaseWipe(response http.ResponseWriter, request *htt ) return } else { - s.TaskNotifier.RequestDeletion() + var userId string + if user, isUser := auth.GetUserFromAuthCtx(ctx.FromRequest(request).AuthCtx); !isUser { + log.Warnf("encountered request analysis for unknown user, this shouldn't happen") + userId = "unknown-user-database-wipe" + } else { + userId = user.ID.String() + } + + if err := s.DB.RequestCollectedGraphDataDeletion(request.Context(), userId); err != nil { + api.HandleDatabaseError(request, response, err) + return + } s.handleAuditLogForDatabaseWipe(request.Context(), &auditEntry, true, "collected graph data") } @@ -125,7 +138,18 @@ func (s Resources) HandleDatabaseWipe(response http.ResponseWriter, request *htt // if deleting `nodes` or deleting `asset group selectors` is successful, kickoff an analysis if kickoffAnalysis { - s.TaskNotifier.RequestAnalysis() + var userId string + if user, isUser := auth.GetUserFromAuthCtx(ctx.FromRequest(request).AuthCtx); !isUser { + log.Warnf("encountered request analysis for unknown user, this shouldn't happen") + userId = "unknown-user-database-wipe" + } else { + userId = user.ID.String() + } + + if err := s.DB.RequestAnalysis(request.Context(), userId); err != nil { + api.HandleDatabaseError(request, response, err) + return + } } // delete file ingest history @@ -142,7 +166,7 @@ func (s Resources) HandleDatabaseWipe(response http.ResponseWriter, request *htt } } - // return a user friendly error message indicating what operations failed + // return a user-friendly error message indicating what operations failed if len(errors) > 0 { api.WriteErrorResponse( request.Context(), diff --git a/cmd/api/src/api/v2/database_wipe_test.go b/cmd/api/src/api/v2/database_wipe_test.go index 4cbf81210..848454dda 100644 --- a/cmd/api/src/api/v2/database_wipe_test.go +++ b/cmd/api/src/api/v2/database_wipe_test.go @@ -21,12 +21,12 @@ import ( "net/http" "testing" + "github.com/gofrs/uuid" graph_mocks "github.com/specterops/bloodhound/dawgs/graph/mocks" "github.com/specterops/bloodhound/headers" "github.com/specterops/bloodhound/mediatypes" v2 "github.com/specterops/bloodhound/src/api/v2" "github.com/specterops/bloodhound/src/api/v2/apitest" - taskerMocks "github.com/specterops/bloodhound/src/daemons/datapipe/mocks" dbMocks "github.com/specterops/bloodhound/src/database/mocks" "github.com/specterops/bloodhound/src/model/appcfg" "go.uber.org/mock/gomock" @@ -34,15 +34,19 @@ import ( func TestDatabaseWipe(t *testing.T) { var ( - mockCtrl = gomock.NewController(t) - mockDB = dbMocks.NewMockDatabase(mockCtrl) - mockTasker = taskerMocks.NewMockTasker(mockCtrl) - mockGraph = graph_mocks.NewMockDatabase(mockCtrl) - resources = v2.Resources{DB: mockDB, TaskNotifier: mockTasker, Graph: mockGraph} + mockCtrl = gomock.NewController(t) + mockDB = dbMocks.NewMockDatabase(mockCtrl) + mockGraph = graph_mocks.NewMockDatabase(mockCtrl) + resources = v2.Resources{DB: mockDB, Graph: mockGraph} + user = setupUser() + userCtx = setupUserCtx(user) ) defer mockCtrl.Finish() apitest.NewHarness(t, resources.HandleDatabaseWipe). + WithCommonRequest(func(input *apitest.Input) { + apitest.SetContext(input, userCtx) + }). Run([]apitest.Case{ { Name: "JSON Malformed", @@ -81,7 +85,7 @@ func TestDatabaseWipe(t *testing.T) { }, }, { - Name: "deletion of collected graph data kicks off tasker", + Name: "deletion of collected graph data kicks off analysis", Input: func(input *apitest.Input) { apitest.SetHeader(input, headers.ContentType.String(), mediatypes.ApplicationJson.String()) apitest.BodyStruct(input, v2.DatabaseWipe{DeleteCollectedGraphData: true}) @@ -91,11 +95,11 @@ func TestDatabaseWipe(t *testing.T) { Enabled: true, }, nil) - taskerIntent := mockTasker.EXPECT().RequestDeletion().Times(1) + successfulRequestDeletion := mockDB.EXPECT().RequestCollectedGraphDataDeletion(gomock.Any(), uuid.UUID{}.String()).Times(1) successfulAuditLogIntent := mockDB.EXPECT().AppendAuditLog(gomock.Any(), gomock.Any()).Return(nil).Times(1) successfulAuditLogWipe := mockDB.EXPECT().AppendAuditLog(gomock.Any(), gomock.Any()).Return(nil).Times(1) - gomock.InOrder(successfulAuditLogIntent, taskerIntent, successfulAuditLogWipe) + gomock.InOrder(successfulAuditLogIntent, successfulRequestDeletion, successfulAuditLogWipe) }, Test: func(output apitest.Output) { @@ -131,7 +135,7 @@ func TestDatabaseWipe(t *testing.T) { successfulAuditLogIntent := mockDB.EXPECT().AppendAuditLog(gomock.Any(), gomock.Any()).Return(nil).Times(1) successfulAssetGroupSelectorsDelete := mockDB.EXPECT().DeleteAssetGroupSelectorsForAssetGroups(gomock.Any(), gomock.Any()).Return(nil).Times(1) successfulAuditLogSuccess := mockDB.EXPECT().AppendAuditLog(gomock.Any(), gomock.Any()).Return(nil).Times(1) - successfulAnalysisKickoff := mockTasker.EXPECT().RequestAnalysis().Times(1) + successfulAnalysisKickoff := mockDB.EXPECT().RequestAnalysis(gomock.Any(), uuid.UUID{}.String()).Times(1) gomock.InOrder(successfulAuditLogIntent, successfulAssetGroupSelectorsDelete, successfulAuditLogSuccess, successfulAnalysisKickoff) @@ -197,7 +201,7 @@ func TestDatabaseWipe(t *testing.T) { }, }, { - Name: "succesful deletion of data quality history", + Name: "successful deletion of data quality history", Input: func(input *apitest.Input) { apitest.SetHeader(input, headers.ContentType.String(), mediatypes.ApplicationJson.String()) apitest.BodyStruct(input, v2.DatabaseWipe{DeleteDataQualityHistory: true}) @@ -255,16 +259,16 @@ func TestDatabaseWipe(t *testing.T) { mockDB.EXPECT().GetFlagByKey(gomock.Any(), gomock.Any()).Return(appcfg.FeatureFlag{ Enabled: true, }, nil) - taskerIntent := mockTasker.EXPECT().RequestDeletion().Times(1) + successfulDeletionRequest := mockDB.EXPECT().RequestCollectedGraphDataDeletion(gomock.Any(), uuid.UUID{}.String()).Times(1) nodesDeletedAuditLog := mockDB.EXPECT().AppendAuditLog(gomock.Any(), gomock.Any()).Return(nil).Times(1) - gomock.InOrder(successfulAuditLogIntent, taskerIntent, nodesDeletedAuditLog) + gomock.InOrder(successfulAuditLogIntent, successfulDeletionRequest, nodesDeletedAuditLog) // high value selector operations assetGroupSelectorsDelete := mockDB.EXPECT().DeleteAssetGroupSelectorsForAssetGroups(gomock.Any(), gomock.Any()).Return(nil).Times(1) assetGroupSelectorsAuditLog := mockDB.EXPECT().AppendAuditLog(gomock.Any(), gomock.Any()).Return(nil).Times(1) // analysis kickoff - analysisKickoff := mockTasker.EXPECT().RequestAnalysis().Times(1) + analysisKickoff := mockDB.EXPECT().RequestAnalysis(gomock.Any(), uuid.UUID{}.String()).Times(1) gomock.InOrder(assetGroupSelectorsDelete, assetGroupSelectorsAuditLog, analysisKickoff) diff --git a/cmd/api/src/api/v2/datapipe.go b/cmd/api/src/api/v2/datapipe.go index 9ffce3b41..779266b23 100644 --- a/cmd/api/src/api/v2/datapipe.go +++ b/cmd/api/src/api/v2/datapipe.go @@ -20,17 +20,8 @@ import ( "net/http" "github.com/specterops/bloodhound/src/api" - "github.com/specterops/bloodhound/log" ) func (s Resources) GetDatapipeStatus(response http.ResponseWriter, request *http.Request) { api.WriteBasicResponse(request.Context(), s.TaskNotifier.GetStatus(), http.StatusOK, response) } - -func (s Resources) RequestAnalysis(response http.ResponseWriter, _ *http.Request) { - defer log.Measure(log.LevelDebug, "Requesting analysis")() - - s.TaskNotifier.RequestAnalysis() - - response.WriteHeader(http.StatusAccepted) -} diff --git a/cmd/api/src/api/v2/file_uploads_test.go b/cmd/api/src/api/v2/file_uploads_test.go index 77ede7360..0c5e915ab 100644 --- a/cmd/api/src/api/v2/file_uploads_test.go +++ b/cmd/api/src/api/v2/file_uploads_test.go @@ -29,7 +29,6 @@ import ( "github.com/specterops/bloodhound/src/api/v2/apitest" "github.com/specterops/bloodhound/src/auth" "github.com/specterops/bloodhound/src/ctx" - taskerMocks "github.com/specterops/bloodhound/src/daemons/datapipe/mocks" dbMocks "github.com/specterops/bloodhound/src/database/mocks" "github.com/specterops/bloodhound/src/database/types/null" "github.com/specterops/bloodhound/src/model" @@ -58,10 +57,9 @@ func setupUserCtx(user model.User) context.Context { func TestResources_ListFileUploadJobs(t *testing.T) { var ( - mockCtrl = gomock.NewController(t) - mockDB = dbMocks.NewMockDatabase(mockCtrl) - mockTasker = taskerMocks.NewMockTasker(mockCtrl) - resources = v2.Resources{DB: mockDB, TaskNotifier: mockTasker} + mockCtrl = gomock.NewController(t) + mockDB = dbMocks.NewMockDatabase(mockCtrl) + resources = v2.Resources{DB: mockDB} ) defer mockCtrl.Finish() @@ -147,10 +145,9 @@ func TestResources_StartFileUploadJob(t *testing.T) { func TestResources_EndFileUploadJob(t *testing.T) { var ( - mockCtrl = gomock.NewController(t) - mockDB = dbMocks.NewMockDatabase(mockCtrl) - mockTasker = taskerMocks.NewMockTasker(mockCtrl) - resources = v2.Resources{DB: mockDB, TaskNotifier: mockTasker} + mockCtrl = gomock.NewController(t) + mockDB = dbMocks.NewMockDatabase(mockCtrl) + resources = v2.Resources{DB: mockDB} ) defer mockCtrl.Finish() diff --git a/cmd/api/src/api/v2/flag.go b/cmd/api/src/api/v2/flag.go index c02550789..c07077978 100644 --- a/cmd/api/src/api/v2/flag.go +++ b/cmd/api/src/api/v2/flag.go @@ -22,7 +22,10 @@ import ( "strconv" "github.com/gorilla/mux" + "github.com/specterops/bloodhound/log" "github.com/specterops/bloodhound/src/api" + "github.com/specterops/bloodhound/src/auth" + "github.com/specterops/bloodhound/src/ctx" "github.com/specterops/bloodhound/src/model/appcfg" ) @@ -59,7 +62,18 @@ func (s Resources) ToggleFlag(response http.ResponseWriter, request *http.Reques } else { // TODO: Cleanup #ADCSFeatureFlag after full launch. if featureFlag.Key == appcfg.FeatureAdcs && !featureFlag.Enabled { - s.TaskNotifier.RequestAnalysis() + var userId string + if user, isUser := auth.GetUserFromAuthCtx(ctx.FromRequest(request).AuthCtx); !isUser { + log.Warnf("encountered request analysis for unknown user, this shouldn't happen") + userId = "unknown-user-toggle-flag" + } else { + userId = user.ID.String() + } + + if err := s.DB.RequestAnalysis(request.Context(), userId); err != nil { + api.HandleDatabaseError(request, response, err) + return + } } api.WriteBasicResponse(request.Context(), ToggleFlagResponse{ Enabled: featureFlag.Enabled, diff --git a/cmd/api/src/daemons/datapipe/datapipe.go b/cmd/api/src/daemons/datapipe/datapipe.go index 986f3cd93..25705558b 100644 --- a/cmd/api/src/daemons/datapipe/datapipe.go +++ b/cmd/api/src/daemons/datapipe/datapipe.go @@ -20,7 +20,6 @@ package datapipe import ( "context" "errors" - "sync/atomic" "time" "github.com/specterops/bloodhound/cache" @@ -39,8 +38,6 @@ const ( ) type Tasker interface { - RequestAnalysis() - RequestDeletion() GetStatus() model.DatapipeStatusWrapper } @@ -49,8 +46,6 @@ type Daemon struct { graphdb graph.Database cache cache.Cache cfg config.Configuration - analysisRequested *atomic.Bool - deletionRequested *atomic.Bool tickInterval time.Duration status model.DatapipeStatusWrapper ctx context.Context @@ -68,8 +63,6 @@ func NewDaemon(ctx context.Context, cfg config.Configuration, connections bootst cache: cache, cfg: cfg, ctx: ctx, - deletionRequested: &atomic.Bool{}, - analysisRequested: &atomic.Bool{}, orphanedFileSweeper: NewOrphanFileSweeper(NewOSFileOperations(), cfg.TempDirectory()), tickInterval: tickInterval, status: model.DatapipeStatusWrapper{ @@ -79,43 +72,17 @@ func NewDaemon(ctx context.Context, cfg config.Configuration, connections bootst } } -func (s *Daemon) RequestAnalysis() { - if s.getDeletionRequested() { - log.Warnf("Rejecting analysis request as deletion is in progress") - return - } - s.setAnalysisRequested(true) -} - -func (s *Daemon) RequestDeletion() { - s.setAnalysisRequested(false) - s.setDeletionRequested(true) -} - func (s *Daemon) GetStatus() model.DatapipeStatusWrapper { return s.status } -func (s *Daemon) getAnalysisRequested() bool { - return s.analysisRequested.Load() -} - -func (s *Daemon) setAnalysisRequested(requested bool) { - s.analysisRequested.Store(requested) -} - -func (s *Daemon) setDeletionRequested(requested bool) { - s.deletionRequested.Store(requested) -} - -func (s *Daemon) getDeletionRequested() bool { - return s.deletionRequested.Load() -} - func (s *Daemon) analyze() { - // Ensure that the user-requested analysis switch is flipped back to false. This is done at the beginning of the + // Ensure that the user-requested analysis switch is deleted. This is done at the beginning of the // function so that any re-analysis requests are caught while analysis is in-progress. - s.setAnalysisRequested(false) + if err := s.db.DeleteAnalysisRequest(s.ctx); err != nil { + log.Errorf("Error deleting analysis request: %v", err) + return + } if s.cfg.DisableAnalysis { return @@ -178,7 +145,7 @@ func (s *Daemon) Start(ctx context.Context) { s.clearOrphanedData() case <-datapipeLoopTimer.C: - if s.getDeletionRequested() { + if s.db.HasCollectedGraphDataDeletionRequest(s.ctx) { s.deleteData() } @@ -194,7 +161,7 @@ func (s *Daemon) Start(ctx context.Context) { // If there are completed file upload jobs or if analysis was user-requested, perform analysis. if hasJobsWaitingForAnalysis, err := HasFileUploadJobsWaitingForAnalysis(s.ctx, s.db); err != nil { log.Errorf("Failed looking up jobs waiting for analysis: %v", err) - } else if hasJobsWaitingForAnalysis || s.getAnalysisRequested() { + } else if hasJobsWaitingForAnalysis || s.db.HasAnalysisRequest(s.ctx) { s.analyze() } @@ -209,8 +176,8 @@ func (s *Daemon) Start(ctx context.Context) { func (s *Daemon) deleteData() { defer func() { s.status.Update(model.DatapipeStatusIdle, false) - s.setDeletionRequested(false) - s.setAnalysisRequested(true) + _ = s.db.DeleteAnalysisRequest(s.ctx) + _ = s.db.RequestAnalysis(s.ctx, "datapie") }() defer log.Measure(log.LevelInfo, "Purge Graph Data Completed")() s.status.Update(model.DatapipeStatusPurging, false) diff --git a/cmd/api/src/daemons/datapipe/mocks/tasker.go b/cmd/api/src/daemons/datapipe/mocks/tasker.go index 5414e7689..371ee565e 100644 --- a/cmd/api/src/daemons/datapipe/mocks/tasker.go +++ b/cmd/api/src/daemons/datapipe/mocks/tasker.go @@ -63,27 +63,3 @@ func (mr *MockTaskerMockRecorder) GetStatus() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatus", reflect.TypeOf((*MockTasker)(nil).GetStatus)) } - -// RequestAnalysis mocks base method. -func (m *MockTasker) RequestAnalysis() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RequestAnalysis") -} - -// RequestAnalysis indicates an expected call of RequestAnalysis. -func (mr *MockTaskerMockRecorder) RequestAnalysis() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestAnalysis", reflect.TypeOf((*MockTasker)(nil).RequestAnalysis)) -} - -// RequestDeletion mocks base method. -func (m *MockTasker) RequestDeletion() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RequestDeletion") -} - -// RequestDeletion indicates an expected call of RequestDeletion. -func (mr *MockTaskerMockRecorder) RequestDeletion() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestDeletion", reflect.TypeOf((*MockTasker)(nil).RequestDeletion)) -} diff --git a/cmd/api/src/database/analysisrequest.go b/cmd/api/src/database/analysisrequest.go new file mode 100644 index 000000000..6d2e5fbf4 --- /dev/null +++ b/cmd/api/src/database/analysisrequest.go @@ -0,0 +1,107 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package database + +import ( + "context" + "database/sql" + "time" + + "github.com/specterops/bloodhound/errors" + "github.com/specterops/bloodhound/log" + "github.com/specterops/bloodhound/src/model" +) + +type AnalysisRequestData interface { + DeleteAnalysisRequest(ctx context.Context) error + GetAnalysisRequest(ctx context.Context) (model.AnalysisRequest, error) + HasAnalysisRequest(ctx context.Context) bool + HasCollectedGraphDataDeletionRequest(ctx context.Context) bool + RequestAnalysis(ctx context.Context, requester string) error + RequestCollectedGraphDataDeletion(ctx context.Context, requester string) error +} + +func (s *BloodhoundDB) DeleteAnalysisRequest(ctx context.Context) error { + tx := s.db.WithContext(ctx).Exec(`truncate analysis_request_switch;`) + return tx.Error +} + +func (s *BloodhoundDB) GetAnalysisRequest(ctx context.Context) (model.AnalysisRequest, error) { + var analysisRequest model.AnalysisRequest + + // Note: GORM Raw does not throw any errors if no row is found. We can inspect rows affected as a workaround + if tx := s.db.WithContext(ctx).Raw(`select requested_by, request_type, requested_at from analysis_request_switch limit 1;`).Scan(&analysisRequest); tx.RowsAffected == 0 { + return analysisRequest, sql.ErrNoRows + } + + return analysisRequest, nil +} + +func (s *BloodhoundDB) HasAnalysisRequest(ctx context.Context) bool { + var exists bool + + tx := s.db.WithContext(ctx).Raw(`select exists(select * from analysis_request_switch where request_type = ? limit 1);`, model.AnalysisRequestAnalysis).Scan(&exists) + if tx.Error != nil { + log.Errorf("Error determining if there's an analysis request: %v", tx.Error) + } + return exists +} + +func (s *BloodhoundDB) HasCollectedGraphDataDeletionRequest(ctx context.Context) bool { + var exists bool + + tx := s.db.WithContext(ctx).Raw(`select exists(select * from analysis_request_switch where request_type = ? limit 1);`, model.AnalysisRequestDeletion).Scan(&exists) + if tx.Error != nil { + log.Errorf("Error determining if there's a deletion request: %v", tx.Error) + } + return exists +} + +// This inserts a row into analysis_request_switch for both a collected graph data deletion request or an analysis request. +// There should only ever be 1 row, if a request is present, subsequent requests no-op +// If an analysis request is present when a deletion request comes in, that overwrites the analysis to deletion but not vice-versa +// To request: Use the helper methods `RequestAnalysis` and `RequestCollectedGraphDataDeletion` +func (s *BloodhoundDB) setAnalysisRequest(ctx context.Context, requestType model.AnalysisRequestType, requestedBy string) error { + if analReq, err := s.GetAnalysisRequest(ctx); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + // Analysis request doesn't exist so insert one + insertSql := `insert into analysis_request_switch (requested_by, request_type, requested_at) values (?, ?, ?);` + tx := s.db.WithContext(ctx).Exec(insertSql, requestedBy, requestType, time.Now().UTC()) + return tx.Error + } else { + // Analysis request existed, we only want to overwrite if request is for a deletion request, otherwise ignore additional requests + if analReq.RequestType == model.AnalysisRequestAnalysis && requestType == model.AnalysisRequestDeletion { + updateSql := `update analysis_request_switch set requested_by = ?, request_type = ?, requested_at = ? limit 1;` + tx := s.db.WithContext(ctx).Exec(updateSql, requestedBy, requestType, time.Now().UTC()) + return tx.Error + } + return nil + } +} + +// This will request an analysis be executed, as long as there isn't an existing analysis request or collected graph data deletion request, then it no-ops +func (s *BloodhoundDB) RequestAnalysis(ctx context.Context, requestedBy string) error { + log.Infof("Analysis requested by %s", requestedBy) + return s.setAnalysisRequest(ctx, model.AnalysisRequestAnalysis, requestedBy) +} + +// This will request collected graph data be deleted, if an analysis request is present, it will overwrite that. +func (s *BloodhoundDB) RequestCollectedGraphDataDeletion(ctx context.Context, requestedBy string) error { + log.Infof("Collected graph data deletion requested by %s", requestedBy) + return s.setAnalysisRequest(ctx, model.AnalysisRequestDeletion, requestedBy) +} diff --git a/cmd/api/src/database/analysisrequest_test.go b/cmd/api/src/database/analysisrequest_test.go new file mode 100644 index 000000000..5c8c7a4ca --- /dev/null +++ b/cmd/api/src/database/analysisrequest_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package database_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/specterops/bloodhound/src/model" + "github.com/specterops/bloodhound/src/test/integration" + "github.com/stretchr/testify/require" +) + +func TestAnalysisRequest(t *testing.T) { + var ( + testCtx = context.Background() + dbInst = integration.SetupDB(t) + ) + + err := dbInst.RequestAnalysis(testCtx, "test") + require.Nil(t, err) + + analReq, err := dbInst.GetAnalysisRequest(testCtx) + require.Nil(t, err) + require.Equal(t, analReq.RequestType, model.AnalysisRequestAnalysis) + require.Equal(t, analReq.RequestedBy, "test") + require.False(t, analReq.RequestedAt.IsZero()) + + err = dbInst.DeleteAnalysisRequest(testCtx) + require.Nil(t, err) + + _, err = dbInst.GetAnalysisRequest(testCtx) + require.ErrorIs(t, err, sql.ErrNoRows) +} diff --git a/cmd/api/src/database/db.go b/cmd/api/src/database/db.go index e78db3f98..f22aebad6 100644 --- a/cmd/api/src/database/db.go +++ b/cmd/api/src/database/db.go @@ -151,6 +151,9 @@ type Database interface { DeleteSavedQuery(ctx context.Context, id int) error SavedQueryBelongsToUser(ctx context.Context, userID uuid.UUID, savedQueryID int) (bool, error) DeleteAssetGroupSelectorsForAssetGroups(ctx context.Context, assetGroupIds []int) error + + // Analysis Request + AnalysisRequestData } type BloodhoundDB struct { diff --git a/cmd/api/src/database/migration/migrations/v5.11.0.sql b/cmd/api/src/database/migration/migrations/v5.11.0.sql new file mode 100644 index 000000000..68fcd69dd --- /dev/null +++ b/cmd/api/src/database/migration/migrations/v5.11.0.sql @@ -0,0 +1,23 @@ +-- Copyright 2024 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE IF NOT EXISTS analysis_request_switch ( + singleton bool PRIMARY KEY DEFAULT true, + request_type text NOT NULL, + requested_by text NOT NULL, + requested_at timestamp with time zone NOT NULL + CONSTRAINT singleton_uni CHECK (singleton) +); diff --git a/cmd/api/src/database/mocks/db.go b/cmd/api/src/database/mocks/db.go index 1e14d28a1..896a3fe6f 100644 --- a/cmd/api/src/database/mocks/db.go +++ b/cmd/api/src/database/mocks/db.go @@ -374,6 +374,20 @@ func (mr *MockDatabaseMockRecorder) DeleteAllIngestTasks(arg0 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllIngestTasks", reflect.TypeOf((*MockDatabase)(nil).DeleteAllIngestTasks), arg0) } +// DeleteAnalysisRequest mocks base method. +func (m *MockDatabase) DeleteAnalysisRequest(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAnalysisRequest", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAnalysisRequest indicates an expected call of DeleteAnalysisRequest. +func (mr *MockDatabaseMockRecorder) DeleteAnalysisRequest(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAnalysisRequest", reflect.TypeOf((*MockDatabase)(nil).DeleteAnalysisRequest), arg0) +} + // DeleteAssetGroup mocks base method. func (m *MockDatabase) DeleteAssetGroup(arg0 context.Context, arg1 model.AssetGroup) error { m.ctrl.T.Helper() @@ -695,6 +709,21 @@ func (mr *MockDatabaseMockRecorder) GetAllUsers(arg0, arg1, arg2 interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUsers", reflect.TypeOf((*MockDatabase)(nil).GetAllUsers), arg0, arg1, arg2) } +// GetAnalysisRequest mocks base method. +func (m *MockDatabase) GetAnalysisRequest(arg0 context.Context) (model.AnalysisRequest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAnalysisRequest", arg0) + ret0, _ := ret[0].(model.AnalysisRequest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAnalysisRequest indicates an expected call of GetAnalysisRequest. +func (mr *MockDatabaseMockRecorder) GetAnalysisRequest(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAnalysisRequest", reflect.TypeOf((*MockDatabase)(nil).GetAnalysisRequest), arg0) +} + // GetAssetGroup mocks base method. func (m *MockDatabase) GetAssetGroup(arg0 context.Context, arg1 int32) (model.AssetGroup, error) { m.ctrl.T.Helper() @@ -1057,6 +1086,34 @@ func (mr *MockDatabaseMockRecorder) GetUserToken(arg0, arg1, arg2 interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserToken", reflect.TypeOf((*MockDatabase)(nil).GetUserToken), arg0, arg1, arg2) } +// HasAnalysisRequest mocks base method. +func (m *MockDatabase) HasAnalysisRequest(arg0 context.Context) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasAnalysisRequest", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// HasAnalysisRequest indicates an expected call of HasAnalysisRequest. +func (mr *MockDatabaseMockRecorder) HasAnalysisRequest(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasAnalysisRequest", reflect.TypeOf((*MockDatabase)(nil).HasAnalysisRequest), arg0) +} + +// HasCollectedGraphDataDeletionRequest mocks base method. +func (m *MockDatabase) HasCollectedGraphDataDeletionRequest(arg0 context.Context) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasCollectedGraphDataDeletionRequest", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// HasCollectedGraphDataDeletionRequest indicates an expected call of HasCollectedGraphDataDeletionRequest. +func (mr *MockDatabaseMockRecorder) HasCollectedGraphDataDeletionRequest(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasCollectedGraphDataDeletionRequest", reflect.TypeOf((*MockDatabase)(nil).HasCollectedGraphDataDeletionRequest), arg0) +} + // HasInstallation mocks base method. func (m *MockDatabase) HasInstallation(arg0 context.Context) (bool, error) { m.ctrl.T.Helper() @@ -1178,6 +1235,34 @@ func (mr *MockDatabaseMockRecorder) Migrate(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Migrate", reflect.TypeOf((*MockDatabase)(nil).Migrate), arg0) } +// RequestAnalysis mocks base method. +func (m *MockDatabase) RequestAnalysis(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RequestAnalysis", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RequestAnalysis indicates an expected call of RequestAnalysis. +func (mr *MockDatabaseMockRecorder) RequestAnalysis(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestAnalysis", reflect.TypeOf((*MockDatabase)(nil).RequestAnalysis), arg0, arg1) +} + +// RequestCollectedGraphDataDeletion mocks base method. +func (m *MockDatabase) RequestCollectedGraphDataDeletion(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RequestCollectedGraphDataDeletion", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RequestCollectedGraphDataDeletion indicates an expected call of RequestCollectedGraphDataDeletion. +func (mr *MockDatabaseMockRecorder) RequestCollectedGraphDataDeletion(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestCollectedGraphDataDeletion", reflect.TypeOf((*MockDatabase)(nil).RequestCollectedGraphDataDeletion), arg0, arg1) +} + // RequiresMigration mocks base method. func (m *MockDatabase) RequiresMigration(arg0 context.Context) (bool, error) { m.ctrl.T.Helper() diff --git a/cmd/api/src/model/analysisrequest.go b/cmd/api/src/model/analysisrequest.go new file mode 100644 index 000000000..87f8cc3fc --- /dev/null +++ b/cmd/api/src/model/analysisrequest.go @@ -0,0 +1,32 @@ +// Copyright 2024 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package model + +import "time" + +type AnalysisRequestType string + +const ( + AnalysisRequestAnalysis AnalysisRequestType = "analysis" + AnalysisRequestDeletion AnalysisRequestType = "deletion" +) + +type AnalysisRequest struct { + RequestedBy string `json:"requested_by"` + RequestType AnalysisRequestType `json:"request_type"` + RequestedAt time.Time `json:"requested_at"` +} diff --git a/cmd/api/src/services/entrypoint.go b/cmd/api/src/services/entrypoint.go index 4b280c620..4893ac20a 100644 --- a/cmd/api/src/services/entrypoint.go +++ b/cmd/api/src/services/entrypoint.go @@ -105,7 +105,9 @@ func Entrypoint(ctx context.Context, cfg config.Configuration, connections boots connections.Graph.SetWriteFlushSize(neo4jParameters.WriteFlushSize) // Trigger analysis on first start - datapipeDaemon.RequestAnalysis() + if err := connections.RDMS.RequestAnalysis(ctx, "init"); err != nil { + return nil, fmt.Errorf("failed to request analysis: %w", err) + } return []daemons.Daemon{ bhapi.NewDaemon(cfg, routerInst.Handler()),