diff --git a/vmlifecycle/runner/runner.go b/vmlifecycle/runner/runner.go index e9e749874..a2ad8effc 100644 --- a/vmlifecycle/runner/runner.go +++ b/vmlifecycle/runner/runner.go @@ -2,6 +2,7 @@ package runner import ( "bytes" + "context" "fmt" "github.com/fatih/color" "github.com/onsi/gomega/gexec" @@ -35,6 +36,10 @@ func (r *Runner) Execute(args []interface{}) (*bytes.Buffer, *bytes.Buffer, erro } func (r *Runner) ExecuteWithEnvVars(env []string, args []interface{}) (*bytes.Buffer, *bytes.Buffer, error) { + return r.ExecuteWithEnvVarsCtx(context.Background(), env, args) +} + +func (r *Runner) ExecuteWithEnvVarsCtx(ctx context.Context, env []string, args []interface{}) (*bytes.Buffer, *bytes.Buffer, error) { var outBufWriter bytes.Buffer var errBufWriter bytes.Buffer @@ -53,7 +58,7 @@ func (r *Runner) ExecuteWithEnvVars(env []string, args []interface{}) (*bytes.Bu } } - command := exec.Command(r.command, stringArgs...) + command := exec.CommandContext(ctx, r.command, stringArgs...) if len(env) > 0 { command.Env = append(os.Environ(), env...) } diff --git a/vmlifecycle/vmmanagers/fakes/govcRunner.go b/vmlifecycle/vmmanagers/fakes/govcRunner.go index 5f6486d63..6017cc416 100644 --- a/vmlifecycle/vmmanagers/fakes/govcRunner.go +++ b/vmlifecycle/vmmanagers/fakes/govcRunner.go @@ -3,6 +3,7 @@ package fakes import ( "bytes" + "context" "sync" ) @@ -23,6 +24,23 @@ type GovcRunner struct { result2 *bytes.Buffer result3 error } + ExecuteWithEnvVarsCtxStub func(context.Context, []string, []interface{}) (*bytes.Buffer, *bytes.Buffer, error) + executeWithEnvVarsCtxMutex sync.RWMutex + executeWithEnvVarsCtxArgsForCall []struct { + arg1 context.Context + arg2 []string + arg3 []interface{} + } + executeWithEnvVarsCtxReturns struct { + result1 *bytes.Buffer + result2 *bytes.Buffer + result3 error + } + executeWithEnvVarsCtxReturnsOnCall map[int]struct { + result1 *bytes.Buffer + result2 *bytes.Buffer + result3 error + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -104,11 +122,91 @@ func (fake *GovcRunner) ExecuteWithEnvVarsReturnsOnCall(i int, result1 *bytes.Bu }{result1, result2, result3} } +func (fake *GovcRunner) ExecuteWithEnvVarsCtx(arg1 context.Context, arg2 []string, arg3 []interface{}) (*bytes.Buffer, *bytes.Buffer, error) { + var arg2Copy []string + if arg2 != nil { + arg2Copy = make([]string, len(arg2)) + copy(arg2Copy, arg2) + } + var arg3Copy []interface{} + if arg3 != nil { + arg3Copy = make([]interface{}, len(arg3)) + copy(arg3Copy, arg3) + } + fake.executeWithEnvVarsCtxMutex.Lock() + ret, specificReturn := fake.executeWithEnvVarsCtxReturnsOnCall[len(fake.executeWithEnvVarsCtxArgsForCall)] + fake.executeWithEnvVarsCtxArgsForCall = append(fake.executeWithEnvVarsCtxArgsForCall, struct { + arg1 context.Context + arg2 []string + arg3 []interface{} + }{arg1, arg2Copy, arg3Copy}) + fake.recordInvocation("ExecuteWithEnvVarsCtx", []interface{}{arg1, arg2Copy, arg3Copy}) + fake.executeWithEnvVarsCtxMutex.Unlock() + if fake.ExecuteWithEnvVarsCtxStub != nil { + return fake.ExecuteWithEnvVarsCtxStub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.executeWithEnvVarsCtxReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *GovcRunner) ExecuteWithEnvVarsCtxCallCount() int { + fake.executeWithEnvVarsCtxMutex.RLock() + defer fake.executeWithEnvVarsCtxMutex.RUnlock() + return len(fake.executeWithEnvVarsCtxArgsForCall) +} + +func (fake *GovcRunner) ExecuteWithEnvVarsCtxCalls(stub func(context.Context, []string, []interface{}) (*bytes.Buffer, *bytes.Buffer, error)) { + fake.executeWithEnvVarsCtxMutex.Lock() + defer fake.executeWithEnvVarsCtxMutex.Unlock() + fake.ExecuteWithEnvVarsCtxStub = stub +} + +func (fake *GovcRunner) ExecuteWithEnvVarsCtxArgsForCall(i int) (context.Context, []string, []interface{}) { + fake.executeWithEnvVarsCtxMutex.RLock() + defer fake.executeWithEnvVarsCtxMutex.RUnlock() + argsForCall := fake.executeWithEnvVarsCtxArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *GovcRunner) ExecuteWithEnvVarsCtxReturns(result1 *bytes.Buffer, result2 *bytes.Buffer, result3 error) { + fake.executeWithEnvVarsCtxMutex.Lock() + defer fake.executeWithEnvVarsCtxMutex.Unlock() + fake.ExecuteWithEnvVarsCtxStub = nil + fake.executeWithEnvVarsCtxReturns = struct { + result1 *bytes.Buffer + result2 *bytes.Buffer + result3 error + }{result1, result2, result3} +} + +func (fake *GovcRunner) ExecuteWithEnvVarsCtxReturnsOnCall(i int, result1 *bytes.Buffer, result2 *bytes.Buffer, result3 error) { + fake.executeWithEnvVarsCtxMutex.Lock() + defer fake.executeWithEnvVarsCtxMutex.Unlock() + fake.ExecuteWithEnvVarsCtxStub = nil + if fake.executeWithEnvVarsCtxReturnsOnCall == nil { + fake.executeWithEnvVarsCtxReturnsOnCall = make(map[int]struct { + result1 *bytes.Buffer + result2 *bytes.Buffer + result3 error + }) + } + fake.executeWithEnvVarsCtxReturnsOnCall[i] = struct { + result1 *bytes.Buffer + result2 *bytes.Buffer + result3 error + }{result1, result2, result3} +} + func (fake *GovcRunner) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() fake.executeWithEnvVarsMutex.RLock() defer fake.executeWithEnvVarsMutex.RUnlock() + fake.executeWithEnvVarsCtxMutex.RLock() + defer fake.executeWithEnvVarsCtxMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/vmlifecycle/vmmanagers/vsphere.go b/vmlifecycle/vmmanagers/vsphere.go index a5fff270f..49127615e 100644 --- a/vmlifecycle/vmmanagers/vsphere.go +++ b/vmlifecycle/vmmanagers/vsphere.go @@ -3,17 +3,18 @@ package vmmanagers import ( "archive/tar" "bytes" + "context" "encoding/json" "errors" "fmt" + "github.com/blang/semver" + "github.com/pivotal-cf/om/vmlifecycle/extractopsmansemver" "io/ioutil" "log" "os" "strconv" "strings" - - "github.com/blang/semver" - "github.com/pivotal-cf/om/vmlifecycle/extractopsmansemver" + "time" ) type VcenterCredential struct { @@ -74,6 +75,7 @@ type networkMapping struct { //go:generate counterfeiter -o ./fakes/govcRunner.go --fake-name GovcRunner . govcRunner type govcRunner interface { ExecuteWithEnvVars(env []string, args []interface{}) (*bytes.Buffer, *bytes.Buffer, error) + ExecuteWithEnvVarsCtx(ctx context.Context, env []string, args []interface{}) (*bytes.Buffer, *bytes.Buffer, error) } type VsphereVMManager struct { @@ -166,7 +168,7 @@ func (v *VsphereVMManager) CreateVM() (Status, StateInfo, error) { ipath := v.createIpath() - errBufWriter, err := v.createVM(env, optionFilename) + errBufWriter, err := v.createVM(env, optionFilename, ipath) fullState := StateInfo{IAAS: "vsphere", ID: ipath} if err != nil { @@ -319,14 +321,47 @@ func (v *VsphereVMManager) validateImage() error { } } -func (v *VsphereVMManager) createVM(env []string, optionFilename string) (errorBuffer *bytes.Buffer, err error) { - _, errBufWriter, err := v.runner.ExecuteWithEnvVars(env, []interface{}{ +func (v *VsphereVMManager) createVM(env []string, optionFilename string, ipath string) (errBufWriter *bytes.Buffer, err error) { + _, errBufWriter, err = v.runner.ExecuteWithEnvVars(env, []interface{}{ "import.ova", "-options=" + optionFilename, v.ImageOVA, }) + if err != nil { + return errBufWriter, checkFormatedError("govc error: %s", err) + } - return errBufWriter, checkFormatedError("govc error: %s", err) + ctx, cancel := context.WithTimeout(context.Background(), 80*time.Second) // 80 seconds is adequate time for OM to get IP; typically it's 43 seconds + defer cancel() + // Wait 80 seconds for VM to boot and acquire its IP + _, errBufWriter, err = v.runner.ExecuteWithEnvVarsCtx(ctx, env, []interface{}{ + "vm.info", + fmt.Sprintf(`-vm.ipath=%s`, ipath), + "-waitip", + }) + if ctx.Err() != nil { + // VM hasn't acquired IP, is likely stuck, reset VM to free it (to boot) + buf, errPowerReset := v.resetVM(env, ipath) + if errPowerReset != nil { + // we don't need to return errBuffWriter because we already know it's nil + // because the ExecuteWithEnvVarsCtx that sets it never completes + return buf, fmt.Errorf("govc error: could not power-reset: %s", errPowerReset) + } + } else { + if err != nil { + return errBufWriter, checkFormatedError("govc error: %s", err) + } + } + return errBufWriter, nil +} + +func (v *VsphereVMManager) resetVM(env []string, ipath string) (errBufWriter *bytes.Buffer, err error) { + _, errBufWriter, err = v.runner.ExecuteWithEnvVars(env, []interface{}{ + "vm.power", + fmt.Sprintf(`-vm.ipath=%s`, ipath), + "-reset", + }) + return errBufWriter, err } func (v *VsphereVMManager) addDefaultConfigFields() { diff --git a/vmlifecycle/vmmanagers/vsphere_test.go b/vmlifecycle/vmmanagers/vsphere_test.go index e2913df34..1935f1747 100644 --- a/vmlifecycle/vmmanagers/vsphere_test.go +++ b/vmlifecycle/vmmanagers/vsphere_test.go @@ -1,12 +1,11 @@ package vmmanagers_test import ( + "archive/tar" "fmt" "io/ioutil" "os" - "archive/tar" - "bytes" "errors" "io" @@ -111,6 +110,14 @@ opsman-configuration: "-on=true", "-vm.ipath=/datacenter/vm/folder/vm_name", )) + + _, _, args = runner.ExecuteWithEnvVarsCtxArgsForCall(0) + Expect(args).To(matchers.OrderedConsistOf( + "vm.info", + "-vm.ipath=/datacenter/vm/folder/vm_name", + "-waitip", + )) + Expect(runner.ExecuteWithEnvVarsCtxCallCount()).To(Equal(1)) }) When("setting custom cpu and memory", func() {