Merge pull request #301 from petabridge/dev

v1.7.1 Release

README.md

@@ -15,17 +15,33 @@ If you do need to make an update to Lighthouse, here are some cases where that m
 The easiest way to run Lighthouse is via [Petabridge's official Lighthouse Docker images on Docker Hub](https://hub.docker.com/r/petabridge/lighthouse):
 
 
-**Linux Images**
+### Linux Images
+
+#### AMD64
+
+```
+docker pull petabridge/lighthouse:latest
+```
+
+or
+
 ```
 docker pull petabridge/lighthouse:linux-latest
 ```
 
-**Windows Images**
+#### ARM64
+
+```
+docker pull petabridge/lighthouse:arm64-latest
+```
+
+### Windows Images
+
 ```
 docker pull petabridge/lighthouse:windows-latest
 ```
 
-All of these images run lighthouse on top of .NET Core 2.1 and expose the Akka.Cluster TCP endpoint on port 4053 by default. These images also come with [`Petabridge.Cmd.Host` installed](https://cmd.petabridge.com/articles/install/host-configuration.html) and exposed on TCP port 9110.
+All of these images run lighthouse on top of .NET 6 and expose the Akka.Cluster TCP endpoint on port 4053 by default. These images also come with [`Petabridge.Cmd.Host` installed](https://cmd.petabridge.com/articles/install/host-configuration.html) and exposed on TCP port 9110.
 
 > Linux images also come with [the `pbm` client](https://cmd.petabridge.com/articles/install/index.html) installed as a global .NET Core tool, so you can remotely execute `pbm` commands inside the containers themselves without exposing `Petabridge.Cmd.Host` over the network. 
 >

RELEASE_NOTES.md

@@ -1,4 +1,5 @@
-#### 1.6.2 June 27 2022 ####
+#### 1.7.1 October 24 2022 ####
 
-* [Bumped Akka.NET version to 1.4.39](https://github.com/akkadotnet/akka.net/releases/tag/1.4.39)
-* [Bumped Akka.Hosting from 0.2.2 to 0.3.4](https://github.com/petabridge/lighthouse/pull/266).
+* [Bumped Akka.NET version to 1.4.45](https://github.com/akkadotnet/akka.net/releases/tag/1.4.45)
+* [Bumped Akka.Hosting to 0.5.1](https://github.com/akkadotnet/Akka.Hosting/releases/tag/0.5.1).
+* [Upgraded to Petabridge.Cmd 1.1.3](https://cmd.petabridge.com/articles/RELEASE_NOTES.html) and used Akka.Hosting support;

build-system/azure-pipeline.template.yaml

@@ -16,7 +16,15 @@ jobs:
         clean: false  # whether to fetch clean each time
         submodules: recursive  # set to 'true' for a single level of submodules or 'recursive' to get submodules of submodules
         persistCredentials: true
+      - task: UseDotNet@2
+        displayName: 'Use .NET 6 SDK 6.0.100'
+        inputs:
+          version: 6.0.100
       # Linux or macOS
+      - script: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+        displayName: Docker - Register QEMU
+        continueOnError: true
+        condition: in( variables['Agent.OS'], 'Linux', 'Darwin' )
       - task: Bash@3 
         displayName: Linux / OSX Build
         inputs:
@@ -38,22 +46,6 @@ jobs:
           testResultsFiles: '**/*.trx' #TestResults folder usually
           testRunTitle: ${{ parameters.name }}
           mergeTestResults: true
-      #Docker build 
-      # Build on Linux
-      - task: Bash@3 
-        displayName: Docker Build (Linux)
-        inputs:
-          filePath: ${{ parameters.scriptFileName }}
-          arguments: ${{ parameters.dockerBuildArgs }}
-        continueOnError: false
-        condition: in( variables['Agent.OS'], 'Linux', 'Darwin' )
-      - task: BatchScript@1
-        displayName: Docker Build (Windows)
-        inputs:
-          filename: ${{ parameters.scriptFileName }}
-          arguments: ${{ parameters.dockerBuildArgs }}
-        continueOnError: true
-        condition: eq( variables['Agent.OS'], 'Windows_NT' )
       - script: 'echo 1>&2'
         failOnStderr: true
         displayName: 'If above is partially succeeded, then fail'

build-system/linux-pr-validation.yaml

@@ -20,3 +20,9 @@ jobs:
     vmImage: 'ubuntu-20.04'
     scriptFileName: ./build.sh
     scriptArgs: all
+- template: azure-pipeline.template.yaml
+  parameters:
+    name: LinuxRuntimesTests
+    vmImage: 'ubuntu-20.04'
+    scriptFileName: ./build.sh
+    scriptArgs: RunTestsOnRuntimes

build-system/windows-release.yaml

@@ -18,9 +18,13 @@ variables:
 jobs:
   - job: publishNetCore
     pool:
-      vmImage: windows-2019
+      vmImage: windows-latest
       demands: Cmd
     steps:
+    - task: UseDotNet@2
+      displayName: 'Use .NET 6 SDK 6.0.100'
+      inputs:
+        version: 6.0.100
     - task: BatchScript@1
       displayName: 'dotnet publish'
       inputs:
@@ -29,8 +33,8 @@ jobs:
     - task: CopyFiles@2
       displayName: 'Copy dotnet publish Output'
       inputs:
-        sourceFolder: src/Lighthouse/bin/Release/netcoreapp3.1/publish/
-        targetFolder: $(Build.ArtifactStagingDirectory)/bin/Release/netcoreapp3.1/publish/
+        sourceFolder: src/Lighthouse/bin/Release/net6.0/publish/
+        targetFolder: $(Build.ArtifactStagingDirectory)/bin/Release/net6.0/publish/
     - task: CopyFiles@2
       displayName: 'Copy Dockerfiles'
       inputs:
@@ -61,33 +65,41 @@ jobs:
           buildType: 'current'
           downloadType: 'single'
           artifactName: 'drop'
-          itemPattern: drop/**
           downloadPath: '$(Agent.BuildDirectory)'
+      - task: CopyFiles@2
+        displayName: 'Copy into PWD'
+        inputs:
+          sourceFolder: $(Agent.BuildDirectory)/drop
+          targetFolder: $(Agent.BuildDirectory)
+        # needed for ARM builds
       - task: Docker@2
         displayName: "Login to Docker Hub"
         inputs:
           command: login
           containerRegistry: $(dockerConnectionName)
-      - task: Docker@2
-        displayName: Docker Build (Linux)
-        inputs:
-          command: buildAndPush
-          Dockerfile: $(Agent.BuildDirectory)/drop/Dockerfile-linux
-          repository: petabridge/lighthouse
-          tags: |
-            latest
-            linux-latest
-            $(Build.SourceBranchName)
-            $(Build.SourceBranchName)-linux
+      - script: docker build --no-cache -f Dockerfile-linux -t petabridge/lighthouse:latest -t petabridge/lighthouse:linux-latest -t petabridge/lighthouse:$(Build.SourceBranchName) -t petabridge/lighthouse:$(Build.SourceBranchName)-linux .
+        displayName: Docker Build (AMD64)
+        workingDirectory: $(Agent.BuildDirectory)
+      - script: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+        displayName: Docker - Register QEMU
+      - script: docker buildx build --platform linux/arm64 -f Dockerfile-arm64 --no-cache -t petabridge/lighthouse:arm64-latest -t petabridge/lighthouse:$(Build.SourceBranchName)-arm64 .
+        displayName: Docker Build (ARM64)
+        workingDirectory: $(Agent.BuildDirectory)
+      - script: docker push -a petabridge/lighthouse
+        displayName: Docker Push
       - task: Docker@2
         displayName: Logout of Docker Hub
         inputs:
           command: logout
           containerRegistry: $(dockerConnectionName)
+      - script: 'echo 1>&2'
+        failOnStderr: true
+        displayName: 'If above is partially succeeded, then fail'
+        condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
 
   - job:  windowsImageDeploy
     pool:
-      vmImage: windows-2019
+      vmImage: windows-latest
     dependsOn: publishNetCore
     steps:
       - task: DownloadBuildArtifacts@0

build.fsx

@@ -37,10 +37,17 @@ let toolsDir = __SOURCE_DIRECTORY__ @@ "tools"
 let output = __SOURCE_DIRECTORY__  @@ "bin"
 let outputTests = __SOURCE_DIRECTORY__ @@ "TestResults"
 let outputPerfTests = __SOURCE_DIRECTORY__ @@ "PerfResults"
+let dockerScriptDir = __SOURCE_DIRECTORY__ @@ "src" @@ "Lighthouse"
 let outputNuGet = output @@ "nuget"
 
 exception ConnectionFailure of string
 
+let composedGetDirName (p:string) =
+    System.IO.Path.GetDirectoryName p
+
+let composedGetFileNameWithoutExtension (p:string) =
+    System.IO.Path.GetFileNameWithoutExtension p
+
 Target "Clean" (fun _ ->
     ActivateFinalTarget "KillCreatedProcesses"
 
@@ -131,24 +138,24 @@ Target "NBench" <| fun _ ->
 
     projects |> Seq.iter runSingleProject
 
+let dockerFilesWithTags =
+    match (isWindows) with
+        | true -> [("src/Lighthouse/Dockerfile-windows", ["windows-latest"])] 
+        | _ -> [("src/Lighthouse/Dockerfile-linux", ["latest";"linux-latest"]); ("src/Lighthouse/Dockerfile-arm64", ["arm64-latest"])] 
+
 Target "RunTestsOnRuntimes" (fun _ ->
 
     let LighthouseConnectTimeout = 20.0 // in seconds
-
-    let dockerFileForTest = 
-        match (isWindows) with
-        | true -> "src/Lighthouse/Dockerfile-windows" 
-        | _ -> "src/Lighthouse/Dockerfile-linux" 
 
     let installPbm () =
         // Install pbm client to test connections
         ExecProcess(fun info ->
             info.FileName <- "dotnet"
             info.Arguments <- "tool install --global pbm") (TimeSpan.FromMinutes 5.0) |> ignore // this is fine if tool is already installed
 
-    let startLighthouseDocker dockerFile =
-        printfn "Starting Lighthouse..."
-        let runArgs = "run -d --name lighthouse --hostname lighthouse1 -p 4053:4053 -p 9110:9110 --env CLUSTER_IP=127.0.0.1 --env CLUSTER_SEEDS=akka.tcp://some@lighthouse1:4053 --env CLUSTER_PORT=4053 lighthouse:latest"
+    let startLighthouseDocker dockerFile tag =
+        printfn "Starting Lighthouse w/ Dockerfile %s" dockerFile
+        let runArgs = sprintf "run -d --name lighthouse --hostname lighthouse1 -p 4053:4053 -p 9110:9110 --env ACTORSYSTEM=some --env CLUSTER_IP=127.0.0.1 --env CLUSTER_SEEDS=akka.tcp://some@127.0.0.1:4053 --env CLUSTER_PORT=4053 lighthouse:%s" tag
         let runResult = ExecProcess(fun info -> 
             info.FileName <- "docker"
             info.WorkingDirectory <- (Directory.GetParent dockerFile).FullName
@@ -162,7 +169,7 @@ Target "RunTestsOnRuntimes" (fun _ ->
                 info.WorkingDirectory <- (Directory.GetParent dockerFile).FullName
                 info.Arguments <- "rm -f lighthouse") (System.TimeSpan.FromMinutes 5.0) |> ignore // cleanup failure should not fail the test
 
-    let startLighhouseLocally exePath =
+    let startLighthouseLocally exePath =
         printfn "Starting Lighthouse locally..."
         try
             let runResult = ExecProcess(fun info -> 
@@ -184,21 +191,15 @@ Target "RunTestsOnRuntimes" (fun _ ->
             | _ -> printfn "Lighthouse was connected successfully"
 
     installPbm()
-    startLighthouseDocker dockerFileForTest
-    try       
-        connectLighthouse()
-    finally
-        stopLighthouseDocker dockerFileForTest
-
-    // Test Full .NET Framework version under windows only
-    // TODO: To make this work, need to start lighthouse and pbm as two parallel processes
-    (*
-    match (isWindows) with
-            | true -> 
-                startLighhouseLocally "src/Lighthouse/bin/Release/net461/Lighthouse.exe"
-                connectLighthouse()
-            | _ -> ()
-    *)
+
+    let runSpec (dockerFile, tags) =    
+        startLighthouseDocker dockerFile (tags |> List.head)
+        try       
+            connectLighthouse()
+        finally
+            stopLighthouseDocker dockerFile
+
+    dockerFilesWithTags |> Seq.iter runSpec
 )
 
 
@@ -243,7 +244,7 @@ Target "SignPackages" (fun _ ->
                 info.FileName <- signPath
                 info.WorkingDirectory <- __SOURCE_DIRECTORY__
                 info.Arguments <- args) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
-            if result <> 0 then failwithf "SignClient failed.%s" args
+            if result <> 0 then failwithf "SignClient failed. %s" args
 
         assemblies |> Seq.iter (signAssembly)
     else
@@ -315,7 +316,7 @@ Target "PublishCode" (fun _ ->
                     Project = project
                     Configuration = configuration
                     VersionSuffix = overrideVersionSuffix project
-                    Framework = "netcoreapp3.1"
+                    Framework = "net6.0"
                     })
 
     projects |> Seq.iter (runSingleProject)
@@ -326,74 +327,19 @@ let mapDockerImageName (projectName:string) =
     | "Lighthouse" -> Some("lighthouse")
     | _ -> None
 
-let composedGetDirName (p:string) =
-    System.IO.Path.GetDirectoryName p
-
-let composedGetFileNameWithoutExtension (p:string) =
-    System.IO.Path.GetFileNameWithoutExtension p
-
 Target "BuildDockerImages" (fun _ ->
-    let projects = !! "src/**/*.csproj" 
-                   -- "src/**/*Tests.csproj" // Don't publish unit tests
-                   -- "src/**/*Tests*.csproj"
-
-    let dockerFile = 
-        match (isWindows) with 
-        | true -> "Dockerfile-windows"
-        | _ -> "Dockerfile-linux"
-
-    let dockerTags (imageName:string, assemblyVersion:string) =
-        match(isWindows) with
-        | true -> [| imageName + ":" + releaseNotes.AssemblyVersion; imageName + ":" + releaseNotes.AssemblyVersion + "-nanoserver1803"; imageName + ":latest" |]
-        | _ -> [| imageName + ":" + releaseNotes.AssemblyVersion; imageName + ":" + releaseNotes.AssemblyVersion + "-linux"; imageName + ":latest" |]
-
-
-    let remoteRegistryUrl = getBuildParamOrDefault "remoteRegistry" ""
-
-    let buildDockerImage imageName projectPath =
-
-        let args = 
-            if(hasBuildParam "remoteRegistry") then
-                StringBuilder()
-                    |> append "build"
-                    |> append "-f"
-                    |> append dockerFile
-                    |> append "-t"
-                    |> append (imageName + ":" + releaseNotes.AssemblyVersion) 
-                    |> append "-t"
-                    |> append (imageName + ":latest") 
-                    |> append "-t"
-                    |> append (remoteRegistryUrl + "/" + imageName + ":" + releaseNotes.AssemblyVersion) 
-                    |> append "-t"
-                    |> append (remoteRegistryUrl + "/" + imageName + ":latest") 
-                    |> append "."
-                    |> toText
-            else
-                StringBuilder()
-                    |> append "build"
-                    |> append "-f"
-                    |> append dockerFile
-                    |> append "-t"
-                    |> append (imageName + ":" + releaseNotes.AssemblyVersion) 
-                    |> append "-t"
-                    |> append (imageName + ":latest") 
-                    |> append "."
-                    |> toText
-
-        ExecProcess(fun info -> 
-                info.FileName <- "docker"
-                info.WorkingDirectory <- composedGetDirName projectPath
-                info.Arguments <- args) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
-
-    let runSingleProject project =
-        let projectName = composedGetFileNameWithoutExtension project
-        let imageName = mapDockerImageName projectName
-        let result = match imageName with
-                        | None -> 0
-                        | Some(name) -> buildDockerImage name project
-        if result <> 0 then failwithf "docker build failed. %s" project
-
-    projects |> Seq.iter (runSingleProject)
+   if(isWindows) then
+        let result = ExecProcess(fun info -> 
+                info.FileName <- "powershell"
+                info.WorkingDirectory <- dockerScriptDir
+                info.Arguments <- sprintf " ./buildWindowsDockerImages.ps1 -tagVersion %s" releaseNotes.AssemblyVersion) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
+        if result <> 0 then failwithf "Unable to build Lighthouse Dockerfiles"
+   else
+       let result = ExecProcess(fun info -> 
+                info.FileName <- "sh"
+                info.WorkingDirectory <- dockerScriptDir
+                info.Arguments <- sprintf " ./buildLinuxDockerImages.sh %s" releaseNotes.AssemblyVersion) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
+       if result <> 0 then failwithf "Unable to build Lighthouse Dockerfiles"
 )
 
 //--------------------------------------------------------------------------------
@@ -460,7 +406,7 @@ Target "Nuget" DoNothing
 
 // tests dependencies
 "Build" ==> "RunTests"
-"PublishCode" ==> "BuildDockerImages" ==> "RunTestsOnRuntimes"
+"BuildDockerImages" ==> "RunTestsOnRuntimes"
 
 // nuget dependencies
 "Clean" ==> "Build" ==> "CreateNuget"
@@ -470,7 +416,7 @@ Target "Nuget" DoNothing
 "Clean" ==> "BuildRelease" ==> "Docfx"
 
 // Docker
-"BuildRelease" ==> "PublishCode" ==> "BuildDockerImages" ==> "Docker"
+"BuildDockerImages" ==> "Docker"
 
 // all
 "BuildRelease" ==> "All"