Merge remote-tracking branch 'upstream/feature/perf' into private/edv…

…int/merge-stdext2-tests2

.git-blame-ignore-revs

@@ -29,6 +29,7 @@ b020cf35a1f2c274f95a4118d4596043cba6113f
 ff39018fd6d91985f9c893a56928771dfe9fa48d
 cbb9edb17dfd122c591beb14d1275acc39492335
 d6ab15362548b8fe270bd14d5153b8d94e1b15c0
+b12cf444edea15da6274975e1b2ca6a7fce2a090
 
 # ocp-indent
 d018d26d6acd4707a23288b327b49e44f732725e

.github/workflows/generate-and-build-sdks.yml

@@ -24,6 +24,12 @@ jobs:
         shell: bash
         run: opam exec -- make sdk
 
+      - name: Store C SDK source
+        uses: actions/upload-artifact@v4
+        with:
+          name: SDK_Source_C
+          path: _build/install/default/xapi/sdk/c/*
+
       - name: Store C# SDK source
         uses: actions/upload-artifact@v4
         with:
@@ -39,6 +45,30 @@ jobs:
       - name: Cleanup XenAPI environment
         uses: ./.github/workflows/cleanup-xapi-environment
 
+  build-c-sdk:
+    name: Build C SDK
+    runs-on: ubuntu-latest
+    needs: generate-sdk-sources
+    steps:
+      - name: Install dependencies
+        run: sudo apt-get install libxml2-dev
+
+      - name: Retrieve C SDK source
+        uses: actions/download-artifact@v4
+        with:
+          name: SDK_Source_C
+          path: source/
+
+      - name: Build C SDK
+        shell: bash
+        run: make -C source
+
+      - name: Store C SDK
+        uses: actions/upload-artifact@v4
+        with:
+          name: SDK_Artifacts_C
+          path: source/*
+
   build-csharp-sdk:
     name: Build C# SDK
     runs-on: windows-2022

.github/workflows/release.yml

@@ -52,6 +52,12 @@ jobs:
           name: XenAPI
           path: dist/
 
+      - name: Retrieve C SDK distribution binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: SDK_Artifacts_C
+          path: libxenserver/usr/local/
+
       - name: Retrieve C# SDK distribution artifacts
         uses: actions/download-artifact@v4
         with:
@@ -70,10 +76,19 @@ jobs:
           name: SDK_Binaries_XenServerPowerShell_NET6
           path: sdk_powershell_7x/
 
+      - name: Package C SDK artifacts for deployment
+        shell: bash
+        run: |
+          mkdir -p libxenserver/usr/local/lib
+          mv libxenserver/usr/local/libxenserver.* libxenserver/usr/local/lib/
+          tar -zcvf libxenserver-prerelease.tar.gz -C ./libxenserver usr/local/lib/ usr/local/include/xen/api
+          rm -rf libxenserver/usr/local/lib/
+          tar -zcvf libxenserver-prerelease.src.tar.gz -C ./libxenserver/usr/local .
+
       - name: Zip PowerShell 5.x SDK artifacts for deployment
         shell: bash
         run: zip PowerShell-SDK-5.x-prerelease-unsigned.zip ./sdk_powershell_5x -r
-      
+
       - name: Zip PowerShell 7.x SDK artifacts for deployment
         shell: bash
         run: zip PowerShell-SDK-7.x-prerelease-unsigned.zip ./sdk_powershell_7x -r
@@ -83,7 +98,8 @@ jobs:
         run: |
           gh release create ${{ github.ref_name }} --repo ${{ github.repository }} --generate-notes dist/* \
           PowerShell-SDK-5.x-prerelease-unsigned.zip \
-          PowerShell-SDK-7.x-prerelease-unsigned.zip
+          PowerShell-SDK-7.x-prerelease-unsigned.zip \
+          libxenserver-prerelease.tar.gz libxenserver-prerelease.src.tar.gz
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

Makefile

@@ -122,10 +122,10 @@ sdk:
 	sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/csharp
 	sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/powershell
 
-.PHONY: sdk-build-c sdk
+.PHONY: sdk-build-c
 
 sdk-build-c: sdk
-	cd _build/install/default/xapi/sdk/c && make -j $(JOBS)
+	cd _build/install/default/xapi/sdk/c && make clean && make -j $(JOBS)
 
 .PHONY: sdk-build-java
 
@@ -207,6 +207,7 @@ install: build doc sdk doc-json
 	install -D -m 755 _build/install/default/bin/xcp-rrdd-iostat $(DESTDIR)$(LIBEXECDIR)/xcp-rrdd-plugins/xcp-rrdd-iostat
 	install -D -m 755 _build/install/default/bin/xcp-rrdd-squeezed $(DESTDIR)$(LIBEXECDIR)/xcp-rrdd-plugins/xcp-rrdd-squeezed
 	install -D -m 755 _build/install/default/bin/xcp-rrdd-xenpm $(DESTDIR)$(LIBEXECDIR)/xcp-rrdd-plugins/xcp-rrdd-xenpm
+	install -D -m 755 _build/install/default/bin/xcp-rrdd-dcmi $(DESTDIR)$(LIBEXECDIR)/xcp-rrdd-plugins/xcp-rrdd-dcmi
 	install -D -m 644 ocaml/xcp-rrdd/bugtool-plugin/rrdd-plugins.xml $(DESTDIR)$(ETCXENDIR)/bugtool/xcp-rrdd-plugins.xml
 	install -D -m 644 ocaml/xcp-rrdd/bugtool-plugin/rrdd-plugins/stuff.xml $(DESTDIR)$(ETCXENDIR)/bugtool/xcp-rrdd-plugins/stuff.xml
 	install -D -m 755 ocaml/xcp-rrdd/bin/rrdp-scripts/sysconfig-rrdd-plugins $(DESTDIR)/etc/sysconfig/xcp-rrdd-plugins

forkexec.opam

@@ -21,6 +21,7 @@ depends: [
   "xapi-log"
   "xapi-stdext-pervasives"
   "xapi-stdext-unix"
+  "xapi-tracing"
 ]
 synopsis: "Sub-process control service for xapi"
 description:

forkexec.opam.template

@@ -19,6 +19,7 @@ depends: [
   "xapi-log"
   "xapi-stdext-pervasives"
   "xapi-stdext-unix"
+  "xapi-tracing"
 ]
 synopsis: "Sub-process control service for xapi"
 description:

message-switch-core.opam

@@ -22,6 +22,7 @@ depends: [
   "ppx_sexp_conv"
   "rpclib"
   "sexplib"
+  "xapi-log"
 ]
 synopsis: "A simple store-and-forward message switch"
 description: """

message-switch-core.opam.template

@@ -20,6 +20,7 @@ depends: [
   "ppx_sexp_conv"
   "rpclib"
   "sexplib"
+  "xapi-log"
 ]
 synopsis: "A simple store-and-forward message switch"
 description: """

ocaml/forkexecd/lib/dune

@@ -7,12 +7,13 @@
    fd-send-recv
    rpclib.core
    rpclib.json
+   rpclib.xml
    uuid
    xapi-backtrace
    xapi-log
    xapi-stdext-pervasives
    xapi-stdext-unix
-   rpclib.xml
+   xapi-tracing
  )
  (preprocess
   (pps ppx_deriving_rpc)))

ocaml/forkexecd/lib/fe.ml

@@ -1,13 +1,13 @@
 (* Disable "Warning 39: unused rec flag." caused by rpc *)
 [@@@warning "-39"]
 
-type syslog_stdout_t = {enabled: bool; key: string option} [@@deriving rpc]
+type syslog_stdout = {enabled: bool; key: string option} [@@deriving rpc]
 
 type setup_cmd = {
     cmdargs: string list
   ; env: string list
   ; id_to_fd_map: (string * int option) list
-  ; syslog_stdout: syslog_stdout_t
+  ; syslog_stdout: syslog_stdout
   ; redirect_stderr_to_stdout: bool
 }
 [@@deriving rpc]

ocaml/forkexecd/lib/forkhelpers.ml

@@ -21,6 +21,8 @@
 
 (* XXX: this is a work in progress *)
 
+module D = Debug.Make (struct let name = __MODULE__ end)
+
 let default_path = ["/sbin"; "/usr/sbin"; "/bin"; "/usr/bin"]
 
 let default_path_env_pair = [|"PATH=" ^ String.concat ":" default_path|]
@@ -34,6 +36,10 @@ let test_path =
 
 let runtime_path = Option.value ~default:"/var" test_path
 
+let _with_tracing ?tracing ~name f =
+  let name = Printf.sprintf "forkhelpers.%s" name in
+  Tracing.with_tracing ?parent:tracing ~name f
+
 let finally = Xapi_stdext_pervasives.Pervasiveext.finally
 
 type pidty = Unix.file_descr * int
@@ -72,14 +78,47 @@ let waitpid (sock, pid) =
       in
       failwith msg
 
-let waitpid_nohang ((sock, _) as x) =
+(* [waitpid_nohang] reports the status of a socket to a process. The
+   intention is to make this non-blocking. If the process is finished,
+   the socket is closed and not otherwise. *)
+let waitpid_nohang (sock, pid) =
+  let verbose = false in
+  if verbose then D.debug "%s pid=%d" __FUNCTION__ pid ;
+  let fail fmt = Printf.kprintf failwith fmt in
   Unix.set_nonblock sock ;
-  let r =
-    try waitpid x
-    with Unix.(Unix_error ((EAGAIN | EWOULDBLOCK), _, _)) ->
-      (0, Unix.WEXITED 0)
-  in
-  Unix.clear_nonblock sock ; r
+  match Fecomms.read_raw_rpc sock with
+  | Ok Fe.(Finished (WEXITED n)) ->
+      if verbose then D.debug "%s pid=%d WEXITED" __FUNCTION__ pid ;
+      Unix.close sock ;
+      (pid, Unix.WEXITED n)
+  | Ok Fe.(Finished (WSIGNALED n)) ->
+      if verbose then D.debug "%s pid=%d WSIGNALED" __FUNCTION__ pid ;
+      Unix.close sock ;
+      (pid, Unix.WSIGNALED n)
+  | Ok Fe.(Finished (WSTOPPED n)) ->
+      if verbose then D.debug "%s pid=%d WSTOPPED" __FUNCTION__ pid ;
+      Unix.close sock ;
+      (pid, Unix.WSTOPPED n)
+  | Ok status ->
+      Unix.clear_nonblock sock ;
+      fail "%s: unexpected status received (%s)" __FUNCTION__
+        (Fe.ferpc_to_string status)
+  | Error msg ->
+      D.debug "%s pid=%d %s" __FUNCTION__ pid msg ;
+      Unix.clear_nonblock sock ;
+      fail "%s: error happened when trying to read the status. %s" __FUNCTION__
+        msg
+  (* it's a bit crazy that we have Result.t and exceptions from
+     read_raw_rpc *)
+  | exception Unix.(Unix_error ((EAGAIN | EWOULDBLOCK), _, _)) ->
+      if verbose then D.debug "%s pid=%d EAGAIN EWOULDBLOCK" __FUNCTION__ pid ;
+      Unix.clear_nonblock sock ;
+      (0, Unix.WEXITED 0) (* this a convention, see MLI *)
+  | exception exn ->
+      D.debug "%s pid=%d %s" __FUNCTION__ pid (Printexc.to_string exn) ;
+      Unix.clear_nonblock sock ;
+      fail "%s: error happened when trying to read the status. %s" __FUNCTION__
+        (Printexc.to_string exn)
 
 let dontwaitpid (sock, _pid) =
   ( try
@@ -136,7 +175,7 @@ let with_logfile_fd ?(delete = true) prefix f =
 
 exception Spawn_internal_error of string * string * Unix.process_status
 
-type syslog_stdout_t =
+type syslog_stdout =
   | NoSyslogging
   | Syslog_DefaultKey
   | Syslog_WithKey of string

ocaml/forkexecd/lib/forkhelpers.mli

@@ -34,7 +34,7 @@
 
 (** {2 High-level interface } *)
 
-type syslog_stdout_t =
+type syslog_stdout =
   | NoSyslogging
   | Syslog_DefaultKey
   | Syslog_WithKey of string
@@ -45,7 +45,7 @@ val default_path_env_pair : string array
 
 val execute_command_get_output :
      ?env:string array
-  -> ?syslog_stdout:syslog_stdout_t
+  -> ?syslog_stdout:syslog_stdout
   -> ?redirect_stderr_to_stdout:bool
   -> ?timeout:float
   -> string
@@ -57,7 +57,7 @@ val execute_command_get_output :
 
 val execute_command_get_output_send_stdin :
      ?env:string array
-  -> ?syslog_stdout:syslog_stdout_t
+  -> ?syslog_stdout:syslog_stdout
   -> ?redirect_stderr_to_stdout:bool
   -> ?timeout:float
   -> string
@@ -97,7 +97,7 @@ val safe_close_and_exec :
   -> Unix.file_descr option
   -> Unix.file_descr option
   -> (string * Unix.file_descr) list
-  -> ?syslog_stdout:syslog_stdout_t
+  -> ?syslog_stdout:syslog_stdout
   -> ?redirect_stderr_to_stdout:bool
   -> string
   -> string list
@@ -111,8 +111,10 @@ val waitpid : pidty -> int * Unix.process_status
 (** [waitpid p] returns the (pid, Unix.process_status) *)
 
 val waitpid_nohang : pidty -> int * Unix.process_status
-(** [waitpid_nohang p] returns the (pid, Unix.process_status) if the process has already
-    	quit or (0, Unix.WEXITTED 0) if the process is still running. *)
+(** [waitpid_nohang p] returns the (pid, Unix.process_status) if the
+    process has already quit or (0, Unix.WEXITTED 0) if the process is
+    still running.  If the process is finished, the socket is closed
+    and not otherwise. *)
 
 val dontwaitpid : pidty -> unit
 (** [dontwaitpid p]: signals the caller's desire to never call waitpid. Note that the final

ocaml/forkexecd/src/child.ml

@@ -3,13 +3,13 @@ let debug (fmt : ('a, unit, string, unit) format4) =
 
 exception Cancelled
 
-type syslog_stdout_t = {enabled: bool; key: string option}
+type syslog_stdout = {enabled: bool; key: string option}
 
 type state_t = {
     cmdargs: string list
   ; env: string list
   ; id_to_fd_map: (string * int option) list
-  ; syslog_stdout: syslog_stdout_t
+  ; syslog_stdout: syslog_stdout
   ; redirect_stderr_to_stdout: bool
   ; ids_received: (string * Unix.file_descr) list
   ; fd_sock2: Unix.file_descr option