CI: Add support in yml for detecting darc dependency changes in eng/Version.Details.xml

eng/pipelines/common/evaluate-changed-darc-deps.yml

@@ -0,0 +1,15 @@
+# This step template evaluates changes in dependencies defined in `eng/Version.Details.xml`.
+# For more information on how this works works look at evaluate-changed-darc-deps.sh docs
+# at the beginning of that file.
+
+parameters:
+  subsetName: ''
+  # Array containing the arguments that are to be passed down to evaluate-changed-paths.sh
+  # Note that --azurevariable is always set to the dependency name, no need to pass it down.
+  arguments: []
+
+steps:
+  - script: eng/pipelines/evaluate-changed-darc-deps.sh
+            ${{ join(' ', parameters.arguments) }}
+    displayName: Evaluate eng/Version.Details.xml for dependency changes
+    name: DarcDependenciesChanged

eng/pipelines/common/evaluate-paths-job.yml

@@ -50,3 +50,10 @@ jobs:
               - --includepaths '${{ join('+', path.include) }}'
             - ${{ if ne(path.exclude[0], '') }}:
               - --excludepaths '${{ join('+', path.exclude) }}'
+
+    - template: evaluate-changed-darc-deps.yml
+      parameters:
+        arguments:
+        # The commit that we're building is always a merge commit that is merging into the target branch.
+        # So the first parent of the commit is on the target branch and the second parent is on the source branch.
+        - --difftarget HEAD^1

eng/pipelines/common/templates/wasm-build-only.yml

@@ -39,6 +39,7 @@ jobs:
       condition: >-
         or(
           eq(variables['alwaysRunVar'], true),
+          eq(variables['wasmDarcDependenciesChanged'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true))

eng/pipelines/common/templates/wasm-build-tests.yml

@@ -34,6 +34,7 @@ jobs:
       condition: >-
         or(
           eq(variables['alwaysRunVar'], true),
+          eq(variables['wasmDarcDependenciesChanged'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_wasmbuildtests.containsChange'], true))
       # extra steps, run tests

eng/pipelines/common/templates/wasm-debugger-tests.yml

@@ -33,6 +33,7 @@ jobs:
       condition: >-
         or(
           eq(variables['alwaysRunVar'], true),
+          eq(variables['wasmDarcDependenciesChanged'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_wasmdebuggertests.containsChange'], true))
       # extra steps, run tests

eng/pipelines/common/templates/wasm-library-tests.yml

@@ -44,6 +44,7 @@ jobs:
       condition: >-
         or(
           eq(variables['alwaysRunVar'], true),
+          eq(variables['wasmDarcDependenciesChanged'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true))

eng/pipelines/common/templates/wasm-runtime-tests.yml

@@ -34,6 +34,7 @@ jobs:
       condition: >-
         or(
           eq(variables['alwaysRunVar'], true),
+          eq(variables['wasmDarcDependenciesChanged'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true))

eng/pipelines/common/xplat-setup.yml

@@ -108,6 +108,14 @@ jobs:
         ${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}:
           value: CoreCLR
 
+      - name: wasmDarcDependenciesChanged
+        ${{ if eq(parameters.archType, 'wasm') }}:
+          value: $[ or(
+                      eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Manifest-7_0_100'], true),
+                      eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_DotNet_Build_Tasks_Workloads'], true),
+                      eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.System_Runtime_TimeZoneData'], true),
+                      eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_NET_ILLink_Tasks'], true)) ]
+
       - ${{ each variable in parameters.variables }}:
         - ${{ variable }}
 

eng/pipelines/evaluate-changed-darc-deps.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+: '
+  Compares contents of `env/Version.Details.xml` between HEAD and difftarget, and emits variables named for
+  dependencies that satisfy either of:
+    1. version, or sha changed
+    2. it is missing from one of the xmls
+
+  The dependency names have `.` replaced with `_`.
+
+  In order to consume these variables in a yaml pipeline, reference them via: $[ dependencies.<JobName>.outputs["<StepName>.<DependencyName>"] ]
+
+  Example:
+  -difftarget ''HEAD^1''
+'
+
+# Disable globbing in this bash script since we iterate over path patterns
+set -f
+
+# Stop script if unbound variable found (use ${var:-} if intentional)
+set -u
+
+# Stop script if command returns non-zero exit code.
+# Prevents hidden errors caused by missing error code propagation.
+set -e
+
+usage()
+{
+  echo "Script that emits an azure devops variable with all the dependencies that changed in 'eng/Version.Details.xml' contained in the current HEAD against the difftarget"
+  echo "  --difftarget <value>       SHA or branch to diff against. (i.e: HEAD^1, origin/main, 0f4hd36, etc.)"
+  echo "  --azurevariableprefix      Name of azure devops variable to create if change meets filter criteria"
+  echo ""
+
+  echo "Arguments can also be passed in with a single hyphen."
+}
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+eng_root=`cd -P "$scriptroot/.." && pwd`
+
+azure_variable_prefix=''
+diff_target=''
+
+while [[ $# > 0 ]]; do
+  opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+  case "$opt" in
+    -help|-h)
+      usage
+      exit 0
+      ;;
+    -difftarget)
+      diff_target=$2
+      shift
+      ;;
+    -azurevariableprefix)
+      azure_variable_prefix=$2
+      shift
+      ;;
+  esac
+
+  shift
+done
+
+if [[ -z "$diff_target" ]]; then
+    echo "Argument -difftarget is required"
+    usage
+    exit 1
+fi
+
+oldXmlPath=`mktemp`
+
+ci=true # Needed in order to use pipeline-logging-functions.sh
+. "$eng_root/common/pipeline-logging-functions.sh"
+
+git show $diff_target:eng/Version.Details.xml > $oldXmlPath
+# FIXME: errors?
+changed_deps=$(python3 "$eng_root/pipelines/get-changed-darc-deps.py" $oldXmlPath eng/Version.Details.xml)
+rm -f $oldXmlPath
+
+if [[ -n "$azure_variable_prefix" ]]; then
+    azure_variable_prefix="${azure_variable_prefix}_"
+fi
+
+for dep in $changed_deps; do
+    dep=`echo $dep | tr \. _`
+    var_name=${azure_variable_prefix}${dep}
+    echo "Setting pipeline variable $var_name=true"
+    Write-PipelineSetVariable -name $var_name -value true
+done

eng/pipelines/get-changed-darc-deps.py

@@ -0,0 +1,62 @@
+#
+# Emits a comma separated list of dependencies from `eng/Version.Details.xml`
+# that changed as compared to another versions file
+#
+# - we don't really care which is old, and which is new
+# - A dependency name is emitted as changed if:
+#   1. version, or sha changed
+#   2. it is missing from one of the xmls
+
+import xml.etree.ElementTree as ET
+import sys
+from os.path import exists
+
+def getDependencies(xmlfile):
+    tree = ET.parse(xmlfile)
+    root = tree.getroot()
+    deps = {}
+    for depElement in root.findall('.//Dependency'):
+        dep = {}
+        dep['Version'] = depElement.attrib['Version']
+        dep['Sha'] = depElement.find('Sha').text
+
+        deps[depElement.attrib['Name']] = dep
+
+    return deps
+
+def compare(dict1, dict2):
+    if dict1 is None or dict2 is None:
+        print('Nones')
+        return False
+
+    if (not isinstance(dict1, dict)) or (not isinstance(dict2, dict)):
+        print('Not dict')
+        return False
+
+    changed_names = []
+    all_keys = set(dict1.keys()) | set(dict2.keys())
+    for key in all_keys:
+        if key not in dict1 or key not in dict2:
+            print(key)
+            # changed_names.append(key)
+        elif dict1[key] != dict2[key]:
+            print(key)
+            # changed_names.append(key)
+
+    print(','.join(changed_names))
+
+if len(sys.argv) != 3:
+    print(f'Usage: {sys.argv[0]} <old Version.Details.xml> <new Version.Details.xml>')
+    exit(1)
+
+if not exists(sys.argv[1]):
+    print(f'Cannot find {sys.argv[1]}')
+    exit(1)
+if not exists(sys.argv[2]):
+    print(f'Cannot find {sys.argv[2]}')
+    exit(1)
+
+newDeps = getDependencies(sys.argv[1])
+oldDeps = getDependencies(sys.argv[2])
+
+compare(oldDeps, newDeps)