Fix ordering of on-cluster resources (#1221)

The creating of on-cluster resources requires access to the cluster.
Per default, the IAM principal creating the cluster gets admin access.
But the k8s provider that's used to create on-cluster can be configured
to use a different IAM role. If that's done, the auth settings (aws-auth
ConfigMap or access entries) need to be applied before creating the
on-cluster resources.

This change adds the missing dependency links for on-cluster resources.

.github/workflows/cron.yml

@@ -292,6 +292,7 @@ jobs:
           - MigrateNodeGroups
           - MNG_withAwsAuth
           - MNG_withMissingRole
+          - MultiRole
           - NodeGroup
           - NodegroupOptions
           - OidcIam

.github/workflows/master.yml

@@ -451,6 +451,7 @@ jobs:
           - MigrateNodeGroups
           - MNG_withAwsAuth
           - MNG_withMissingRole
+          - MultiRole
           - NodeGroup
           - NodegroupOptions
           - OidcIam

.github/workflows/release.yml

@@ -466,6 +466,7 @@ jobs:
           - MigrateNodeGroups
           - MNG_withAwsAuth
           - MNG_withMissingRole
+          - MultiRole
           - NodeGroup
           - NodegroupOptions
           - OidcIam

.github/workflows/run-acceptance-tests.yml

@@ -440,6 +440,7 @@ jobs:
           - MigrateNodeGroups
           - MNG_withAwsAuth
           - MNG_withMissingRole
+          - MultiRole
           - NodeGroup
           - NodegroupOptions
           - OidcIam

examples/examples_nodejs_test.go

@@ -708,6 +708,25 @@ func TestAccAuthenticationMode(t *testing.T) {
 	integration.ProgramTest(t, &test)
 }
 
+func TestAccMultiRole(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	test := getJSBaseOptions(t).
+		With(integration.ProgramTestOptions{
+			Dir: path.Join(getCwd(t), "tests", "multi-role"),
+			ExtraRuntimeValidation: func(t *testing.T, info integration.RuntimeValidationStackInfo) {
+				// Verify that the cluster is working.
+				utils.RunEKSSmokeTest(t,
+					info.Deployment.Resources,
+					info.Outputs["kubeconfig"],
+				)
+			},
+		})
+
+	integration.ProgramTest(t, &test)
+}
+
 func TestAccAuthenticationModeMigration(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")

examples/tests/multi-role/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: example-cluster
+description: EKS cluster example
+runtime: nodejs

examples/tests/multi-role/iam.ts

@@ -0,0 +1,27 @@
+import * as aws from "@pulumi/aws";
+import * as pulumi from "@pulumi/pulumi";
+
+const managedPolicyArns: string[] = [
+    "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
+    "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
+    "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
+];
+
+// Creates a role and attches the EKS worker node IAM managed policies
+export function createRole(name: string): aws.iam.Role {
+    const role = new aws.iam.Role(name, {
+        assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({
+            Service: "ec2.amazonaws.com",
+        }),
+    });
+
+    let counter = 0;
+    for (const policy of managedPolicyArns) {
+        // Create RolePolicyAttachment without returning it.
+        const rpa = new aws.iam.RolePolicyAttachment(`${name}-policy-${counter++}`,
+            { policyArn: policy, role: role },
+        );
+    }
+
+    return role;
+}

examples/tests/multi-role/index.ts

@@ -0,0 +1,85 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as awsx from "@pulumi/awsx";
+import * as eks from "@pulumi/eks";
+import * as aws from "@pulumi/aws";
+import * as iam from "./iam";
+
+const projectName = pulumi.getProject();
+
+// Create a VPC with public subnets only
+const vpc = new awsx.ec2.Vpc(`${projectName}-vpc`, {
+    tags: {"Name": `${projectName}-2`},
+    subnetSpecs: [
+        { type: "Public" }
+    ],
+    natGateways: {
+        strategy: "None",
+    }
+});
+
+const accessIamRole = new aws.iam.Role(`${projectName}-role`, {
+    assumeRolePolicy: {
+        Version: "2012-10-17",
+        Statement: [{
+            Action: "sts:AssumeRole",
+            Effect: "Allow",
+            Principal: {
+                AWS: aws.getCallerIdentityOutput().arn,
+            },
+        }],
+    },
+});
+
+/**
+ * Identical IAM for all NodeGroups: all NodeGroups share the same `instanceRole`.
+ */
+const role0 = iam.createRole("example-role0");
+const instanceProfile0 = new aws.iam.InstanceProfile("example-instanceProfile0", {role: role0});
+
+const cluster = new eks.Cluster(`${projectName}-cluster`, {
+    vpcId: vpc.vpcId,
+    publicSubnetIds: vpc.publicSubnetIds,
+    skipDefaultNodeGroup: true,
+    authenticationMode: eks.AuthenticationMode.API,
+    instanceRole: role0,
+    storageClasses: {
+        "mygp2": {
+            type: "gp2",
+            default: true,
+            encrypted: true,
+        },
+    },
+    accessEntries: {
+        // Grant the IAM role admin access to the cluster
+        [`${projectName}-role`]: {
+            principalArn: accessIamRole.arn,
+            accessPolicies: {
+                admin: {
+                    policyArn: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy",
+                    accessScope: {
+                        type: "cluster",
+                    },
+                }
+            }
+        }
+    },
+    providerCredentialOpts: {
+        // Use the IAM role as the provider's credentials source
+        roleArn: accessIamRole.arn,
+    }
+});
+
+cluster.createNodeGroup("example-ng-simple-ondemand", {
+    instanceType: "t3.medium",
+    desiredCapacity: 1,
+    minSize: 1,
+    maxSize: 2,
+    labels: {"ondemand": "true"},
+    instanceProfile: instanceProfile0,
+});
+
+// Export the clusters' kubeconfig.
+export const kubeconfig = cluster.kubeconfig;
+
+// export the IAM Role ARN
+export const iamRoleArn = accessIamRole.arn;

examples/tests/multi-role/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "multi-role",
+    "devDependencies": {
+        "typescript": "^4.0.0",
+        "@types/node": "latest"
+    },
+    "dependencies": {
+        "@pulumi/pulumi": "^3.0.0",
+        "@pulumi/awsx": "^2.0.2",
+        "@pulumi/eks": "latest"
+    }
+}

examples/tests/multi-role/tsconfig.json

@@ -0,0 +1,24 @@
+{
+    "compilerOptions": {
+        "outDir": "bin",
+        "target": "es6",
+        "lib": [
+            "es6"
+        ],
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "declaration": true,
+        "sourceMap": true,
+        "stripInternal": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitAny": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true,
+        "strictNullChecks": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

nodejs/eks/cluster.ts

@@ -740,41 +740,8 @@ export function createCore(
         { parent: parent },
     );
 
-    // Add any requested StorageClasses.
-    const storageClasses = args.storageClasses || {};
-    const userStorageClasses = {} as UserStorageClasses;
-    if (typeof storageClasses === "string") {
-        const storageClass = { type: storageClasses, default: true };
-        userStorageClasses[storageClasses] = pulumi.output(
-            createStorageClass(`${name.toLowerCase()}-${storageClasses}`, storageClass, {
-                parent,
-                provider: k8sProvider,
-            }),
-        );
-    } else {
-        for (const key of Object.keys(storageClasses)) {
-            userStorageClasses[key] = pulumi.output(
-                createStorageClass(`${name.toLowerCase()}-${key}`, storageClasses[key], {
-                    parent,
-                    provider: k8sProvider,
-                }),
-            );
-        }
-    }
-
     const skipDefaultNodeGroup = args.skipDefaultNodeGroup || args.fargate;
 
-    // Create the VPC CNI management resource.
-    let vpcCni: VpcCni | undefined;
-    if (!args.useDefaultVpcCni) {
-        vpcCni = new VpcCni(
-            `${name}-vpc-cni`,
-            kubeconfig.apply(JSON.stringify),
-            args.vpcCniOptions,
-            { parent },
-        );
-    }
-
     let instanceRoles: pulumi.Output<aws.iam.Role[]>;
     let defaultInstanceRole: pulumi.Output<aws.iam.Role> | undefined;
     // Create role mappings of the instance roles specified for aws-auth.
@@ -873,13 +840,11 @@ export function createCore(
     }
 
     // Create the access entries if the authentication mode supports it.
-    let accessEntries: pulumi.Output<aws.eks.AccessEntry[]> | undefined = undefined;
+    let accessEntries: aws.eks.AccessEntry[] | undefined = undefined;
     if (supportsAccessEntries(args.authenticationMode)) {
-        let createdAccessEntries: aws.eks.AccessEntry[] = [];
-
         // This additionally maps the defaultInstanceRole to a EC2_LINUX access entry which allows the nodes to register & communicate with the EKS control plane.
         if (defaultInstanceRole) {
-            createdAccessEntries = createAccessEntries(
+            accessEntries = createAccessEntries(
                 name,
                 eksCluster.name,
                 {
@@ -888,17 +853,57 @@ export function createCore(
                         type: AccessEntryType.EC2_LINUX,
                     },
                 },
-                { parent, provider },
+                { parent, provider, dependsOn: [eksCluster] },
             );
         }
 
-        createdAccessEntries = createdAccessEntries.concat(
+        accessEntries = (accessEntries || []).concat(
             createAccessEntries(name, eksCluster.name, args.accessEntries || {}, {
                 parent,
                 provider,
+                dependsOn: [eksCluster],
             }),
         );
-        accessEntries = pulumi.output(createdAccessEntries);
+    }
+
+    const authDependencies = [
+        ...(accessEntries ? accessEntries : []),
+        ...(eksNodeAccess ? [eksNodeAccess] : []),
+    ];
+
+    // Add any requested StorageClasses.
+    const storageClasses = args.storageClasses || {};
+    const userStorageClasses = {} as UserStorageClasses;
+    if (typeof storageClasses === "string") {
+        const storageClass = { type: storageClasses, default: true };
+        userStorageClasses[storageClasses] = pulumi.output(
+            createStorageClass(`${name.toLowerCase()}-${storageClasses}`, storageClass, {
+                parent,
+                provider: k8sProvider,
+                dependsOn: authDependencies,
+            }),
+        );
+    } else {
+        for (const key of Object.keys(storageClasses)) {
+            userStorageClasses[key] = pulumi.output(
+                createStorageClass(`${name.toLowerCase()}-${key}`, storageClasses[key], {
+                    parent,
+                    provider: k8sProvider,
+                    dependsOn: authDependencies,
+                }),
+            );
+        }
+    }
+
+    // Create the VPC CNI management resource.
+    let vpcCni: VpcCni | undefined;
+    if (!args.useDefaultVpcCni) {
+        vpcCni = new VpcCni(
+            `${name}-vpc-cni`,
+            kubeconfig.apply(JSON.stringify),
+            args.vpcCniOptions,
+            { parent, dependsOn: authDependencies },
+        );
     }
 
     const fargateProfile: pulumi.Output<aws.eks.FargateProfile | undefined> = pulumi
@@ -1061,7 +1066,7 @@ export function createCore(
         oidcProvider: oidcProvider,
         encryptionConfig: encryptionConfig,
         clusterIamRole: eksRole,
-        accessEntries: accessEntries,
+        accessEntries: accessEntries ? pulumi.output(accessEntries) : undefined,
     };
 }