fix(custom-resources): fix circular dependency when a custom role provided to Provider (#33600)

### Issue # (if applicable)

Closes #20360

### Reason for this change

When users specify a isCompletehandler and specifies a custom role for the provider framework, the output template is not deployable due to circular dependencies.

### Description of changes

The change here is to deprecate the old `role` property because this `role` is shared between the 3 framework lambda functions. The state machine will depends on the sfn default policy. The default policy depends on isCompleteLambda (granting invoke function permission). isCompleteLambda depends on common default role policy. The common role  default policy has startExecution permission to SFN.

The solution is to deprecate `role` and introduce new roles for the onEvent lambda and isComplete/onTimeout lambda


### Describe any new or updated permissions being added

N/A

### Description of how you validated changes

New tests

### Checklist
- [ ] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: GavinZZ

packages/@aws-cdk-testing/framework-integ/test/custom-resources/test/provider-framework/integ.provider-with-waiter-state-machine-custom-role.js.snapshot/IntegProviderWithWaiterStateMachineCustomRoleDefaultTestDeployAssert1C38BF52.assets.json

@@ -0,0 +1,58 @@
+/// !cdk-integ *
+import { App, Stack } from 'aws-cdk-lib';
+import * as integ from '@aws-cdk/integ-tests-alpha';
+import { Construct } from 'constructs';
+import { Code, Function, Runtime } from 'aws-cdk-lib/aws-lambda';
+import { Provider } from 'aws-cdk-lib/custom-resources';
+import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
+
+// This test is not deployable prior to this PR fix https://github.com/aws/aws-cdk/pull/32404/files
+// due to integ-provider-with-waiter-state-machine-custom-role failed: ValidationError: Circular dependency
+// between resources: [MyRoleDefaultPolicyA36BE1DD, MyProviderWithCustomRolewaiterstatemachineRoleDefaultPolicy4808872B,
+// MyProviderWithCustomRoleframeworkonEventCE6B50CD, MyProviderWithCustomRoleframeworkonTimeout1A7D4C59,
+// MyProviderWithCustomRoleframeworkisComplete10E48A2A, MyProviderWithCustomRolewaiterstatemachineA313C5FC,
+// MyProviderWithCustomRolewaiterstatemachineLogGroup836672C3]
+
+class TestStack extends Stack {
+  constructor(scope: Construct, id: string) {
+    super(scope, id);
+
+    const onEventHandler = new Function(this, 'OnEvent', {
+      code: Code.fromInline('foo'),
+      handler: 'index.onEvent',
+      runtime: Runtime.NODEJS_LATEST,
+    });
+    const isCompleteHandler = new Function(this, 'IsComplete', {
+      code: Code.fromInline('foo'),
+      handler: 'index.isComplete',
+      runtime: Runtime.NODEJS_LATEST,
+    });
+
+    new Provider(this, 'MyProviderWithCustomRole', {
+      frameworkOnEventRole: new Role(this, 'MyRole', {
+        assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+        managedPolicies: [
+          ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole'),
+        ],
+      }),
+      frameworkCompleteAndTimeoutRole: new Role(this, 'MyRole2', {
+        assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+        managedPolicies: [
+          ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole'),
+        ],
+      }),
+      onEventHandler,
+      isCompleteHandler,
+    });
+  }
+}
+
+const app = new App();
+const stack = new TestStack(app, 'integ-provider-with-waiter-state-machine-custom-role');
+
+new integ.IntegTest(app, 'IntegProviderWithWaiterStateMachineCustomRole', {
+  testCases: [stack],
+  diffAssets: true,
+});
+
+app.synth();

packages/@aws-cdk-testing/framework-integ/test/custom-resources/test/provider-framework/integ.provider-with-waiter-state-machine-custom-role.js.snapshot/IntegProviderWithWaiterStateMachineCustomRoleDefaultTestDeployAssert1C38BF52.template.json

@@ -9,7 +9,7 @@ import * as iam from '../../../aws-iam';
 import * as kms from '../../../aws-kms';
 import * as lambda from '../../../aws-lambda';
 import * as logs from '../../../aws-logs';
-import { Duration } from '../../../core';
+import { Duration, ValidationError } from '../../../core';
 
 const RUNTIME_HANDLER_PATH = path.join(__dirname, 'runtime');
 const FRAMEWORK_HANDLER_TIMEOUT = Duration.minutes(15); // keep it simple for now
@@ -117,13 +117,37 @@ export interface ProviderProps {
   /**
    * AWS Lambda execution role.
    *
-   * The role that will be assumed by the AWS Lambda.
-   * Must be assumable by the 'lambda.amazonaws.com' service principal.
+   * The role is shared by provider framework's onEvent, isComplete lambda, and onTimeout Lambda functions.
+   * This role will be assumed by the AWS Lambda, so it must be assumable by the 'lambda.amazonaws.com'
+   * service principal.
    *
    * @default - A default role will be created.
+   * @deprecated - Use frameworkOnEventLambdaRole, frameworkIsCompleteLambdaRole, frameworkOnTimeoutLambdaRole
    */
   readonly role?: iam.IRole;
 
+  /**
+   * Lambda execution role for provider framework's onEvent Lambda function. Note that this role must be assumed
+   * by the 'lambda.amazonaws.com' service principal.
+   *
+   * This property cannot be used with 'role' property
+   *
+   * @default - A default role will be created.
+   */
+  readonly frameworkOnEventRole?: iam.IRole;
+
+  /**
+   * Lambda execution role for provider framework's isComplete/onTimeout Lambda function. Note that this role
+   * must be assumed by the 'lambda.amazonaws.com' service principal. To prevent circular dependency problem
+   * in the provider framework, please ensure you specify a different IAM Role for 'frameworkCompleteAndTimeoutRole'
+   * from 'frameworkOnEventRole'.
+   *
+   * This property cannot be used with 'role' property
+   *
+   * @default - A default role will be created.
+   */
+  readonly frameworkCompleteAndTimeoutRole?: iam.IRole;
+
   /**
    * Provider Lambda name.
    *
@@ -202,6 +226,13 @@ export class Provider extends Construct implements ICustomResourceProvider {
       }
     }
 
+    if (props.role && (props.frameworkOnEventRole || props.frameworkCompleteAndTimeoutRole)) {
+      throw new ValidationError('Cannot specify both "role" and any of "frameworkOnEventRole" or "frameworkCompleteAndTimeoutRole".', this);
+    }
+    if (!props.isCompleteHandler && props.frameworkCompleteAndTimeoutRole) {
+      throw new ValidationError('Cannot specify "frameworkCompleteAndTimeoutRole" when "isCompleteHandler" is not specified.', this);
+    }
+
     this.onEventHandler = props.onEventHandler;
     this.isCompleteHandler = props.isCompleteHandler;
 
@@ -214,11 +245,11 @@ export class Provider extends Construct implements ICustomResourceProvider {
     this.role = props.role;
     this.providerFunctionEnvEncryption = props.providerFunctionEnvEncryption;
 
-    const onEventFunction = this.createFunction(consts.FRAMEWORK_ON_EVENT_HANDLER_NAME, props.providerFunctionName);
+    const onEventFunction = this.createFunction(consts.FRAMEWORK_ON_EVENT_HANDLER_NAME, props.providerFunctionName, props.frameworkOnEventRole);
 
     if (this.isCompleteHandler) {
-      const isCompleteFunction = this.createFunction(consts.FRAMEWORK_IS_COMPLETE_HANDLER_NAME);
-      const timeoutFunction = this.createFunction(consts.FRAMEWORK_ON_TIMEOUT_HANDLER_NAME);
+      const isCompleteFunction = this.createFunction(consts.FRAMEWORK_IS_COMPLETE_HANDLER_NAME, undefined, props.frameworkCompleteAndTimeoutRole);
+      const timeoutFunction = this.createFunction(consts.FRAMEWORK_ON_TIMEOUT_HANDLER_NAME, undefined, props.frameworkCompleteAndTimeoutRole);
 
       const retry = calculateRetryPolicy(props);
       const waiterStateMachine = new WaiterStateMachine(this, 'waiter-state-machine', {
@@ -263,7 +294,7 @@ export class Provider extends Construct implements ICustomResourceProvider {
     }));
   }
 
-  private createFunction(entrypoint: string, name?: string) {
+  private createFunction(entrypoint: string, name?: string, role?: iam.IRole) {
     const fn = new lambda.Function(this, `framework-${entrypoint}`, {
       code: lambda.Code.fromAsset(RUNTIME_HANDLER_PATH, {
         exclude: ['*.ts'],
@@ -279,7 +310,7 @@ export class Provider extends Construct implements ICustomResourceProvider {
       vpc: this.vpc,
       vpcSubnets: this.vpcSubnets,
       securityGroups: this.securityGroups,
-      role: this.role,
+      role: this.role ?? role,
       functionName: name,
       environmentEncryption: this.providerFunctionEnvEncryption,
     });

packages/@aws-cdk-testing/framework-integ/test/custom-resources/test/provider-framework/integ.provider-with-waiter-state-machine-custom-role.js.snapshot/asset.39472b1c2875cf306d4ba429aeccdd34cb49bcf59dbde81f7e6b6cb9deac23a6/cfn-response.js

@@ -770,6 +770,256 @@ describe('role', () => {
       },
     });
   });
+
+  it('cannot specify both role and framework onEvent roles', () => {
+    // GIVEN
+    const stack = new Stack();
+
+    // WHEN
+    expect(() => new cr.Provider(stack, 'MyProvider', {
+      onEventHandler: new lambda.Function(stack, 'OnEventHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.onEvent',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      isCompleteHandler: new lambda.Function(stack, 'IsCompleteHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.onEvent',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      role: new iam.Role(stack, 'MyRole', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+      frameworkOnEventRole: new iam.Role(stack, 'MyRole2', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+    })).toThrow('Cannot specify both "role" and any of "frameworkOnEventRole" or "frameworkCompleteAndTimeoutRole"');
+  });
+
+  it('cannot specify both role and framework complete/timeout roles', () => {
+    // GIVEN
+    const stack = new Stack();
+
+    // WHEN
+    expect(() => new cr.Provider(stack, 'MyProvider', {
+      onEventHandler: new lambda.Function(stack, 'OnEventHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.onEvent',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      isCompleteHandler: new lambda.Function(stack, 'IsCompleteHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.onEvent',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      role: new iam.Role(stack, 'MyRole', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+      frameworkCompleteAndTimeoutRole: new iam.Role(stack, 'MyRole2', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+    })).toThrow('Cannot specify both "role" and any of "frameworkOnEventRole" or "frameworkCompleteAndTimeoutRole"');
+  });
+
+  it('Cannot specify "frameworkCompleteAndTimeoutRole" when "isCompleteHandler" is not specified.', () => {
+    // GIVEN
+    const stack = new Stack();
+
+    // WHEN
+    expect(() => new cr.Provider(stack, 'MyProvider', {
+      onEventHandler: new lambda.Function(stack, 'OnEventHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.onEvent',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      frameworkCompleteAndTimeoutRole: new iam.Role(stack, 'MyRole2', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+    })).toThrow('Cannot specify "frameworkCompleteAndTimeoutRole" when "isCompleteHandler" is not specified.');
+  });
+
+  it('No circular dependency thrown.', () => {
+    // GIVEN
+    const app = new App({
+      context: {
+        '@aws-cdk/aws-lambda:createNewPoliciesWithAddToRolePolicy': false,
+      },
+    });
+    const stack = new Stack(app);
+
+    // WHEN
+    new cr.Provider(stack, 'MyProvider', {
+      onEventHandler: new lambda.Function(stack, 'OnEventHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.onEvent',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      isCompleteHandler: new lambda.Function(stack, 'IsCompleteHandler', {
+        code: new lambda.InlineCode('foo'),
+        handler: 'index.isComplete',
+        runtime: lambda.Runtime.NODEJS_LATEST,
+      }),
+      frameworkOnEventRole: new iam.Role(stack, 'MyRole', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+      frameworkCompleteAndTimeoutRole: new iam.Role(stack, 'MyRole2', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.como') }),
+    });
+
+    const template = Template.fromStack(stack);
+    template.hasResourceProperties('AWS::IAM::Policy', {
+      PolicyDocument: {
+        Statement: [
+          {
+            Action: 'lambda:InvokeFunction',
+            Effect: 'Allow',
+            Resource: [
+              {
+                'Fn::GetAtt': [
+                  'OnEventHandler42BEBAE0',
+                  'Arn',
+                ],
+              },
+              {
+                'Fn::Join': [
+                  '',
+                  [
+                    {
+                      'Fn::GetAtt': [
+                        'OnEventHandler42BEBAE0',
+                        'Arn',
+                      ],
+                    },
+                    ':*',
+                  ],
+                ],
+              },
+            ],
+          },
+          {
+            Action: 'lambda:GetFunction',
+            Effect: 'Allow',
+            Resource: {
+              'Fn::GetAtt': [
+                'OnEventHandler42BEBAE0',
+                'Arn',
+              ],
+            },
+          },
+          {
+            Action: 'lambda:InvokeFunction',
+            Effect: 'Allow',
+            Resource: [
+              {
+                'Fn::GetAtt': [
+                  'IsCompleteHandler7073F4DA',
+                  'Arn',
+                ],
+              },
+              {
+                'Fn::Join': [
+                  '',
+                  [
+                    {
+                      'Fn::GetAtt': [
+                        'IsCompleteHandler7073F4DA',
+                        'Arn',
+                      ],
+                    },
+                    ':*',
+                  ],
+                ],
+              },
+            ],
+          },
+          {
+            Action: 'lambda:GetFunction',
+            Effect: 'Allow',
+            Resource: {
+              'Fn::GetAtt': [
+                'IsCompleteHandler7073F4DA',
+                'Arn',
+              ],
+            },
+          },
+          {
+            Action: 'states:StartExecution',
+            Effect: 'Allow',
+            Resource: {
+              Ref: 'MyProviderwaiterstatemachineC1FBB9F9',
+            },
+          },
+        ],
+        Version: '2012-10-17',
+      },
+    });
+    template.hasResourceProperties('AWS::IAM::Policy', {
+      PolicyDocument: {
+        Statement: [
+          {
+            Action: 'lambda:InvokeFunction',
+            Effect: 'Allow',
+            Resource: [
+              {
+                'Fn::GetAtt': [
+                  'OnEventHandler42BEBAE0',
+                  'Arn',
+                ],
+              },
+              {
+                'Fn::Join': [
+                  '',
+                  [
+                    {
+                      'Fn::GetAtt': [
+                        'OnEventHandler42BEBAE0',
+                        'Arn',
+                      ],
+                    },
+                    ':*',
+                  ],
+                ],
+              },
+            ],
+          },
+          {
+            Action: 'lambda:GetFunction',
+            Effect: 'Allow',
+            Resource: {
+              'Fn::GetAtt': [
+                'OnEventHandler42BEBAE0',
+                'Arn',
+              ],
+            },
+          },
+          {
+            Action: 'lambda:InvokeFunction',
+            Effect: 'Allow',
+            Resource: [
+              {
+                'Fn::GetAtt': [
+                  'IsCompleteHandler7073F4DA',
+                  'Arn',
+                ],
+              },
+              {
+                'Fn::Join': [
+                  '',
+                  [
+                    {
+                      'Fn::GetAtt': [
+                        'IsCompleteHandler7073F4DA',
+                        'Arn',
+                      ],
+                    },
+                    ':*',
+                  ],
+                ],
+              },
+            ],
+          },
+          {
+            Action: 'lambda:GetFunction',
+            Effect: 'Allow',
+            Resource: {
+              'Fn::GetAtt': [
+                'IsCompleteHandler7073F4DA',
+                'Arn',
+              ],
+            },
+          },
+        ],
+        Version: '2012-10-17',
+      },
+    });
+  });
 });
 
 describe('name', () => {