Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ecr-assets): Support cache-from and cache-to flags #24024

Merged
merged 13 commits into from Mar 8, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 12 additions & 0 deletions packages/@aws-cdk/aws-ecr-assets/README.md
Expand Up @@ -118,6 +118,18 @@ const asset = new DockerImageAsset(this, 'MyBuildImage', {
})
```

You can optionally pass cache from and cache to options to cache images:

```ts
import { DockerImageAsset, Platform } from '@aws-cdk/aws-ecr-assets';

const asset = new DockerImageAsset(this, 'MyBuildImage', {
directory: path.join(__dirname, 'my-image'),
cacheFrom: ['type=registry,ref=ghcr.io/myorg/myimage:cache'],
cacheTo: 'type=registry,ref=ghcr.io/myorg/myimage:cache,mode=max,compression=zstd'
})
```

## Images from Tarball

Images are loaded from a local tarball, uploaded to ECR by the CDK toolkit and/or your app's CI-CD pipeline, and can be
Expand Down
32 changes: 32 additions & 0 deletions packages/@aws-cdk/aws-ecr-assets/lib/image-asset.ts
Expand Up @@ -212,6 +212,22 @@ export interface DockerImageAssetOptions extends FingerprintOptions, FileFingerp
* @see https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs
*/
readonly outputs?: string[];

/**
* Cache from options to pass to the `docker build` command.
*
* @default - no cache from options are passed to the build command
* @see https://docs.docker.com/build/cache/backends/
*/
readonly cacheFrom?: string[];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we more strongly type this rather than just a string array?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was following the pattern of the other arguments as these don't have a firm schema. I think there is a general shape I could use if we want a bit more of a structure though!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@TheRealAmazonKendra I updated the PR to have a more strongishly-type cache option. Lmk how it looks!


/**
* Cache to options to pass to the `docker build` command.
*
* @default - no cache to options are passed to the build command
* @see https://docs.docker.com/build/cache/backends/
*/
readonly cacheTo?: string;
RichiCoder1 marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down Expand Up @@ -287,6 +303,16 @@ export class DockerImageAsset extends Construct implements IAsset {
*/
private readonly dockerOutputs?: string[];

/**
* Cache from options to pass to the `docker build` command.
*/
private readonly dockerCacheFrom?: string[];

/**
* Cache to options to pass to the `docker build` command.
*/
private readonly dockerCacheTo?: string;

/**
* Docker target to build to
*/
Expand Down Expand Up @@ -376,6 +402,8 @@ export class DockerImageAsset extends Construct implements IAsset {
this.dockerBuildArgs = props.buildArgs;
this.dockerBuildTarget = props.target;
this.dockerOutputs = props.outputs;
this.dockerCacheFrom = props.cacheFrom;
this.dockerCacheTo = props.cacheTo;

const location = stack.synthesizer.addDockerImageAsset({
directoryName: this.assetPath,
Expand All @@ -386,6 +414,8 @@ export class DockerImageAsset extends Construct implements IAsset {
networkMode: props.networkMode?.mode,
platform: props.platform?.platform,
dockerOutputs: this.dockerOutputs,
dockerCacheFrom: this.dockerCacheFrom,
dockerCacheTo: this.dockerCacheTo,
});

this.repository = ecr.Repository.fromRepositoryName(this, 'Repository', location.repositoryName);
Expand Down Expand Up @@ -423,6 +453,8 @@ export class DockerImageAsset extends Construct implements IAsset {
resource.cfnOptions.metadata[cxapi.ASSET_RESOURCE_METADATA_DOCKER_BUILD_TARGET_KEY] = this.dockerBuildTarget;
resource.cfnOptions.metadata[cxapi.ASSET_RESOURCE_METADATA_PROPERTY_KEY] = resourceProperty;
resource.cfnOptions.metadata[cxapi.ASSET_RESOURCE_METADATA_DOCKER_OUTPUTS_KEY] = this.dockerOutputs;
resource.cfnOptions.metadata[cxapi.ASSET_RESOURCE_METADATA_DOCKER_CACHE_FROM_KEY] = this.dockerCacheFrom;
resource.cfnOptions.metadata[cxapi.ASSET_RESOURCE_METADATA_DOCKER_CACHE_TO_KEY] = this.dockerCacheTo;
}

}
Expand Down
Expand Up @@ -76,6 +76,11 @@
"Value": {
"Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:fa08370824fa0a7eab2c59a4f371fe7631019044d6c906b4268193120dc213b4"
}
},
"ImageUri5": {
"Value": {
"Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:0a3355be12051c9984bf2b0b2bba4e6ea535968e5b6e7396449701732fe5ed14"
}
}
},
"Parameters": {
Expand Down
Expand Up @@ -69,6 +69,12 @@
"data": "ImageUri4"
}
],
"/integ-assets-docker/ImageUri5": [
{
"type": "aws:cdk:logicalId",
"data": "ImageUri5"
}
],
"/integ-assets-docker/BootstrapVersion": [
{
"type": "aws:cdk:logicalId",
Expand Down
Expand Up @@ -236,6 +236,14 @@
"version": "0.0.0"
}
},
"ImageUri4": {
"id": "ImageUri4",
"path": "integ-assets-docker/ImageUri5",
"constructInfo": {
"fqn": "@aws-cdk/core.CfnOutput",
"version": "0.0.0"
}
},
"BootstrapVersion": {
"id": "BootstrapVersion",
"path": "integ-assets-docker/BootstrapVersion",
Expand Down
8 changes: 8 additions & 0 deletions packages/@aws-cdk/aws-ecr-assets/test/integ.assets-docker.ts
Expand Up @@ -24,15 +24,23 @@ const asset4 = new assets.DockerImageAsset(stack, 'DockerImage4', {
outputs: ['type=docker'],
});

const asset5 = new assets.DockerImageAsset(stack, 'DockerImage5', {
directory: path.join(__dirname, 'demo-image'),
cacheFrom: ['type=registry,ref=notan/image'],
cacheTo: 'type=inline',
});

const user = new iam.User(stack, 'MyUser');
asset.repository.grantPull(user);
asset2.repository.grantPull(user);
asset3.repository.grantPull(user);
asset4.repository.grantPull(user);
asset5.repository.grantPull(user);

new cdk.CfnOutput(stack, 'ImageUri', { value: asset.imageUri });
new cdk.CfnOutput(stack, 'ImageUri2', { value: asset2.imageUri });
new cdk.CfnOutput(stack, 'ImageUri3', { value: asset3.imageUri });
new cdk.CfnOutput(stack, 'ImageUri4', { value: asset4.imageUri });
new cdk.CfnOutput(stack, 'ImageUri5', { value: asset5.imageUri });

app.synth();
Expand Up @@ -88,6 +88,22 @@ export interface DockerImageSource {
* @see https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs
*/
readonly dockerOutputs?: string[];

/**
* Cache from options to pass to the `docker build` command.
*
* @default - no cache from options are passed to the build command
* @see https://docs.docker.com/build/cache/backends/
*/
readonly cacheFrom?: string[];

/**
* Cache to options to pass to the `docker build` command.
*
* @default - no cache to options are passed to the build command
* @see https://docs.docker.com/build/cache/backends/
*/
readonly cacheTo?: string;
}

/**
Expand Down
Expand Up @@ -153,6 +153,22 @@ export interface ContainerImageAssetMetadataEntry extends BaseAssetMetadataEntry
* @see https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs
*/
readonly outputs?: string[];

/**
* Cache from options to pass to the `docker build` command.
*
* @default - no cache from options are passed to the build command
* @see https://docs.docker.com/build/cache/backends/
*/
readonly cacheFrom?: string[];

/**
* Cache to options to pass to the `docker build` command.
*
* @default - no cache to options are passed to the build command
* @see https://docs.docker.com/build/cache/backends/
*/
readonly cacheTo?: string;
}

/**
Expand Down
11 changes: 11 additions & 0 deletions packages/@aws-cdk/cloud-assembly-schema/schema/assets.schema.json
Expand Up @@ -169,6 +169,17 @@
"items": {
"type": "string"
}
},
"cacheFrom": {
"description": "Cache from options to pass to the `docker build` command. (Default - no cache from options are passed to the build command)",
"type": "array",
"items": {
"type": "string"
}
},
"cacheTo": {
"description": "Cache to options to pass to the `docker build` command. (Default - no cache to options are passed to the build command)",
"type": "string"
}
}
},
Expand Down
Expand Up @@ -241,6 +241,17 @@
"type": "string"
}
},
"cacheFrom": {
"description": "Cache from options to pass to the `docker build` command. (Default - no cache from options are passed to the build command)",
"type": "array",
"items": {
"type": "string"
}
},
"cacheTo": {
"description": "Cache to options to pass to the `docker build` command. (Default - no cache to options are passed to the build command)",
"type": "string"
},
"id": {
"description": "Logical identifier for the asset",
"type": "string"
Expand Down
@@ -1 +1 @@
{"version":"29.0.0"}
{"version":"30.0.0"}
12 changes: 12 additions & 0 deletions packages/@aws-cdk/core/lib/assets.ts
Expand Up @@ -229,6 +229,18 @@ export interface DockerImageAssetSource {
*/
readonly dockerOutputs?: string[];

/**
* Cache from options to pass to the `docker build` command.
* @default - no cache from args are passed
*/
readonly dockerCacheFrom?: string[];

/**
* Cache to options to pass to the `docker build` command.
* @default - no cache to args are passed
*/
readonly dockerCacheTo?: string;

RichiCoder1 marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down
Expand Up @@ -70,6 +70,8 @@ export class AssetManifestBuilder {
networkMode: asset.networkMode,
platform: asset.platform,
dockerOutputs: asset.dockerOutputs,
cacheFrom: asset.dockerCacheFrom,
cacheTo: asset.dockerCacheTo,
}, {
repositoryName: target.repositoryName,
imageTag,
Expand Down
2 changes: 2 additions & 0 deletions packages/@aws-cdk/core/lib/stack-synthesizers/legacy.ts
Expand Up @@ -148,6 +148,8 @@ export class LegacyStackSynthesizer extends StackSynthesizer implements IReusabl
networkMode: asset.networkMode,
platform: asset.platform,
outputs: asset.dockerOutputs,
cacheFrom: asset.dockerCacheFrom,
cacheTo: asset.dockerCacheTo,
};

this.boundStack.node.addMetadata(cxschema.ArtifactMetadataEntryType.ASSET, metadata);
Expand Down
2 changes: 2 additions & 0 deletions packages/@aws-cdk/cx-api/lib/assets.ts
Expand Up @@ -16,6 +16,8 @@ export const ASSET_RESOURCE_METADATA_DOCKER_BUILD_TARGET_KEY = 'aws:asset:docker
export const ASSET_RESOURCE_METADATA_PROPERTY_KEY = 'aws:asset:property';
export const ASSET_RESOURCE_METADATA_IS_BUNDLED_KEY = 'aws:asset:is-bundled';
export const ASSET_RESOURCE_METADATA_DOCKER_OUTPUTS_KEY = 'aws:asset:docker-outputs';
export const ASSET_RESOURCE_METADATA_DOCKER_CACHE_FROM_KEY = 'aws:asset:docker-cache-from';
export const ASSET_RESOURCE_METADATA_DOCKER_CACHE_TO_KEY = 'aws:asset:docker-cache-to';

/**
* Separator string that separates the prefix separator from the object key separator.
Expand Down
3 changes: 3 additions & 0 deletions packages/aws-cdk-lib/package.json
Expand Up @@ -482,6 +482,7 @@
"./aws-datasync": "./aws-datasync/index.js",
"./aws-dax": "./aws-dax/index.js",
"./aws-detective": "./aws-detective/index.js",
"./aws-devicefarm": "./aws-devicefarm/index.js",
"./aws-devopsguru": "./aws-devopsguru/index.js",
"./aws-directoryservice": "./aws-directoryservice/index.js",
"./aws-dlm": "./aws-dlm/index.js",
Expand Down Expand Up @@ -583,6 +584,7 @@
"./aws-networkmanager": "./aws-networkmanager/index.js",
"./aws-nimblestudio": "./aws-nimblestudio/index.js",
"./aws-oam": "./aws-oam/index.js",
"./aws-omics": "./aws-omics/index.js",
"./aws-opensearchserverless": "./aws-opensearchserverless/index.js",
"./aws-opensearchservice": "./aws-opensearchservice/index.js",
"./aws-opsworks": "./aws-opsworks/index.js",
Expand Down Expand Up @@ -631,6 +633,7 @@
"./aws-ses": "./aws-ses/index.js",
"./aws-ses-actions": "./aws-ses-actions/index.js",
"./aws-signer": "./aws-signer/index.js",
"./aws-simspaceweaver": "./aws-simspaceweaver/index.js",
"./aws-sns": "./aws-sns/index.js",
"./aws-sns-subscriptions": "./aws-sns-subscriptions/index.js",
"./aws-sqs": "./aws-sqs/index.js",
Expand Down
4 changes: 4 additions & 0 deletions packages/cdk-assets/lib/private/docker.ts
Expand Up @@ -18,6 +18,8 @@ interface BuildOptions {
readonly networkMode?: string;
readonly platform?: string;
readonly outputs?: string[];
readonly cacheFrom?: string[];
readonly cacheTo?: string;
}

export interface DockerCredentialsConfig {
Expand Down Expand Up @@ -60,6 +62,8 @@ export class Docker {
...options.networkMode ? ['--network', options.networkMode] : [],
...options.platform ? ['--platform', options.platform] : [],
...options.outputs ? options.outputs.map(output => [`--output=${output}`]) : [],
...options.cacheFrom ? options.cacheFrom.map(cacheFrom => [`--cache-from ${cacheFrom}`]) : [],
...options.cacheTo ? [`--cache-to ${options.cacheTo}`] : [],
'.',
];
await this.execute(buildCommand, { cwd: options.directory });
Expand Down
2 changes: 2 additions & 0 deletions packages/cdk-assets/lib/private/handlers/container-images.ts
Expand Up @@ -172,6 +172,8 @@ class ContainerImageBuilder {
networkMode: source.networkMode,
platform: source.platform,
outputs: source.dockerOutputs,
cacheFrom: source.cacheFrom,
cacheTo: source.cacheTo,
});
}

Expand Down