Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Drop per-pipeline transformation cache #9459

Merged
merged 5 commits into from
Jan 5, 2024
Merged

Drop per-pipeline transformation cache #9459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
772cd62
Drop per-pipeline transformation cache
devongovett Jan 3, 2024
1063e69
remove asset.hash
devongovett Jan 3, 2024
917ddf9
Remove unused AssetOptions.hash
mischnic Jan 3, 2024
eabf6e9
Merge branch 'v2' into remove-pipeline-cache
devongovett Jan 4, 2024
111fccd
Merge branch 'v2' into remove-pipeline-cache
devongovett Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 2 additions & 9 deletions packages/core/core/src/BundleGraph.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1943,14 +1943,7 @@ export default class BundleGraph {
this.traverseAssets(bundle, asset => {
{
hash.writeString(
[
this.getAssetPublicId(asset),
asset.outputHash,
asset.filePath,
asset.query,
asset.type,
asset.uniqueKey,
].join(':'),
[this.getAssetPublicId(asset), asset.id, asset.outputHash].join(':'),
);
}
});
Expand Down Expand Up @@ -1997,7 +1990,7 @@ export default class BundleGraph {
getHash(bundle: Bundle): string {
let hash = new Hash();
hash.writeString(
bundle.id + bundle.target.publicUrl + this.getContentHash(bundle),
bundle.id + JSON.stringify(bundle.target) + this.getContentHash(bundle),
);

if (bundle.isPlaceholder) {
Expand Down
185 changes: 7 additions & 178 deletions packages/core/core/src/Transformation.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,12 @@ import {
mutableAssetToUncommittedAsset,
} from './public/Asset';
import UncommittedAsset from './UncommittedAsset';
import {
createAsset,
getInvalidationId,
getInvalidationHash,
} from './assetUtils';
import {createAsset, getInvalidationId} from './assetUtils';
import summarizeRequest from './summarizeRequest';
import PluginOptions from './public/PluginOptions';
import {PARCEL_VERSION, FILE_CREATE} from './constants';
import {optionsProxy} from './utils';
import {createConfig} from './InternalConfig';
import {
getConfigHash,
loadPluginConfig,
getConfigRequests,
type ConfigRequest,
Expand Down Expand Up @@ -197,6 +191,7 @@ export default class Transformation {
let assets, error;
try {
let results = await this.runPipelines(pipeline, asset);
await Promise.all(results.map(asset => asset.commit()));
assets = results.map(a => a.value);
} catch (e) {
error = e;
Expand Down Expand Up @@ -237,7 +232,6 @@ export default class Transformation {
let {
content,
size,
hash,
isSource: summarizedIsSource,
} = await summarizeRequest(this.options.inputFS, {
filePath: fromProjectPath(this.options.projectRoot, filePath),
Expand All @@ -251,7 +245,7 @@ export default class Transformation {
// to the filename as the base for the id to ensure it is unique.
let idBase = fromProjectPathRelative(filePath);
if (code != null) {
idBase += hash;
idBase += hashString(code);
}
return new UncommittedAsset({
idBase,
Expand All @@ -260,7 +254,6 @@ export default class Transformation {
filePath,
isSource,
type: path.extname(fromProjectPathRelative(filePath)).slice(1),
hash,
pipeline,
env,
query,
Expand All @@ -282,16 +275,10 @@ export default class Transformation {
initialAsset: UncommittedAsset,
): Promise<Array<UncommittedAsset>> {
let initialType = initialAsset.value.type;
let initialPipelineHash = await this.getPipelineHash(pipeline);
let initialAssetCacheKey = this.getCacheKey(
[initialAsset],
await getInvalidationHash(this.request.invalidations, this.options), // TODO: should be per-pipeline
initialPipelineHash,
let assets: Array<UncommittedAsset> = await this.runPipeline(
pipeline,
initialAsset,
);
let initialCacheEntry = await this.readFromCache(initialAssetCacheKey);

let assets: Array<UncommittedAsset> =
initialCacheEntry || (await this.runPipeline(pipeline, initialAsset));

// Add dev dep requests for each transformer
for (let transformer of pipeline.transformers) {
Expand All @@ -308,30 +295,6 @@ export default class Transformation {
await this.addDevDependency(devDep);
}

if (!initialCacheEntry) {
let pipelineHash = await this.getPipelineHash(pipeline);
let invalidationCacheKey = await getInvalidationHash(
assets.flatMap(asset => asset.getInvalidations()),
this.options,
);
let resultCacheKey = this.getCacheKey(
[initialAsset],
invalidationCacheKey,
pipelineHash,
);
await this.writeToCache(
resultCacheKey,
assets,
invalidationCacheKey,
pipelineHash,
);
} else {
// See above TODO, this should be per-pipeline
for (let i of this.request.invalidations) {
this.invalidations.set(getInvalidationId(i), i);
}
}

let finalAssets: Array<UncommittedAsset> = [];
for (let asset of assets) {
let nextPipeline;
Expand All @@ -357,59 +320,13 @@ export default class Transformation {
return finalAssets;
}

let pipelineHash = await this.getPipelineHash(pipeline);
let invalidationHash = await getInvalidationHash(
finalAssets.flatMap(asset => asset.getInvalidations()),
this.options,
);
let processedCacheEntry = await this.readFromCache(
this.getCacheKey(finalAssets, invalidationHash, pipelineHash),
);

invariant(pipeline.postProcess != null);
let processedFinalAssets: Array<UncommittedAsset> =
processedCacheEntry ?? (await pipeline.postProcess(finalAssets)) ?? [];

if (!processedCacheEntry) {
await this.writeToCache(
this.getCacheKey(processedFinalAssets, invalidationHash, pipelineHash),
processedFinalAssets,

invalidationHash,
pipelineHash,
);
}
(await pipeline.postProcess(finalAssets)) ?? [];

return processedFinalAssets;
}

async getPipelineHash(pipeline: Pipeline): Promise<string> {
let hashes = '';
for (let transformer of pipeline.transformers) {
let key = `${transformer.name}:${fromProjectPathRelative(
transformer.resolveFrom,
)}`;
hashes +=
this.request.devDeps.get(key) ??
this.devDepRequests.get(key)?.hash ??
':';

let config = this.configs.get(transformer.name);
if (config) {
hashes += await getConfigHash(config, transformer.name, this.options);

for (let devDep of config.devDeps) {
let key = `${devDep.specifier}:${fromProjectPathRelative(
devDep.resolveFrom,
)}`;
hashes += nullthrows(this.devDepRequests.get(key)).hash;
}
}
}

return hashString(hashes);
}

async addDevDependency(opts: InternalDevDepOptions): Promise<void> {
let {specifier, resolveFrom, range} = opts;
let key = `${specifier}:${fromProjectPathRelative(resolveFrom)}`;
Expand Down Expand Up @@ -560,94 +477,6 @@ export default class Transformation {
return finalAssets.concat(resultingAssets);
}

async readFromCache(cacheKey: string): Promise<?Array<UncommittedAsset>> {
if (
this.options.shouldDisableCache ||
this.request.code != null ||
this.request.invalidateReason & FILE_CREATE
) {
return null;
}

let cached = await this.options.cache.get<{|assets: Array<AssetValue>|}>(
cacheKey,
);
if (!cached) {
return null;
}

let cachedAssets = cached.assets;

return Promise.all(
cachedAssets.map(async (value: AssetValue) => {
let content =
value.contentKey != null
? value.isLargeBlob
? this.options.cache.getStream(value.contentKey)
: await this.options.cache.getBlob(value.contentKey)
: null;
let mapBuffer =
value.astKey != null
? await this.options.cache.getBlob(value.astKey)
: null;
let ast =
value.astKey != null
? // TODO: Capture with a test and likely use cache.get() as this returns a buffer.
// $FlowFixMe[incompatible-call]
await this.options.cache.getBlob(value.astKey)
: null;

return new UncommittedAsset({
value,
options: this.options,
content,
mapBuffer,
ast,
});
}),
);
}

async writeToCache(
cacheKey: string,
assets: Array<UncommittedAsset>,
invalidationHash: string,
pipelineHash: string,
): Promise<void> {
await Promise.all(
assets.map(asset => asset.commit(invalidationHash + pipelineHash)),
);

this.options.cache.set(cacheKey, {
$$raw: true,
assets: assets.map(a => a.value),
});
}

getCacheKey(
assets: Array<UncommittedAsset>,
invalidationHash: string,
pipelineHash: string,
): string {
let assetsKeyInfo = assets
.map(a => [
a.value.filePath,
a.value.pipeline,
a.value.hash,
a.value.uniqueKey,
a.value.query ?? '',
])
.join('');

return hashString(
PARCEL_VERSION +
assetsKeyInfo +
this.request.env.id +
invalidationHash +
pipelineHash,
);
}

async loadPipeline(
filePath: ProjectPath,
isSource: boolean,
Expand Down