Skip to content

Commit 773bda8

Browse files
authoredJan 8, 2025··
Remove defaults for batch-max-* pipeline parameters and define value ranges (#7576)
* Update help message for `batch-max-*` pipeline parameters Closes https://jira.cfdata.org/browse/PIPE-152 The defaults are currently in flux and changing as we determine what makes sense for the system. To support these changing values, remove the defaults from the client as they will be set by the API. The values used to create the pipeline are visible to user when doing `wrangler pipeline show <name>`. Create three-chefs-bathe.md * Covert MB to bytes for the API
1 parent 17c2cdd commit 773bda8

File tree

4 files changed

+17
-12
lines changed

4 files changed

+17
-12
lines changed
 

‎.changeset/three-chefs-bathe.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"wrangler": patch
3+
---
4+
5+
Remove defaults for `batch-max-*` pipeline parameters and define value ranges

‎packages/wrangler/src/__tests__/pipelines.test.ts

+3-6
Original file line numberDiff line numberDiff line change
@@ -360,12 +360,9 @@ describe("pipelines", () => {
360360
OPTIONS
361361
--secret-access-key The R2 service token Access Key to write data [string]
362362
--access-key-id The R2 service token Secret Key to write data [string]
363-
--batch-max-mb The approximate maximum size of a batch before flush in megabytes
364-
Default: 10 [number]
365-
--batch-max-rows The approximate maximum size of a batch before flush in rows
366-
Default: 10000 [number]
367-
--batch-max-seconds The approximate maximum duration of a batch before flush in seconds
368-
Default: 15 [number]
363+
--batch-max-mb The approximate maximum size (in megabytes) for each batch before flushing (range: 1 - 100) [number]
364+
--batch-max-rows The approximate maximum number of rows in a batch before flushing (range: 100 - 1000000) [number]
365+
--batch-max-seconds The approximate maximum age (in seconds) of a batch before flushing (range: 1 - 300) [number]
369366
--transform The worker and entrypoint of the PipelineTransform implementation in the format \\"worker.entrypoint\\"
370367
Default: No transformation worker [string]
371368
--compression Sets the compression format of output files

‎packages/wrangler/src/pipelines/client.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export type PipelineUserConfig = {
3939
};
4040
batch: {
4141
max_duration_s?: number;
42-
max_mb?: number;
42+
max_bytes?: number;
4343
max_rows?: number;
4444
};
4545
path: {

‎packages/wrangler/src/pipelines/index.ts

+8-5
Original file line numberDiff line numberDiff line change
@@ -101,19 +101,19 @@ function addCreateAndUpdateOptions(yargs: Argv<CommonYargsOptions>) {
101101
})
102102
.option("batch-max-mb", {
103103
describe:
104-
"The approximate maximum size of a batch before flush in megabytes \nDefault: 10",
104+
"The approximate maximum size (in megabytes) for each batch before flushing (range: 1 - 100)",
105105
type: "number",
106106
demandOption: false,
107107
})
108108
.option("batch-max-rows", {
109109
describe:
110-
"The approximate maximum size of a batch before flush in rows \nDefault: 10000",
110+
"The approximate maximum number of rows in a batch before flushing (range: 100 - 1000000)",
111111
type: "number",
112112
demandOption: false,
113113
})
114114
.option("batch-max-seconds", {
115115
describe:
116-
"The approximate maximum duration of a batch before flush in seconds \nDefault: 15",
116+
"The approximate maximum age (in seconds) of a batch before flushing (range: 1 - 300)",
117117
type: "number",
118118
demandOption: false,
119119
})
@@ -196,7 +196,9 @@ export function pipelines(pipelineYargs: CommonYargsArgv) {
196196
args.compression === undefined ? "gzip" : args.compression;
197197

198198
const batch = {
199-
max_mb: args["batch-max-mb"],
199+
max_bytes: args["batch-max-mb"]
200+
? args["batch-max-mb"] * 1000 * 1000 // convert to bytes for the API
201+
: undefined,
200202
max_duration_s: args["batch-max-seconds"],
201203
max_rows: args["batch-max-rows"],
202204
};
@@ -386,7 +388,8 @@ export function pipelines(pipelineYargs: CommonYargsArgv) {
386388
pipelineConfig.destination.compression.type = args.compression;
387389
}
388390
if (args["batch-max-mb"]) {
389-
pipelineConfig.destination.batch.max_mb = args["batch-max-mb"];
391+
pipelineConfig.destination.batch.max_bytes =
392+
args["batch-max-mb"] * 1000 * 1000; // convert to bytes for the API
390393
}
391394
if (args["batch-max-seconds"]) {
392395
pipelineConfig.destination.batch.max_duration_s =

0 commit comments

Comments
 (0)
Please sign in to comment.