--iterations and --duration options completely override scenarios

[mem]

Brief summary

If you specify either --iterations or --duration on the command line, this will completely override the scenario configuration provided in the script.

k6 version

v0.51.0

OS

Linux

Docker version and image (if applicable)

N/A

Steps to reproduce the problem

Start with this script:

export const options = {
  scenarios: {
    "scenario_1": {
      executor: "per-vu-iterations",
      vus: 2,
      iterations: 3,
      exec: "scenario_1",
    },

    "scenario_2": {
      executor: "shared-iterations",
      vus: 1,
      iterations: 1,
      exec: "scenario_2",
    },
  },
};

export function scenario_1() {
  console.log("scenario 1");
}

export function scenario_2() {
  console.log("scenario 2");
}

export default function() {
  console.log("default");
}

Run it like this:

$ k6 run ./test.js

          /\      |‾‾| /‾‾/   /‾‾/
     /\  /  \     |  |/  /   /  /
    /  \/    \    |     (   /   ‾‾\
   /          \   |  |\  \ |  (‾)  |
  / __________ \  |__| \__\ \_____/ .io

     execution: local
        script: ./test.js
        output: -

     scenarios: (100.00%) 2 scenarios, 3 max VUs, 10m30s max duration (incl. graceful stop):
              * scenario_1: 3 iterations for each of 2 VUs (maxDuration: 10m0s, exec: scenario_1, gracefulStop: 30s)
              * scenario_2: 1 iterations shared among 1 VUs (maxDuration: 10m0s, exec: scenario_2, gracefulStop: 30s)

INFO[0000] scenario 2                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console

     data_received........: 0 B 0 B/s
     data_sent............: 0 B 0 B/s
     iteration_duration...: avg=45.2µs min=11.6µs med=18.65µs max=131.31µs p(90)=90.1µs p(95)=110.7µs
     iterations...........: 7   28293.231047/s


running (00m00.0s), 0/3 VUs, 7 complete and 0 interrupted iterations
scenario_1 ✓ [======================================] 2 VUs  00m00.0s/10m0s  6/6 iters, 3 per VU
scenario_2 ✓ [======================================] 1 VUs  00m00.0s/10m0s  1/1 shared iters

That's expected.

Now run it like this:

$ k6 run --iterations 1 ./test.js

          /\      |‾‾| /‾‾/   /‾‾/
     /\  /  \     |  |/  /   /  /
    /  \/    \    |     (   /   ‾‾\
   /          \   |  |\  \ |  (‾)  |
  / __________ \  |__| \__\ \_____/ .io

     execution: local
        script: ./test.js
        output: -

     scenarios: (100.00%) 1 scenario, 1 max VUs, 10m30s max duration (incl. graceful stop):
              * default: 1 iterations shared among 1 VUs (maxDuration: 10m0s, gracefulStop: 30s)

INFO[0000] default                                       source=console

     data_received........: 0 B 0 B/s
     data_sent............: 0 B 0 B/s
     iteration_duration...: avg=64.51µs min=64.51µs med=64.51µs max=64.51µs p(90)=64.51µs p(95)=64.51µs
     iterations...........: 1   6904.55148/s


running (00m00.0s), 0/1 VUs, 1 complete and 0 interrupted iterations
default ✓ [======================================] 1 VUs  00m00.0s/10m0s  1/1 shared iters

note the change in console output, and the fact that the scenario changed to default, and the scenarios specified in the script are ignored.

Now run it like this:

$ k6 run --duration 1s ./test.js

          /\      |‾‾| /‾‾/   /‾‾/
     /\  /  \     |  |/  /   /  /
    /  \/    \    |     (   /   ‾‾\
   /          \   |  |\  \ |  (‾)  |
  / __________ \  |__| \__\ \_____/ .io

     execution: local
        script: ./test.js
        output: -

     scenarios: (100.00%) 1 scenario, 1 max VUs, 31s max duration (incl. graceful stop):
              * default: 1 looping VUs for 1s (gracefulStop: 30s)

time="2024-05-14T08:20:01-06:00" level=info msg=default source=console
time="2024-05-14T08:20:01-06:00" level=info msg=default source=console
time="2024-05-14T08:20:01-06:00" level=info msg=default source=console
time="2024-05-14T08:20:01-06:00" level=info msg=default source=console
...
time="2024-05-14T08:20:01-06:00" level=info msg=default source=console
time="2024-05-14T08:20:01-06:00" level=info msg=default source=console

running (01.0s), 1/1 VUs, 943 complete and 0 interrupted iterations
default ↓ [ 100% ] 1 VUs  1s

running (12.0s), 0/1 VUs, 944 complete and 0 interrupted iterations
default ✓ [ 100% ] 1 VUs  1s

     data_received........: 0 B 0 B/s
     data_sent............: 0 B 0 B/s
     iteration_duration...: avg=12.66ms min=5.86µs med=7.05µs max=11.94s p(90)=10.96µs p(95)=13.43µs
     iterations...........: 944 78.942635/s
     vus..................: 1   min=1       max=1
     vus_max..............: 1   min=1       max=1


running (12.0s), 0/1 VUs, 944 complete and 0 interrupted iterations
default ✓ [ 100% ] 1 VUs  1s

Note how the executor changed.

Both flags together:

$ k6 run --duration 1s --iterations 1 ./test.js

          /\      |‾‾| /‾‾/   /‾‾/
     /\  /  \     |  |/  /   /  /
    /  \/    \    |     (   /   ‾‾\
   /          \   |  |\  \ |  (‾)  |
  / __________ \  |__| \__\ \_____/ .io

     execution: local
        script: ./test.js
        output: -

     scenarios: (100.00%) 1 scenario, 1 max VUs, 31s max duration (incl. graceful stop):
              * default: 1 iterations shared among 1 VUs (maxDuration: 1s, gracefulStop: 30s)

INFO[0000] default                                       source=console

     data_received........: 0 B 0 B/s
     data_sent............: 0 B 0 B/s
     iteration_duration...: avg=54.56µs min=54.56µs med=54.56µs max=54.56µs p(90)=54.56µs p(95)=54.56µs
     iterations...........: 1   8077.609674/s


running (00.0s), 0/1 VUs, 1 complete and 0 interrupted iterations
default ✓ [======================================] 1 VUs  0.0s/1s  1/1 shared iters

--iterations 1 seems to override --duration 1s (the executor is "shared iterations" instead of "constant VUs").

Expected behaviour

Not sure, but not this.

I understand that --iterations and --duration are shorthand for specifying a particular scenario configuration. Somehow I expected them to override the iterations / durations for the scenarios that require those values.

The most confusing part about this is that a different function is executed if these options are used.

At a minimum, I would expect something like (not all of them):

• An error if both options are used, because one nullifies the other.
• An error if the scenario configuration in the script is incompatible with the command line flags.
• A warning if scenarios in the script are going to be ignored because of these flags. An error would be better.

Actual behaviour

The scenarios option is completely ignored and replaced with a configuration that matches (?) the command line flags, but it's not obvious to the user that what's going on.

It's even more confusing because --vus doesn't behave like this:

$ k6 run --vus 5 ./test.js

          /\      |‾‾| /‾‾/   /‾‾/
     /\  /  \     |  |/  /   /  /
    /  \/    \    |     (   /   ‾‾\
   /          \   |  |\  \ |  (‾)  |
  / __________ \  |__| \__\ \_____/ .io

     execution: local
        script: ./test.js
        output: -

     scenarios: (100.00%) 2 scenarios, 3 max VUs, 10m30s max duration (incl. graceful stop):
              * scenario_1: 3 iterations for each of 2 VUs (maxDuration: 10m0s, exec: scenario_1, gracefulStop: 30s)
              * scenario_2: 1 iterations shared among 1 VUs (maxDuration: 10m0s, exec: scenario_2, gracefulStop: 30s)

INFO[0000] scenario 2                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console
INFO[0000] scenario 1                                    source=console

     data_received........: 0 B 0 B/s
     data_sent............: 0 B 0 B/s
     iteration_duration...: avg=58.88µs min=16.51µs med=46.03µs max=125.61µs p(90)=122.86µs p(95)=124.23µs
     iterations...........: 7   22789.796682/s


running (00m00.0s), 0/3 VUs, 7 complete and 0 interrupted iterations
scenario_1 ✓ [======================================] 2 VUs  00m00.0s/10m0s  6/6 iters, 3 per VU
scenario_2 ✓ [======================================] 1 VUs  00m00.0s/10m0s  1/1 shared iters

--vus is simply ignored. It looks like --vus is only used if --iterations or --duration is used.

And it's even more confusing because with a script like this:

export const options = {
  scenarios: {
    "default": {
      executor: "shared-iterations",
      vus: 2,
      iterations: 3,
      exec: "default",
    },
  },
};

export default function() {
  console.log("default");
}

it looks like it's doing what you'd expect (override the "iterations" value in the scenario):

$ k6 run --iterations 10 ./test.js

          /\      |‾‾| /‾‾/   /‾‾/
     /\  /  \     |  |/  /   /  /
    /  \/    \    |     (   /   ‾‾\
   /          \   |  |\  \ |  (‾)  |
  / __________ \  |__| \__\ \_____/ .io

     execution: local
        script: ./test-2.js
        output: -

     scenarios: (100.00%) 1 scenario, 1 max VUs, 10m30s max duration (incl. graceful stop):
              * default: 10 iterations shared among 1 VUs (maxDuration: 10m0s, gracefulStop: 30s)

INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console
INFO[0000] default                                       source=console

     data_received........: 0 B 0 B/s
     data_sent............: 0 B 0 B/s
     iteration_duration...: avg=20.28µs min=12.2µs med=17.49µs max=53.8µs p(90)=24.27µs p(95)=39.03µs
     iterations...........: 10  32778.824879/s


running (00m00.0s), 0/1 VUs, 10 complete and 0 interrupted iterations
default ✓ [======================================] 1 VUs  00m00.0s/10m0s  10/10 shared iters

you can tell that it's not doing that because it's ignoring the vus setting in the script (it goes from 2 without flags to 1 with this flag). This is because it's discarding the scenario configuration in the script.

[mem]

(adding the ux label because to me this is UX; please remove if that's inappropriate for the current usage of that label)