Skip to content

Commit a53fd34

Browse files
jadedevin13sindresorhussholladay
authoredMar 10, 2025··
Add onUploadProgress option (#632)
Co-authored-by: Sindre Sorhus <sindresorhus@gmail.com> Co-authored-by: Seth Holladay <me@seth-holladay.com>
1 parent f9a9f38 commit a53fd34

File tree

10 files changed

+391
-68
lines changed

10 files changed

+391
-68
lines changed
 

‎readme.md

+33-3
Original file line numberDiff line numberDiff line change
@@ -404,9 +404,12 @@ Type: `Function`
404404

405405
Download progress event handler.
406406

407-
The function receives a `progress` and `chunk` argument:
408-
- The `progress` object contains the following elements: `percent`, `transferredBytes` and `totalBytes`. If it's not possible to retrieve the body size, `totalBytes` will be `0`.
409-
- The `chunk` argument is an instance of `Uint8Array`. It's empty for the first call.
407+
The function receives these arguments:
408+
- `progress` is an object with the these properties:
409+
- - `percent` is a number between 0 and 1 representing the progress percentage.
410+
- - `transferredBytes` is the number of bytes transferred so far.
411+
- - `totalBytes` is the total number of bytes to be transferred. This is an estimate and may be 0 if the total size cannot be determined.
412+
- `chunk` is an instance of `Uint8Array` containing the data that was sent. Note: It's empty for the first call.
410413

411414
```js
412415
import ky from 'ky';
@@ -421,6 +424,33 @@ const response = await ky('https://example.com', {
421424
});
422425
```
423426

427+
##### onUploadProgress
428+
429+
Type: `Function`
430+
431+
Upload progress event handler.
432+
433+
The function receives these arguments:
434+
- `progress` is an object with the these properties:
435+
- - `percent` is a number between 0 and 1 representing the progress percentage.
436+
- - `transferredBytes` is the number of bytes transferred so far.
437+
- - `totalBytes` is the total number of bytes to be transferred. This is an estimate and may be 0 if the total size cannot be determined.
438+
- `chunk` is an instance of `Uint8Array` containing the data that was sent. Note: It's empty for the last call.
439+
440+
```js
441+
import ky from 'ky';
442+
443+
const response = await ky.post('https://example.com/upload', {
444+
body: largeFile,
445+
onUploadProgress: (progress, chunk) => {
446+
// Example output:
447+
// `0% - 0 of 1271 bytes`
448+
// `100% - 1271 of 1271 bytes`
449+
console.log(`${progress.percent * 100}% - ${progress.transferredBytes} of ${progress.totalBytes} bytes`);
450+
}
451+
});
452+
```
453+
424454
##### parseJson
425455

426456
Type: `Function`\

‎source/core/Ky.ts

+18-59
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import type {
88
SearchParamsInit,
99
} from '../types/options.js';
1010
import {type ResponsePromise} from '../types/ResponsePromise.js';
11+
import {streamRequest, streamResponse} from '../utils/body.js';
1112
import {mergeHeaders, mergeHooks} from '../utils/merge.js';
1213
import {normalizeRequestMethod, normalizeRetryOptions} from '../utils/normalize.js';
1314
import timeout, {type TimeoutOptions} from '../utils/timeout.js';
@@ -64,7 +65,6 @@ export class Ky {
6465
}
6566

6667
// If `onDownloadProgress` is passed, it uses the stream API internally
67-
/* istanbul ignore next */
6868
if (ky._options.onDownloadProgress) {
6969
if (typeof ky._options.onDownloadProgress !== 'function') {
7070
throw new TypeError('The `onDownloadProgress` option must be a function');
@@ -74,7 +74,7 @@ export class Ky {
7474
throw new Error('Streams are not supported in your environment. `ReadableStream` is missing.');
7575
}
7676

77-
return ky._stream(response.clone(), ky._options.onDownloadProgress);
77+
return streamResponse(response.clone(), ky._options.onDownloadProgress);
7878
}
7979

8080
return response;
@@ -205,6 +205,22 @@ export class Ky {
205205
// The spread of `this.request` is required as otherwise it misses the `duplex` option for some reason and throws.
206206
this.request = new globalThis.Request(new globalThis.Request(url, {...this.request}), this._options as RequestInit);
207207
}
208+
209+
// If `onUploadProgress` is passed, it uses the stream API internally
210+
if (this._options.onUploadProgress) {
211+
if (typeof this._options.onUploadProgress !== 'function') {
212+
throw new TypeError('The `onUploadProgress` option must be a function');
213+
}
214+
215+
if (!supportsRequestStreams) {
216+
throw new Error('Request streams are not supported in your environment. The `duplex` option for `Request` is not available.');
217+
}
218+
219+
const originalBody = this.request.body;
220+
if (originalBody) {
221+
this.request = streamRequest(this.request, this._options.onUploadProgress);
222+
}
223+
}
208224
}
209225

210226
protected _calculateRetryDelay(error: unknown) {
@@ -310,61 +326,4 @@ export class Ky {
310326

311327
return timeout(mainRequest, nonRequestOptions, this.abortController, this._options as TimeoutOptions);
312328
}
313-
314-
/* istanbul ignore next */
315-
protected _stream(response: Response, onDownloadProgress: Options['onDownloadProgress']) {
316-
const totalBytes = Number(response.headers.get('content-length')) || 0;
317-
let transferredBytes = 0;
318-
319-
if (response.status === 204) {
320-
if (onDownloadProgress) {
321-
onDownloadProgress({percent: 1, totalBytes, transferredBytes}, new Uint8Array());
322-
}
323-
324-
return new globalThis.Response(
325-
null,
326-
{
327-
status: response.status,
328-
statusText: response.statusText,
329-
headers: response.headers,
330-
},
331-
);
332-
}
333-
334-
return new globalThis.Response(
335-
new globalThis.ReadableStream({
336-
async start(controller) {
337-
const reader = response.body!.getReader();
338-
339-
if (onDownloadProgress) {
340-
onDownloadProgress({percent: 0, transferredBytes: 0, totalBytes}, new Uint8Array());
341-
}
342-
343-
async function read() {
344-
const {done, value} = await reader.read();
345-
if (done) {
346-
controller.close();
347-
return;
348-
}
349-
350-
if (onDownloadProgress) {
351-
transferredBytes += value.byteLength;
352-
const percent = totalBytes === 0 ? 0 : transferredBytes / totalBytes;
353-
onDownloadProgress({percent, transferredBytes, totalBytes}, value);
354-
}
355-
356-
controller.enqueue(value);
357-
await read();
358-
}
359-
360-
await read();
361-
},
362-
}),
363-
{
364-
status: response.status,
365-
statusText: response.statusText,
366-
headers: response.headers,
367-
},
368-
);
369-
}
370329
}

‎source/core/constants.ts

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ export const responseTypes = {
5454
// The maximum value of a 32bit int (see issue #117)
5555
export const maxSafeTimeout = 2_147_483_647;
5656

57+
// Size in bytes of a typical form boundary, used to help estimate upload size
58+
export const usualFormBoundarySize = new TextEncoder().encode('------WebKitFormBoundaryaxpyiPgbbPti10Rw').length;
59+
5760
export const stop = Symbol('stop');
5861

5962
export const kyOptionKeys: KyOptionsRegistry = {
@@ -67,6 +70,7 @@ export const kyOptionKeys: KyOptionsRegistry = {
6770
hooks: true,
6871
throwHttpErrors: true,
6972
onDownloadProgress: true,
73+
onUploadProgress: true,
7074
fetch: true,
7175
};
7276

‎source/index.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export type {
4242
NormalizedOptions,
4343
RetryOptions,
4444
SearchParamsOption,
45-
DownloadProgress,
45+
Progress,
4646
} from './types/options.js';
4747

4848
export type {

‎source/types/options.ts

+28-3
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export type HttpMethod = 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete';
1212

1313
export type Input = string | URL | Request;
1414

15-
export type DownloadProgress = {
15+
export type Progress = {
1616
percent: number;
1717
transferredBytes: number;
1818

@@ -170,7 +170,8 @@ export type KyOptions = {
170170
/**
171171
Download progress event handler.
172172
173-
@param chunk - Note: It's empty for the first call.
173+
@param progress - Object containing download progress information.
174+
@param chunk - Data that was received. Note: It's empty for the first call.
174175
175176
@example
176177
```
@@ -186,7 +187,30 @@ export type KyOptions = {
186187
});
187188
```
188189
*/
189-
onDownloadProgress?: (progress: DownloadProgress, chunk: Uint8Array) => void;
190+
onDownloadProgress?: (progress: Progress, chunk: Uint8Array) => void;
191+
192+
/**
193+
Upload progress event handler.
194+
195+
@param progress - Object containing upload progress information.
196+
@param chunk - Data that was sent. Note: It's empty for the last call.
197+
198+
@example
199+
```
200+
import ky from 'ky';
201+
202+
const response = await ky.post('https://example.com/upload', {
203+
body: largeFile,
204+
onUploadProgress: (progress, chunk) => {
205+
// Example output:
206+
// `0% - 0 of 1271 bytes`
207+
// `100% - 1271 of 1271 bytes`
208+
console.log(`${progress.percent * 100}% - ${progress.transferredBytes} of ${progress.totalBytes} bytes`);
209+
}
210+
});
211+
```
212+
*/
213+
onUploadProgress?: (progress: Progress, chunk: Uint8Array) => void;
190214

191215
/**
192216
User-defined `fetch` function.
@@ -287,6 +311,7 @@ export interface NormalizedOptions extends RequestInit { // eslint-disable-line
287311
retry: RetryOptions;
288312
prefixUrl: string;
289313
onDownloadProgress: Options['onDownloadProgress'];
314+
onUploadProgress: Options['onUploadProgress'];
290315
}
291316

292317
export type {RetryOptions} from './retry.js';

‎source/utils/body.ts

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
import type {Options} from '../types/options.js';
2+
import {usualFormBoundarySize} from '../core/constants.js';
3+
4+
// eslint-disable-next-line @typescript-eslint/ban-types
5+
export const getBodySize = (body?: BodyInit | null): number => {
6+
if (!body) {
7+
return 0;
8+
}
9+
10+
if (body instanceof FormData) {
11+
// This is an approximation, as FormData size calculation is not straightforward
12+
let size = 0;
13+
14+
for (const [key, value] of body) {
15+
size += usualFormBoundarySize;
16+
size += new TextEncoder().encode(`Content-Disposition: form-data; name="${key}"`).length;
17+
size += typeof value === 'string'
18+
? new TextEncoder().encode(value).length
19+
: value.size;
20+
}
21+
22+
return size;
23+
}
24+
25+
if (body instanceof Blob) {
26+
return body.size;
27+
}
28+
29+
if (body instanceof ArrayBuffer) {
30+
return body.byteLength;
31+
}
32+
33+
if (typeof body === 'string') {
34+
return new TextEncoder().encode(body).length;
35+
}
36+
37+
if (body instanceof URLSearchParams) {
38+
return new TextEncoder().encode(body.toString()).length;
39+
}
40+
41+
if ('byteLength' in body) {
42+
return (body).byteLength;
43+
}
44+
45+
if (typeof body === 'object' && body !== null) {
46+
try {
47+
const jsonString = JSON.stringify(body);
48+
return new TextEncoder().encode(jsonString).length;
49+
} catch {
50+
return 0;
51+
}
52+
}
53+
54+
return 0; // Default case, unable to determine size
55+
};
56+
57+
export const streamResponse = (response: Response, onDownloadProgress: Options['onDownloadProgress']) => {
58+
const totalBytes = Number(response.headers.get('content-length')) || 0;
59+
let transferredBytes = 0;
60+
61+
if (response.status === 204) {
62+
if (onDownloadProgress) {
63+
onDownloadProgress({percent: 1, totalBytes, transferredBytes}, new Uint8Array());
64+
}
65+
66+
return new Response(
67+
null,
68+
{
69+
status: response.status,
70+
statusText: response.statusText,
71+
headers: response.headers,
72+
},
73+
);
74+
}
75+
76+
return new Response(
77+
new ReadableStream({
78+
async start(controller) {
79+
const reader = response.body!.getReader();
80+
81+
if (onDownloadProgress) {
82+
onDownloadProgress({percent: 0, transferredBytes: 0, totalBytes}, new Uint8Array());
83+
}
84+
85+
async function read() {
86+
const {done, value} = await reader.read();
87+
if (done) {
88+
controller.close();
89+
return;
90+
}
91+
92+
if (onDownloadProgress) {
93+
transferredBytes += value.byteLength;
94+
const percent = totalBytes === 0 ? 0 : transferredBytes / totalBytes;
95+
onDownloadProgress({percent, transferredBytes, totalBytes}, value);
96+
}
97+
98+
controller.enqueue(value);
99+
await read();
100+
}
101+
102+
await read();
103+
},
104+
}),
105+
{
106+
status: response.status,
107+
statusText: response.statusText,
108+
headers: response.headers,
109+
},
110+
);
111+
};
112+
113+
export const streamRequest = (request: Request, onUploadProgress: Options['onUploadProgress']) => {
114+
const totalBytes = getBodySize(request.body);
115+
let transferredBytes = 0;
116+
117+
return new Request(request, {
118+
// @ts-expect-error - Types are outdated.
119+
duplex: 'half',
120+
body: new ReadableStream({
121+
async start(controller) {
122+
const reader = request.body instanceof ReadableStream ? request.body.getReader() : new Response('').body!.getReader();
123+
124+
async function read() {
125+
const {done, value} = await reader.read();
126+
if (done) {
127+
// Ensure 100% progress is reported when the upload is complete
128+
if (onUploadProgress) {
129+
onUploadProgress({percent: 1, transferredBytes, totalBytes: Math.max(totalBytes, transferredBytes)}, new Uint8Array());
130+
}
131+
132+
controller.close();
133+
return;
134+
}
135+
136+
transferredBytes += value.byteLength;
137+
let percent = totalBytes === 0 ? 0 : transferredBytes / totalBytes;
138+
if (totalBytes < transferredBytes || percent === 1) {
139+
percent = 0.99;
140+
}
141+
142+
if (onUploadProgress) {
143+
onUploadProgress({percent: Number(percent.toFixed(2)), transferredBytes, totalBytes}, value);
144+
}
145+
146+
controller.enqueue(value);
147+
await read();
148+
}
149+
150+
await read();
151+
},
152+
}),
153+
});
154+
};

‎test/browser.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import busboy from 'busboy';
33
import express from 'express';
44
import {chromium, webkit, type Page} from 'playwright';
55
import type ky from '../source/index.js'; // eslint-disable-line import/no-duplicates
6-
import type {DownloadProgress} from '../source/index.js'; // eslint-disable-line import/no-duplicates
6+
import type {Progress} from '../source/index.js'; // eslint-disable-line import/no-duplicates
77
import {createHttpTestServer, type ExtendedHttpTestServer, type HttpServerOptions} from './helpers/create-http-test-server.js';
88
import {parseRawBody} from './helpers/parse-body.js';
99
import {browserTest, defaultBrowsersTest} from './helpers/with-page.js';
@@ -265,7 +265,7 @@ browserTest('onDownloadProgress works', [chromium, webkit], async (t: ExecutionC
265265
await addKyScriptToPage(page);
266266

267267
const result = await page.evaluate(async (url: string) => {
268-
const data: Array<Array<(DownloadProgress | string)>> = [];
268+
const data: Array<Array<(Progress | string)>> = [];
269269
const text = await window
270270
.ky(url, {
271271
onDownloadProgress(progress, chunk) {

‎test/helpers/create-large-file.ts

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Helper function to create a large Blob
2+
export function createLargeBlob(sizeInMB: number): Blob {
3+
const chunkSize = 1024 * 1024; // 1MB
4+
// eslint-disable-next-line unicorn/no-new-array
5+
const chunks = new Array(sizeInMB).fill('x'.repeat(chunkSize));
6+
return new Blob(chunks, {type: 'application/octet-stream'});
7+
}

‎test/helpers/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
export * from './create-http-test-server.js';
22
export * from './parse-body.js';
33
export * from './with-page.js';
4+
export * from './create-large-file.js';

‎test/stream.ts

+143
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import test from 'ava';
2+
import ky, {type Progress} from '../source/index.js';
3+
import {createLargeBlob} from './helpers/create-large-file.js';
4+
import {createHttpTestServer} from './helpers/create-http-test-server.js';
5+
import {parseRawBody} from './helpers/parse-body.js';
6+
7+
test('POST JSON with upload progress', async t => {
8+
const server = await createHttpTestServer({bodyParser: false});
9+
server.post('/', async (request, response) => {
10+
response.json(await parseRawBody(request));
11+
});
12+
13+
const json = {test: 'test'};
14+
const data: Progress[] = [];
15+
const chunks: string[] = [];
16+
const responseJson = await ky
17+
.post(server.url, {
18+
json,
19+
onUploadProgress(progress, chunk) {
20+
data.push(progress);
21+
chunks.push(new TextDecoder().decode(chunk));
22+
},
23+
})
24+
.json();
25+
26+
// Check if we have at least two progress updates
27+
t.true(data.length >= 2, 'Should have at least two progress updates');
28+
t.deepEqual(
29+
chunks,
30+
[
31+
'{"test":"test"}',
32+
'',
33+
],
34+
'Should have chunks for all but the last event',
35+
);
36+
37+
// Check the first progress update
38+
t.true(
39+
data[0].percent >= 0 && data[0].percent < 1,
40+
'First update should have progress between 0 and 100%',
41+
);
42+
t.true(
43+
data[0].transferredBytes >= 0,
44+
'First update should have non-negative transferred bytes',
45+
);
46+
47+
// Check intermediate updates (if any)
48+
for (let i = 1; i < data.length - 1; i++) {
49+
t.true(
50+
data[i].percent >= data[i - 1].percent,
51+
`Update ${i} should have higher or equal percent than previous`,
52+
);
53+
t.true(
54+
data[i].transferredBytes >= data[i - 1].transferredBytes,
55+
`Update ${i} should have more or equal transferred bytes than previous`,
56+
);
57+
}
58+
59+
// Check the last progress update
60+
const lastUpdate = data.at(-1);
61+
t.is(lastUpdate.percent, 1, 'Last update should have 100% progress');
62+
t.true(
63+
lastUpdate.totalBytes > 0,
64+
'Last update should have positive total bytes',
65+
);
66+
t.is(
67+
lastUpdate.transferredBytes,
68+
lastUpdate.totalBytes,
69+
'Last update should have transferred all bytes',
70+
);
71+
72+
await server.close();
73+
});
74+
75+
test('POST FormData with 10MB file upload progress', async t => {
76+
const server = await createHttpTestServer({bodyParser: false});
77+
server.post('/', async (request, response) => {
78+
let totalBytes = 0;
79+
for await (const chunk of request) {
80+
totalBytes += chunk.length as number;
81+
}
82+
83+
response.json({receivedBytes: totalBytes});
84+
});
85+
86+
const largeBlob = createLargeBlob(10); // 10MB Blob
87+
const formData = new FormData();
88+
formData.append('file', largeBlob, 'large-file.bin');
89+
90+
const data: Array<{
91+
percent: number;
92+
transferredBytes: number;
93+
totalBytes: number;
94+
}> = [];
95+
const response = await ky
96+
.post(server.url, {
97+
body: formData,
98+
onUploadProgress(progress) {
99+
data.push(progress);
100+
},
101+
})
102+
.json<{receivedBytes: number}>();
103+
104+
// Check if we have at least two progress updates
105+
t.true(data.length >= 2, 'Should have at least two progress updates');
106+
107+
// Check the first progress update
108+
t.true(
109+
data[0].percent >= 0 && data[0].percent < 1,
110+
'First update should have progress between 0 and 100%',
111+
);
112+
t.true(
113+
data[0].transferredBytes >= 0,
114+
'First update should have non-negative transferred bytes',
115+
);
116+
117+
// Check intermediate updates (if any)
118+
for (let i = 1; i < data.length - 1; i++) {
119+
t.true(
120+
data[i].percent >= data[i - 1].percent,
121+
`Update ${i} should have higher or equal percent than previous`,
122+
);
123+
t.true(
124+
data[i].transferredBytes >= data[i - 1].transferredBytes,
125+
`Update ${i} should have more or equal transferred bytes than previous`,
126+
);
127+
}
128+
129+
// Check the last progress update
130+
const lastUpdate = data.at(-1);
131+
t.is(lastUpdate.percent, 1, 'Last update should have 100% progress');
132+
t.true(
133+
lastUpdate.totalBytes > 0,
134+
'Last update should have positive total bytes',
135+
);
136+
t.is(
137+
lastUpdate.transferredBytes,
138+
lastUpdate.totalBytes,
139+
'Last update should have transferred all bytes',
140+
);
141+
142+
await server.close();
143+
});

0 commit comments

Comments
 (0)
Please sign in to comment.