Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: improve serialization for errors and bigints #17282

Merged
merged 6 commits into from Jun 5, 2023
Merged

feat: improve serialization for errors and bigints #17282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e90bd20
feat: support `cause` for errors in serialization
alexander-akait May 29, 2023
b1dac04
feat: support bigint serialization
alexander-akait May 30, 2023
11ed4ab
feat: less size for short bigint
alexander-akait May 30, 2023
0ea2e99
refactor: store some bigints as numbers
alexander-akait May 30, 2023
7333777
test: cover all cases
alexander-akait May 30, 2023
a5e9568
chore: docs update
alexander-akait May 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions .eslintrc.js
View file
Open in desktop
Expand Up @@ -109,6 +109,9 @@ module.exports = {
env: {
"jest/globals": true
},
parserOptions: {
ecmaVersion: 2020
},
TheLarkInn marked this conversation as resolved.
Show resolved Hide resolved
globals: {
nsObj: false,
jasmine: false
Expand Down
144 changes: 143 additions & 1 deletion lib/serialization/BinaryMiddleware.js
View file
Open in desktop
Expand Up @@ -21,6 +21,9 @@ Section -> NullsSection |
I32NumbersSection |
I8NumbersSection |
ShortStringSection |
BigIntSection |
I32BigIntSection |
I8BigIntSection
StringSection |
BufferSection |
NopSection
Expand All @@ -39,6 +42,9 @@ ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
StringSection -> StringSectionHeaderByte i32:length utf8-byte*
BufferSection -> BufferSectionHeaderByte i32:length byte*
NopSection --> NopSectionHeaderByte
BigIntSection -> BigIntSectionHeaderByte i32:length ascii-byte*
I32BigIntSection -> I32BigIntSectionHeaderByte i32
I8BigIntSection -> I8BigIntSectionHeaderByte i8

ShortStringSectionHeaderByte -> 0b1nnn_nnnn (n:length)

Expand All @@ -58,6 +64,9 @@ BooleansCountAndBitsByte ->
StringSectionHeaderByte -> 0b0000_1110
BufferSectionHeaderByte -> 0b0000_1111
NopSectionHeaderByte -> 0b0000_1011
BigIntSectionHeaderByte -> 0b0001_1010
I32BigIntSectionHeaderByte -> 0b0001_1100
I8BigIntSectionHeaderByte -> 0b0001_1011
FalseHeaderByte -> 0b0000_1100
TrueHeaderByte -> 0b0000_1101

Expand All @@ -78,6 +87,9 @@ const NULL_AND_I8_HEADER = 0x15;
const NULL_AND_I32_HEADER = 0x16;
const NULL_AND_TRUE_HEADER = 0x17;
const NULL_AND_FALSE_HEADER = 0x18;
const BIGINT_HEADER = 0x1a;
const BIGINT_I8_HEADER = 0x1b;
const BIGINT_I32_HEADER = 0x1c;
const STRING_HEADER = 0x1e;
const BUFFER_HEADER = 0x1f;
const I8_HEADER = 0x60;
Expand All @@ -86,7 +98,7 @@ const F64_HEADER = 0x20;
const SHORT_STRING_HEADER = 0x80;

/** Uplift high-order bits */
const NUMBERS_HEADER_MASK = 0xe0;
const NUMBERS_HEADER_MASK = 0xe0; // 0b1010_0000
const NUMBERS_COUNT_MASK = 0x1f; // 0b0001_1111
const SHORT_STRING_LENGTH_MASK = 0x7f; // 0b0111_1111

Expand All @@ -113,6 +125,16 @@ const identifyNumber = n => {
return 2;
};

/**
* @param {bigint} n bigint
* @returns {0 | 1 | 2} type of bigint for serialization
*/
const identifyBigInt = n => {
if (n <= BigInt(127) && n >= BigInt(-128)) return 0;
if (n <= BigInt(2147483647) && n >= BigInt(-2147483648)) return 1;
return 2;
};

/**
* @typedef {PrimitiveSerializableType[]} DeserializedType
* @typedef {BufferSerializableType[]} SerializedType
Expand Down Expand Up @@ -331,6 +353,62 @@ class BinaryMiddleware extends SerializerMiddleware {
}
break;
}
case "bigint": {
const type = identifyBigInt(thing);
if (type === 0 && thing >= 0 && thing <= BigInt(10)) {
// shortcut for very small bigints
allocate(HEADER_SIZE + I8_SIZE);
writeU8(BIGINT_I8_HEADER);
writeU8(Number(thing));
break;
}

switch (type) {
case 0: {
let n = 1;
allocate(HEADER_SIZE + I8_SIZE * n);
writeU8(BIGINT_I8_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeInt8(
Number(/** @type {bigint} */ (data[i])),
currentPosition
);
currentPosition += I8_SIZE;
n--;
i++;
}
i--;
break;
}
case 1: {
let n = 1;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In theory it can be optimized and we can write sections, but it requires a lot of work and our serialization optimized for numbers and strings (they are used more often than BigInts), also it will cause a big cache broken errors on CI (i.e. if you have cache from old webpack version and the new version of webpack can't deserialize because we will use new bytes for headers)

allocate(HEADER_SIZE + I32_SIZE * n);
writeU8(BIGINT_I32_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeInt32LE(
Number(/** @type {bigint} */ (data[i])),
currentPosition
);
currentPosition += I32_SIZE;
n--;
i++;
}
i--;
break;
}
default: {
const value = thing.toString();
const len = Buffer.byteLength(value);
allocate(len + HEADER_SIZE + I32_SIZE);
writeU8(BIGINT_HEADER);
writeU32(len);
currentBuffer.write(value, currentPosition);
currentPosition += len;
break;
}
}
break;
}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why it is storing as string:

  • we can't use writeBigInt64LE, because BigInt can be longer than 64bits
  • storing by bytes (uint8 or other) will require multiple/shift them in deserialization
  • we can use thing.toString(16) to reduce size, but I doesn't bring a lot of wins, 4294967296n.toString(16) -> 100000000, also it will require to store sing (-/+) and will multiple on -1

so storing them as string, I think, is the best solution

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Improved, now we bigints as int8 (when n <= BigInt(127) && n >= BigInt(-128)) and as int32 (when n <= BigInt(2147483647) && n >= BigInt(-2147483648)), other bigints we store as a string with ascii characters, so you can store any bigint

case "number": {
const type = identifyNumber(thing);
if (type === 0 && thing >= 0 && thing <= 10) {
Expand Down Expand Up @@ -847,6 +925,70 @@ class BinaryMiddleware extends SerializerMiddleware {
result.push(read(1).readInt8(0));
}
};
case BIGINT_I8_HEADER: {
const len = 1;
return () => {
const need = I8_SIZE * len;

if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
const value =
/** @type {Buffer} */
(currentBuffer).readInt8(currentPosition);
result.push(BigInt(value));
currentPosition += I8_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
const value = buf.readInt8(i * I8_SIZE);
result.push(BigInt(value));
}
}
};
}
case BIGINT_I32_HEADER: {
const len = 1;
return () => {
const need = I32_SIZE * len;
if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
const value = /** @type {Buffer} */ (currentBuffer).readInt32LE(
currentPosition
);
result.push(BigInt(value));
currentPosition += I32_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
const value = buf.readInt32LE(i * I32_SIZE);
result.push(BigInt(value));
}
}
};
}
case BIGINT_HEADER: {
return () => {
const len = readU32();
if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
const value = currentBuffer.toString(
undefined,
currentPosition,
currentPosition + len
);

result.push(BigInt(value));
currentPosition += len;
checkOverflow();
} else {
const value = read(len).toString();
result.push(BigInt(value));
}
};
}
default:
if (header <= 10) {
return () => result.push(header);
Expand Down
3 changes: 3 additions & 0 deletions lib/serialization/ErrorObjectSerializer.js
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@ class ErrorObjectSerializer {
serialize(obj, context) {
context.write(obj.message);
context.write(obj.stack);
context.write(/** @type {Error & { cause: "unknown" }} */ (obj).cause);
}
/**
* @param {ObjectDeserializerContext} context context
Expand All @@ -31,6 +32,8 @@ class ErrorObjectSerializer {

err.message = context.read();
err.stack = context.read();
/** @type {Error & { cause: "unknown" }} */
(err).cause = context.read();

return err;
}
Expand Down
3 changes: 3 additions & 0 deletions lib/serialization/ObjectMiddleware.js
View file
Open in desktop
Expand Up @@ -397,6 +397,9 @@ class ObjectMiddleware extends SerializerMiddleware {
", "
)} }`;
}
if (typeof item === "bigint") {
return `BigInt ${item}n`;
}
try {
return `${item}`;
} catch (e) {
Expand Down
2 changes: 1 addition & 1 deletion lib/serialization/types.js
View file
Open in desktop
Expand Up @@ -6,7 +6,7 @@

/** @typedef {undefined|null|number|string|boolean|Buffer|Object|(() => ComplexSerializableType[] | Promise<ComplexSerializableType[]>)} ComplexSerializableType */

/** @typedef {undefined|null|number|string|boolean|Buffer|(() => PrimitiveSerializableType[] | Promise<PrimitiveSerializableType[]>)} PrimitiveSerializableType */
/** @typedef {undefined|null|number|bigint|string|boolean|Buffer|(() => PrimitiveSerializableType[] | Promise<PrimitiveSerializableType[]>)} PrimitiveSerializableType */

/** @typedef {Buffer|(() => BufferSerializableType[] | Promise<BufferSerializableType[]>)} BufferSerializableType */

Expand Down
95 changes: 95 additions & 0 deletions test/Compiler-filesystem-caching.test.js
View file
Open in desktop
Expand Up @@ -41,6 +41,101 @@ describe("Compiler (filesystem caching)", () => {
]
};

const isBigIntSupported = typeof BigInt !== "undefined";
const isErrorCaseSupported =
typeof new Error("test", { cause: new Error("cause") }).cause !==
"undefined";

options.plugins = [
{
apply(compiler) {
const name = "TestCachePlugin";

compiler.hooks.thisCompilation.tap(name, compilation => {
compilation.hooks.processAssets.tapPromise(
{
name,
stage:
compiler.webpack.Compilation
.PROCESS_ASSETS_STAGE_OPTIMIZE_SIZE
},
async () => {
const cache = compilation.getCache(name);
const ident = "test.ext";
const cacheItem = cache.getItemCache(ident, null);

const result = await cacheItem.getPromise(ident);

if (result) {
expect(result.number).toEqual(42);
expect(result.number1).toEqual(3.14);
expect(result.number2).toEqual(6.2);
expect(result.string).toEqual("string");

if (isErrorCaseSupported) {
expect(result.error.cause.message).toEqual("cause");
expect(result.error1.cause.string).toBe("string");
expect(result.error1.cause.number).toBe(42);
}

if (isBigIntSupported) {
expect(result.bigint).toEqual(5n);
expect(result.bigint1).toEqual(124n);
expect(result.bigint2).toEqual(125n);
expect(result.bigint3).toEqual(12345678901234567890n);
expect(result.bigint4).toEqual(5n);
expect(result.bigint5).toEqual(1000000n);
expect(result.bigint6).toEqual(128n);
expect(result.bigint7).toEqual(2147483647n);
expect(result.obj.foo).toBe(BigInt(-10));
expect(Array.from(result.set)).toEqual([
BigInt(1),
BigInt(2)
]);
expect(result.arr).toEqual([256n, 257n, 258n]);
}

return;
}

const storeValue = {};

storeValue.number = 42;
storeValue.number1 = 3.14;
storeValue.number2 = 6.2;
storeValue.string = "string";

if (isErrorCaseSupported) {
storeValue.error = new Error("error", {
cause: new Error("cause")
});
storeValue.error1 = new Error("error", {
cause: { string: "string", number: 42 }
});
}

if (isBigIntSupported) {
storeValue.bigint = BigInt(5);
storeValue.bigint1 = BigInt(124);
storeValue.bigint2 = BigInt(125);
storeValue.bigint3 = 12345678901234567890n;
storeValue.bigint4 = 5n;
storeValue.bigint5 = 1000000n;
storeValue.bigint6 = 128n;
storeValue.bigint7 = 2147483647n;
storeValue.obj = { foo: BigInt(-10) };
storeValue.set = new Set([BigInt(1), BigInt(2)]);
storeValue.arr = [256n, 257n, 258n];
}

await cacheItem.storePromise(storeValue);
}
);
});
}
}
];

function runCompiler(onSuccess, onError) {
const c = webpack(options);
c.hooks.compilation.tap(
Expand Down