Skip to content

Commit

Permalink
Merge pull request #17282 from webpack/feat-improve-serialization
Browse files Browse the repository at this point in the history 
feat: improve serialization for errors and bigints
  • Loading branch information
@TheLarkInn
TheLarkInn committed Jun 5, 2023
2 parents 33b4ba0 + a5e9568 commit c9f32bc
Show file tree
Hide file tree
Showing 6 changed files with 248 additions and 2 deletions.
3 changes: 3 additions & 0 deletions .eslintrc.js
View file
Expand Up @@ -109,6 +109,9 @@ module.exports = {
env: {
"jest/globals": true
},
parserOptions: {
ecmaVersion: 2020
},
globals: {
nsObj: false,
jasmine: false
Expand Down
144 changes: 143 additions & 1 deletion lib/serialization/BinaryMiddleware.js
View file
Expand Up @@ -21,6 +21,9 @@ Section -> NullsSection |
I32NumbersSection |
I8NumbersSection |
ShortStringSection |
BigIntSection |
I32BigIntSection |
I8BigIntSection
StringSection |
BufferSection |
NopSection
Expand All @@ -39,6 +42,9 @@ ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
StringSection -> StringSectionHeaderByte i32:length utf8-byte*
BufferSection -> BufferSectionHeaderByte i32:length byte*
NopSection --> NopSectionHeaderByte
BigIntSection -> BigIntSectionHeaderByte i32:length ascii-byte*
I32BigIntSection -> I32BigIntSectionHeaderByte i32
I8BigIntSection -> I8BigIntSectionHeaderByte i8
ShortStringSectionHeaderByte -> 0b1nnn_nnnn (n:length)
Expand All @@ -58,6 +64,9 @@ BooleansCountAndBitsByte ->
StringSectionHeaderByte -> 0b0000_1110
BufferSectionHeaderByte -> 0b0000_1111
NopSectionHeaderByte -> 0b0000_1011
BigIntSectionHeaderByte -> 0b0001_1010
I32BigIntSectionHeaderByte -> 0b0001_1100
I8BigIntSectionHeaderByte -> 0b0001_1011
FalseHeaderByte -> 0b0000_1100
TrueHeaderByte -> 0b0000_1101
Expand All @@ -78,6 +87,9 @@ const NULL_AND_I8_HEADER = 0x15;
const NULL_AND_I32_HEADER = 0x16;
const NULL_AND_TRUE_HEADER = 0x17;
const NULL_AND_FALSE_HEADER = 0x18;
const BIGINT_HEADER = 0x1a;
const BIGINT_I8_HEADER = 0x1b;
const BIGINT_I32_HEADER = 0x1c;
const STRING_HEADER = 0x1e;
const BUFFER_HEADER = 0x1f;
const I8_HEADER = 0x60;
Expand All @@ -86,7 +98,7 @@ const F64_HEADER = 0x20;
const SHORT_STRING_HEADER = 0x80;

/** Uplift high-order bits */
const NUMBERS_HEADER_MASK = 0xe0;
const NUMBERS_HEADER_MASK = 0xe0; // 0b1010_0000
const NUMBERS_COUNT_MASK = 0x1f; // 0b0001_1111
const SHORT_STRING_LENGTH_MASK = 0x7f; // 0b0111_1111

Expand All @@ -113,6 +125,16 @@ const identifyNumber = n => {
return 2;
};

/**
* @param {bigint} n bigint
* @returns {0 | 1 | 2} type of bigint for serialization
*/
const identifyBigInt = n => {
if (n <= BigInt(127) && n >= BigInt(-128)) return 0;
if (n <= BigInt(2147483647) && n >= BigInt(-2147483648)) return 1;
return 2;
};

/**
* @typedef {PrimitiveSerializableType[]} DeserializedType
* @typedef {BufferSerializableType[]} SerializedType
Expand Down Expand Up @@ -331,6 +353,62 @@ class BinaryMiddleware extends SerializerMiddleware {
}
break;
}
case "bigint": {
const type = identifyBigInt(thing);
if (type === 0 && thing >= 0 && thing <= BigInt(10)) {
// shortcut for very small bigints
allocate(HEADER_SIZE + I8_SIZE);
writeU8(BIGINT_I8_HEADER);
writeU8(Number(thing));
break;
}

switch (type) {
case 0: {
let n = 1;
allocate(HEADER_SIZE + I8_SIZE * n);
writeU8(BIGINT_I8_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeInt8(
Number(/** @type {bigint} */ (data[i])),
currentPosition
);
currentPosition += I8_SIZE;
n--;
i++;
}
i--;
break;
}
case 1: {
let n = 1;
allocate(HEADER_SIZE + I32_SIZE * n);
writeU8(BIGINT_I32_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeInt32LE(
Number(/** @type {bigint} */ (data[i])),
currentPosition
);
currentPosition += I32_SIZE;
n--;
i++;
}
i--;
break;
}
default: {
const value = thing.toString();
const len = Buffer.byteLength(value);
allocate(len + HEADER_SIZE + I32_SIZE);
writeU8(BIGINT_HEADER);
writeU32(len);
currentBuffer.write(value, currentPosition);
currentPosition += len;
break;
}
}
break;
}
case "number": {
const type = identifyNumber(thing);
if (type === 0 && thing >= 0 && thing <= 10) {
Expand Down Expand Up @@ -847,6 +925,70 @@ class BinaryMiddleware extends SerializerMiddleware {
result.push(read(1).readInt8(0));
}
};
case BIGINT_I8_HEADER: {
const len = 1;
return () => {
const need = I8_SIZE * len;

if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
const value =
/** @type {Buffer} */
(currentBuffer).readInt8(currentPosition);
result.push(BigInt(value));
currentPosition += I8_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
const value = buf.readInt8(i * I8_SIZE);
result.push(BigInt(value));
}
}
};
}
case BIGINT_I32_HEADER: {
const len = 1;
return () => {
const need = I32_SIZE * len;
if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
const value = /** @type {Buffer} */ (currentBuffer).readInt32LE(
currentPosition
);
result.push(BigInt(value));
currentPosition += I32_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
const value = buf.readInt32LE(i * I32_SIZE);
result.push(BigInt(value));
}
}
};
}
case BIGINT_HEADER: {
return () => {
const len = readU32();
if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
const value = currentBuffer.toString(
undefined,
currentPosition,
currentPosition + len
);

result.push(BigInt(value));
currentPosition += len;
checkOverflow();
} else {
const value = read(len).toString();
result.push(BigInt(value));
}
};
}
default:
if (header <= 10) {
return () => result.push(header);
Expand Down
3 changes: 3 additions & 0 deletions lib/serialization/ErrorObjectSerializer.js
View file
Expand Up @@ -21,6 +21,7 @@ class ErrorObjectSerializer {
serialize(obj, context) {
context.write(obj.message);
context.write(obj.stack);
context.write(/** @type {Error & { cause: "unknown" }} */ (obj).cause);
}
/**
* @param {ObjectDeserializerContext} context context
Expand All @@ -31,6 +32,8 @@ class ErrorObjectSerializer {

err.message = context.read();
err.stack = context.read();
/** @type {Error & { cause: "unknown" }} */
(err).cause = context.read();

return err;
}
Expand Down
3 changes: 3 additions & 0 deletions lib/serialization/ObjectMiddleware.js
View file
Expand Up @@ -397,6 +397,9 @@ class ObjectMiddleware extends SerializerMiddleware {
", "
)} }`;
}
if (typeof item === "bigint") {
return `BigInt ${item}n`;
}
try {
return `${item}`;
} catch (e) {
Expand Down
2 changes: 1 addition & 1 deletion lib/serialization/types.js
View file
Expand Up @@ -6,7 +6,7 @@

/** @typedef {undefined|null|number|string|boolean|Buffer|Object|(() => ComplexSerializableType[] | Promise<ComplexSerializableType[]>)} ComplexSerializableType */

/** @typedef {undefined|null|number|string|boolean|Buffer|(() => PrimitiveSerializableType[] | Promise<PrimitiveSerializableType[]>)} PrimitiveSerializableType */
/** @typedef {undefined|null|number|bigint|string|boolean|Buffer|(() => PrimitiveSerializableType[] | Promise<PrimitiveSerializableType[]>)} PrimitiveSerializableType */

/** @typedef {Buffer|(() => BufferSerializableType[] | Promise<BufferSerializableType[]>)} BufferSerializableType */

Expand Down
95 changes: 95 additions & 0 deletions test/Compiler-filesystem-caching.test.js
View file
Expand Up @@ -41,6 +41,101 @@ describe("Compiler (filesystem caching)", () => {
]
};

const isBigIntSupported = typeof BigInt !== "undefined";
const isErrorCaseSupported =
typeof new Error("test", { cause: new Error("cause") }).cause !==
"undefined";

options.plugins = [
{
apply(compiler) {
const name = "TestCachePlugin";

compiler.hooks.thisCompilation.tap(name, compilation => {
compilation.hooks.processAssets.tapPromise(
{
name,
stage:
compiler.webpack.Compilation
.PROCESS_ASSETS_STAGE_OPTIMIZE_SIZE
},
async () => {
const cache = compilation.getCache(name);
const ident = "test.ext";
const cacheItem = cache.getItemCache(ident, null);

const result = await cacheItem.getPromise(ident);

if (result) {
expect(result.number).toEqual(42);
expect(result.number1).toEqual(3.14);
expect(result.number2).toEqual(6.2);
expect(result.string).toEqual("string");

if (isErrorCaseSupported) {
expect(result.error.cause.message).toEqual("cause");
expect(result.error1.cause.string).toBe("string");
expect(result.error1.cause.number).toBe(42);
}

if (isBigIntSupported) {
expect(result.bigint).toEqual(5n);
expect(result.bigint1).toEqual(124n);
expect(result.bigint2).toEqual(125n);
expect(result.bigint3).toEqual(12345678901234567890n);
expect(result.bigint4).toEqual(5n);
expect(result.bigint5).toEqual(1000000n);
expect(result.bigint6).toEqual(128n);
expect(result.bigint7).toEqual(2147483647n);
expect(result.obj.foo).toBe(BigInt(-10));
expect(Array.from(result.set)).toEqual([
BigInt(1),
BigInt(2)
]);
expect(result.arr).toEqual([256n, 257n, 258n]);
}

return;
}

const storeValue = {};

storeValue.number = 42;
storeValue.number1 = 3.14;
storeValue.number2 = 6.2;
storeValue.string = "string";

if (isErrorCaseSupported) {
storeValue.error = new Error("error", {
cause: new Error("cause")
});
storeValue.error1 = new Error("error", {
cause: { string: "string", number: 42 }
});
}

if (isBigIntSupported) {
storeValue.bigint = BigInt(5);
storeValue.bigint1 = BigInt(124);
storeValue.bigint2 = BigInt(125);
storeValue.bigint3 = 12345678901234567890n;
storeValue.bigint4 = 5n;
storeValue.bigint5 = 1000000n;
storeValue.bigint6 = 128n;
storeValue.bigint7 = 2147483647n;
storeValue.obj = { foo: BigInt(-10) };
storeValue.set = new Set([BigInt(1), BigInt(2)]);
storeValue.arr = [256n, 257n, 258n];
}

await cacheItem.storePromise(storeValue);
}
);
});
}
}
];

function runCompiler(onSuccess, onError) {
const c = webpack(options);
c.hooks.compilation.tap(
Expand Down

0 comments on commit c9f32bc

Please sign in to comment.