Add support for JS macros

crates/node-bindings/Cargo.toml

@@ -15,12 +15,13 @@ dashmap = "5.4.0"
 xxhash-rust = { version = "0.8.2", features = ["xxh3"] }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-napi = {version =  "2.12.6", features = ["serde-json", "napi4"]}
+napi = {version =  "2.12.6", features = ["serde-json", "napi4", "napi5"]}
 parcel-dev-dep-resolver = { path = "../../packages/utils/dev-dep-resolver" }
 oxipng = "8.0.0"
 mozjpeg-sys = "1.0.0"
 libc = "0.2"
 rayon = "1.7.0"
+crossbeam-channel = "0.5.6"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 napi = {version =  "2.12.6", features = ["serde-json"]}

crates/node-bindings/src/transformer.rs

@@ -1,27 +1,235 @@
-use napi::{Env, JsObject, JsUnknown, Result};
+use napi::{Env, JsObject, JsUnknown};
 use napi_derive::napi;
 
 #[napi]
-pub fn transform(opts: JsObject, env: Env) -> Result<JsUnknown> {
+pub fn transform(opts: JsObject, env: Env) -> napi::Result<JsUnknown> {
   let config: parcel_js_swc_core::Config = env.from_js_value(opts)?;
 
-  let result = parcel_js_swc_core::transform(config)?;
+  let result = parcel_js_swc_core::transform(config, None)?;
   env.to_js_value(&result)
 }
 
 #[cfg(not(target_arch = "wasm32"))]
-#[napi]
-pub fn transform_async(opts: JsObject, env: Env) -> Result<JsObject> {
-  let config: parcel_js_swc_core::Config = env.from_js_value(opts)?;
-  let (deferred, promise) = env.create_deferred()?;
+mod native_only {
+  use super::*;
+  use crossbeam_channel::{Receiver, Sender};
+  use napi::{
+    threadsafe_function::{ThreadSafeCallContext, ThreadsafeFunctionCallMode},
+    JsBoolean, JsFunction, JsNumber, JsString, ValueType,
+  };
+  use parcel_js_swc_core::{JsValue, SourceLocation};
+  use std::sync::Arc;
+
+  // Allocate a single channel per thread to communicate with the JS thread.
+  thread_local! {
+    static CHANNEL: (Sender<Result<JsValue, String>>, Receiver<Result<JsValue, String>>) = crossbeam_channel::unbounded();
+  }
+
+  struct CallMacroMessage {
+    src: String,
+    export: String,
+    args: Vec<JsValue>,
+    loc: SourceLocation,
+  }
+
+  #[napi]
+  pub fn transform_async(opts: JsObject, env: Env) -> napi::Result<JsObject> {
+    let call_macro_tsfn = if opts.has_named_property("callMacro")? {
+      let func = opts.get_named_property::<JsUnknown>("callMacro")?;
+      if let Ok(func) = func.try_into() {
+        Some(env.create_threadsafe_function(
+          &func,
+          0,
+          |ctx: ThreadSafeCallContext<CallMacroMessage>| {
+            let src = ctx.env.create_string(&ctx.value.src)?.into_unknown();
+            let export = ctx.env.create_string(&ctx.value.export)?.into_unknown();
+            let args = js_value_to_napi(JsValue::Array(ctx.value.args), ctx.env)?;
+            let loc = ctx.env.to_js_value(&ctx.value.loc)?.into_unknown();
+            Ok(vec![src, export, args, loc])
+          },
+        )?)
+      } else {
+        None
+      }
+    } else {
+      None
+    };
+
+    let config: parcel_js_swc_core::Config = env.from_js_value(opts)?;
+    let (deferred, promise) = env.create_deferred()?;
+
+    // Get around Env not being Send. See safety note below.
+    let unsafe_env = env.raw() as usize;
+
+    rayon::spawn(move || {
+      let res = parcel_js_swc_core::transform(
+        config,
+        if let Some(tsfn) = call_macro_tsfn {
+          Some(Arc::new(move |src, export, args, loc| {
+            CHANNEL.with(|channel| {
+              // Call JS function to run the macro.
+              let tx = channel.0.clone();
+              tsfn.call_with_return_value(
+                Ok(CallMacroMessage {
+                  src,
+                  export,
+                  args,
+                  loc,
+                }),
+                ThreadsafeFunctionCallMode::Blocking,
+                move |v: JsUnknown| {
+                  // When the JS function returns, await the promise, and send the result
+                  // through the channel back to the native thread.
+                  // SAFETY: this function is called from the JS thread.
+                  await_promise(unsafe { Env::from_raw(unsafe_env as _) }, v, tx)?;
+                  Ok(())
+                },
+              );
+              // Lock the transformer thread until the JS thread returns a result.
+              channel.1.recv().expect("receive failure")
+            })
+          }))
+        } else {
+          None
+        },
+      );
+      match res {
+        Ok(result) => deferred.resolve(move |env| env.to_js_value(&result)),
+        Err(err) => deferred.reject(err.into()),
+      }
+    });
+
+    Ok(promise)
+  }
+
+  /// Convert a JsValue macro argument from the transformer to a napi value.
+  fn js_value_to_napi(value: JsValue, env: Env) -> napi::Result<napi::JsUnknown> {
+    match value {
+      JsValue::Undefined => Ok(env.get_undefined()?.into_unknown()),
+      JsValue::Null => Ok(env.get_null()?.into_unknown()),
+      JsValue::Bool(b) => Ok(env.get_boolean(b)?.into_unknown()),
+      JsValue::Number(n) => Ok(env.create_double(n)?.into_unknown()),
+      JsValue::String(s) => Ok(env.create_string_from_std(s)?.into_unknown()),
+      JsValue::Regex { source, flags } => {
+        let regexp_class: JsFunction = env.get_global()?.get_named_property("RegExp")?;
+        let source = env.create_string_from_std(source)?;
+        let flags = env.create_string_from_std(flags)?;
+        let re = regexp_class.new_instance(&[source, flags])?;
+        Ok(re.into_unknown())
+      }
+      JsValue::Array(arr) => {
+        let mut res = env.create_array(arr.len() as u32)?;
+        for (i, val) in arr.into_iter().enumerate() {
+          res.set(i as u32, js_value_to_napi(val, env)?)?;
+        }
+        Ok(res.coerce_to_object()?.into_unknown())
+      }
+      JsValue::Object(obj) => {
+        let mut res = env.create_object()?;
+        for (k, v) in obj {
+          res.set_named_property(&k, js_value_to_napi(v, env)?)?;
+        }
+        Ok(res.into_unknown())
+      }
+      JsValue::Function(_) => {
+        // Functions can only be returned from macros, not passed in.
+        unreachable!()
+      }
+    }
+  }
+
+  /// Convert a napi value returned as a result of a macro to a JsValue for the transformer.
+  fn napi_to_js_value(value: napi::JsUnknown, env: Env) -> napi::Result<JsValue> {
+    match value.get_type()? {
+      ValueType::Undefined => Ok(JsValue::Undefined),
+      ValueType::Null => Ok(JsValue::Null),
+      ValueType::Number => Ok(JsValue::Number(
+        unsafe { value.cast::<JsNumber>() }.get_double()?,
+      )),
+      ValueType::Boolean => Ok(JsValue::Bool(
+        unsafe { value.cast::<JsBoolean>() }.get_value()?,
+      )),
+      ValueType::String => Ok(JsValue::String(
+        unsafe { value.cast::<JsString>() }
+          .into_utf8()?
+          .into_owned()?,
+      )),
+      ValueType::Object => {
+        let obj = unsafe { value.cast::<JsObject>() };
+        if obj.is_array()? {
+          let len = obj.get_array_length()?;
+          let mut arr = Vec::with_capacity(len as usize);
+          for i in 0..len {
+            let elem = napi_to_js_value(obj.get_element(i)?, env)?;
+            arr.push(elem);
+          }
+          Ok(JsValue::Array(arr))
+        } else {
+          let regexp_class: JsFunction = env.get_global()?.get_named_property("RegExp")?;
+          if obj.instanceof(regexp_class)? {
+            let source: JsString = obj.get_named_property("source")?;
+            let flags: JsString = obj.get_named_property("flags")?;
+            return Ok(JsValue::Regex {
+              source: source.into_utf8()?.into_owned()?,
+              flags: flags.into_utf8()?.into_owned()?,
+            });
+          }
+
+          let names = obj.get_property_names()?;
+          let len = names.get_array_length()?;
+          let mut props = Vec::with_capacity(len as usize);
+          for i in 0..len {
+            let prop = names.get_element::<JsString>(i)?;
+            let name = prop.into_utf8()?.into_owned()?;
+            let value = napi_to_js_value(obj.get_property(prop)?, env)?;
+            props.push((name, value));
+          }
+          Ok(JsValue::Object(props))
+        }
+      }
+      ValueType::Function => {
+        let f = unsafe { value.cast::<JsFunction>() };
+        let source = f.coerce_to_string()?.into_utf8()?.into_owned()?;
+        Ok(JsValue::Function(source))
+      }
+      ValueType::Symbol | ValueType::External | ValueType::Unknown => Err(napi::Error::new(
+        napi::Status::GenericFailure,
+        "Could not convert value returned from macro to AST.",
+      )),
+    }
+  }
 
-  rayon::spawn(move || {
-    let res = parcel_js_swc_core::transform(config);
-    match res {
-      Ok(result) => deferred.resolve(move |env| env.to_js_value(&result)),
-      Err(err) => deferred.reject(err.into()),
+  fn await_promise(
+    env: Env,
+    result: JsUnknown,
+    tx: Sender<Result<JsValue, String>>,
+  ) -> napi::Result<()> {
+    // If the result is a promise, wait for it to resolve, and send the result to the channel.
+    // Otherwise, send the result immediately.
+    if result.is_promise()? {
+      let result: JsObject = result.try_into()?;
+      let then: JsFunction = result.get_named_property("then")?;
+      let tx2 = tx.clone();
+      let cb = env.create_function_from_closure("callback", move |ctx| {
+        let res = napi_to_js_value(ctx.get::<JsUnknown>(0)?, env)?;
+        tx.send(Ok(res)).expect("send failure");
+        ctx.env.get_undefined()
+      })?;
+      let eb = env.create_function_from_closure("error_callback", move |ctx| {
+        let res = ctx.get::<JsUnknown>(0)?;
+        let message = match napi_to_js_value(res, env)? {
+          JsValue::String(s) => s,
+          _ => "Unknown error".into(),
+        };
+        tx2.send(Err(message)).expect("send failure");
+        ctx.env.get_undefined()
+      })?;
+      then.call(Some(&result), &[cb, eb])?;
+    } else {
+      tx.send(Ok(napi_to_js_value(result, env)?))
+        .expect("send failure");
     }
-  });
 
-  Ok(promise)
+    Ok(())
+  }
 }

packages/core/core/src/Transformation.js

@@ -13,12 +13,11 @@ import type {WorkerApi} from '@parcel/workers';
 import type {
   Asset as AssetValue,
   TransformationRequest,
-  RequestInvalidation,
   Config,
   DevDepRequest,
   ParcelOptions,
-  InternalFileCreateInvalidation,
   InternalDevDepOptions,
+  Invalidations,
 } from './types';
 import type {LoadedPlugin} from './ParcelConfig';
 
@@ -46,7 +45,7 @@ import {
   mutableAssetToUncommittedAsset,
 } from './public/Asset';
 import UncommittedAsset from './UncommittedAsset';
-import {createAsset, getInvalidationId} from './assetUtils';
+import {createAsset} from './assetUtils';
 import summarizeRequest from './summarizeRequest';
 import PluginOptions from './public/PluginOptions';
 import {optionsProxy} from './utils';
@@ -68,7 +67,7 @@ import {
   toProjectPathUnsafe,
   toProjectPath,
 } from './projectPath';
-import {invalidateOnFileCreateToInternal} from './utils';
+import {invalidateOnFileCreateToInternal, createInvalidations} from './utils';
 import invariant from 'assert';
 import {tracer, PluginTracer} from '@parcel/profiler';
 
@@ -89,8 +88,7 @@ export type TransformationResult = {|
   assets?: Array<AssetValue>,
   error?: Array<Diagnostic>,
   configRequests: Array<ConfigRequest>,
-  invalidations: Array<RequestInvalidation>,
-  invalidateOnFileCreate: Array<InternalFileCreateInvalidation>,
+  invalidations: Invalidations,
   devDepRequests: Array<DevDepRequest>,
 |};
 
@@ -103,8 +101,7 @@ export default class Transformation {
   pluginOptions: PluginOptions;
   workerApi: WorkerApi;
   parcelConfig: ParcelConfig;
-  invalidations: Map<string, RequestInvalidation>;
-  invalidateOnFileCreate: Array<InternalFileCreateInvalidation>;
+  invalidations: Invalidations;
   resolverRunner: ResolverRunner;
 
   constructor({request, options, config, workerApi}: TransformationOpts) {
@@ -113,8 +110,7 @@ export default class Transformation {
     this.options = options;
     this.request = request;
     this.workerApi = workerApi;
-    this.invalidations = new Map();
-    this.invalidateOnFileCreate = [];
+    this.invalidations = createInvalidations();
     this.devDepRequests = new Map();
     this.pluginDevDeps = [];
     this.resolverRunner = new ResolverRunner({
@@ -127,12 +123,7 @@ export default class Transformation {
       optionsProxy(
         this.options,
         option => {
-          let invalidation: RequestInvalidation = {
-            type: 'option',
-            key: option,
-          };
-
-          this.invalidations.set(getInvalidationId(invalidation), invalidation);
+          this.invalidations.invalidateOnOptionChange.add(option);
         },
         devDep => {
           this.pluginDevDeps.push(devDep);
@@ -213,8 +204,7 @@ export default class Transformation {
       configRequests,
       // When throwing an error, this (de)serialization is done automatically by the WorkerFarm
       error: error ? anyToDiagnostic(error) : undefined,
-      invalidateOnFileCreate: this.invalidateOnFileCreate,
-      invalidations: [...this.invalidations.values()],
+      invalidations: this.invalidations,
       devDepRequests,
     };
   }
@@ -266,7 +256,6 @@ export default class Transformation {
       options: this.options,
       content,
       invalidations: this.invalidations,
-      fileCreateInvalidations: this.invalidateOnFileCreate,
     });
   }
 
@@ -598,7 +587,7 @@ export default class Transformation {
       );
 
       if (result.invalidateOnFileCreate) {
-        this.invalidateOnFileCreate.push(
+        this.invalidations.invalidateOnFileCreate.push(
           ...result.invalidateOnFileCreate.map(i =>
             invalidateOnFileCreateToInternal(this.options.projectRoot, i),
           ),
@@ -607,12 +596,9 @@ export default class Transformation {
 
       if (result.invalidateOnFileChange) {
         for (let filePath of result.invalidateOnFileChange) {
-          let invalidation = {
-            type: 'file',
-            filePath: toProjectPath(this.options.projectRoot, filePath),
-          };
-
-          this.invalidations.set(getInvalidationId(invalidation), invalidation);
+          this.invalidations.invalidateOnFileChange.add(
+            toProjectPath(this.options.projectRoot, filePath),
+          );
         }
       }