[mono][tasks] Extract EmitWasmBundle into generic EmitBundle task and enable bundling in mono self-contained library

Directory.Build.props

@@ -152,6 +152,7 @@
     <LibraryBuilderDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'LibraryBuilder', 'Debug', '$(NetCoreAppToolCurrent)'))</LibraryBuilderDir>
     <MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoAOTCompilerDir>
     <MonoTargetsTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoTargetsTasksDir>
+    <EmitBundleDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'EmitBundle', 'Debug', '$(NetCoreAppToolCurrent)'))</EmitBundleDir>
     <TestExclusionListTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</TestExclusionListTasksDir>
     <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
     <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
@@ -166,6 +167,7 @@
     <LibraryBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll'))</LibraryBuilderTasksAssemblyPath>
     <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
     <MonoTargetsTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll'))</MonoTargetsTasksAssemblyPath>
+    <EmitBundleTasksAssemblyPath>$([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll'))</EmitBundleTasksAssemblyPath>
     <TestExclusionListTasksAssemblyPath>$([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll'))</TestExclusionListTasksAssemblyPath>
     <CoreCLRToolPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfiguration)'))</CoreCLRToolPath>
     <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true' or '$(BuildArchitecture)' == 's390x' or '$(BuildArchitecture)' == 'ppc64le'">$(CoreCLRToolPath)</ILAsmToolPath>

src/mono/mono/mini/aot-runtime.c

@@ -2413,6 +2413,59 @@ mono_aot_register_module (gpointer *aot_info)
 		mono_aot_unlock ();
 }
 
+static GHashTable *bundled_assemblies = NULL;
+
+static int bundles_added = 0;
+
+void
+mono_add_bundled_assembly (const char *name, const unsigned char *data, unsigned int size)
+{
+    if (!bundled_assemblies)
+        bundled_assemblies = g_hash_table_new (g_str_hash, g_str_equal);
+
+    MonoBundledAssembly *bundled_assembly = g_new0 (MonoBundledAssembly, 1);
+    bundled_assembly->name = g_strdup (name);
+    g_assert (bundled_assembly->name);
+    bundled_assembly->data = data;
+    bundled_assembly->size = size;
+    g_hash_table_insert (bundled_assemblies, (gpointer) name, bundled_assembly);
+    bundles_added++;
+}
+
+void
+mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size)
+{
+    *out_data = NULL;
+    *out_size = 0;
+    MonoBundledAssembly *bundled_assembly;
+
+    if (g_hash_table_lookup_extended (bundled_assemblies, name, NULL, (gpointer *)&bundled_assembly)) {
+        *out_data = bundled_assembly->data;
+        *out_size = bundled_assembly->size;
+    }
+}
+
+static void
+populate_bundle (gpointer key, gpointer value, gpointer user_data)
+{
+    MonoBundledAssembly *bundled_assembly = (MonoBundledAssembly *)value;
+    MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data;
+    g_assert (bundles_added > 0);
+    bundle [bundles_added-- - 1] = bundled_assembly;
+}
+
+void
+mono_register_bundle (void)
+{
+    if (!bundled_assemblies)
+        return;
+
+    MonoBundledAssembly **bundle = g_new0 (MonoBundledAssembly*, bundles_added + 1);
+    g_hash_table_foreach (bundled_assemblies, populate_bundle, bundle);
+
+    mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle);
+}
+
 void
 mono_aot_init (void)
 {

src/mono/mono/mini/aot-runtime.h

@@ -248,6 +248,9 @@ typedef struct MonoAotFileInfo
 /* Number of symbols in the MonoAotFileInfo structure */
 #define MONO_AOT_FILE_INFO_NUM_SYMBOLS (((G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_LAST_SYMBOL) - G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_FIRST_SYMBOL)) / sizeof (gpointer)) + 1)
 
+void      mono_add_bundled_assembly         (const char *name, const unsigned char *data, unsigned int size);
+void      mono_get_bundled_assembly         (const char *name, const unsigned char **out_data, unsigned int *out_size);
+void      mono_register_bundle              (void);
 void      mono_aot_init                     (void);
 gpointer  mono_aot_get_method               (MonoMethod *method, MonoError *error);
 gpointer  mono_aot_get_method_from_token    (MonoImage *image, guint32 token, MonoError *error);

src/mono/msbuild/android/build/AndroidBuild.targets

@@ -247,7 +247,8 @@
   <Target Name="_AndroidGenerateRuntimeConfig"
           Inputs="$(_AndroidRuntimeConfigFilePath)"
           Outputs="$(_ParsedRuntimeConfigFilePath)"
-          Condition="Exists('$(_AndroidRuntimeConfigFilePath)')">
+          Condition="Exists('$(_AndroidRuntimeConfigFilePath)')"
+          BeforeTargets="_GenerateBundle">
     <ItemGroup>
       <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/>
       <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/>

src/mono/msbuild/apple/build/AppleBuild.targets

@@ -274,7 +274,8 @@
   <Target Name="_AppleGenerateRuntimeConfig"
           Inputs="$(_AppleRuntimeConfigFilePath)"
           Outputs="$(_ParsedRuntimeConfigFilePath)"
-          Condition="Exists('$(_AppleRuntimeConfigFilePath)')">
+          Condition="Exists('$(_AppleRuntimeConfigFilePath)')"
+          BeforeTargets="_GenerateBundle">
     <ItemGroup>
       <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/>
       <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/>

src/mono/msbuild/common/LibraryBuilder.targets

@@ -1,9 +1,12 @@
 <Project>
+  <UsingTask TaskName="EmitBundleSourceFiles"
+             AssemblyFile="$(EmitBundleTasksAssemblyPath)" />
   <UsingTask TaskName="LibraryBuilderTask" 
              AssemblyFile="$(LibraryBuilderTasksAssemblyPath)" />
 
   <Target Name="_BuildNativeLibrary"
-          Condition="'$(_IsLibraryMode)' == 'true' and '$(RunAOTCompilation)' == 'true'">
+          Condition="'$(_IsLibraryMode)' == 'true' and '$(RunAOTCompilation)' == 'true'"
+          DependsOnTargets="_GenerateBundle">
 
     <PropertyGroup>
       <_IsSharedLibrary>false</_IsSharedLibrary>
@@ -14,12 +17,15 @@
 
     <ItemGroup>
       <_ExtraLibrarySources Include="$(_AotModuleTablePath)" />
+      <_ExtraLibrarySources Include="$(_AssembliesToBundleSourceFile)" />
+      <_ExtraLibrarySources Include="%(_AssembliesToBundleWithHashes.DestinationFile)" />
       <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" />
     </ItemGroup>
 
     <LibraryBuilderTask
       Assemblies="@(_AssembliesToBundleInternal)"
       AssembliesLocation="$(AssembliesLocation)"
+      BundlesAssemblies="$(BundlesAssemblies)"
       ExtraLinkerArguments="@(_ExtraLinkerArgs)"
       ExtraSources="@(_ExtraLibrarySources)"
       IsSharedLibrary="$(_IsSharedLibrary)"
@@ -35,4 +41,35 @@
     </LibraryBuilderTask>
   </Target>
 
+  <Target Name="_GenerateBundle"
+          Condition="'$(BundlesAssemblies)' == 'true'">
+
+    <PropertyGroup>
+      <_AssembliesToBundleSourceFile>$(BundleDir)bundled_assemblies.c</_AssembliesToBundleSourceFile>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_AssembliesToBundle Remove="@(_AssembliesToBundle)" />
+      <_AssembliesToBundle Include="@(_AssembliesToBundleInternal)" />
+      <_AssembliesToBundle Include="$(_ParsedRuntimeConfigFilePath)" />
+    </ItemGroup>
+
+    <GetFileHash Files="@(_AssembliesToBundle)">
+      <Output TaskParameter="Items" ItemName="_AssembliesToBundleWithHashes" />
+    </GetFileHash>
+
+    <ItemGroup>
+      <_AssembliesToBundleWithHashes Update="@(_AssembliesToBundleWithHashes)">
+        <DestinationFile>$(BundleDir)%(_AssembliesToBundleWithHashes.Filename)%(_AssembliesToBundleWithHashes.Extension).$([System.String]::Copy(%(_AssembliesToBundleWithHashes.FileHash)).Substring(0, 8)).c</DestinationFile>
+      </_AssembliesToBundleWithHashes>
+    </ItemGroup>
+
+    <EmitBundleSourceFiles
+      FilesToBundle="@(_AssembliesToBundleWithHashes)"
+      BundleName="assemblies"
+      BundleFile="$(_AssembliesToBundleSourceFile)"
+      RegistrationCallbackFunctionName="mono_add_bundled_assembly"
+      />
+  </Target>
+
 </Project>

src/mono/wasi/build/WasiApp.Native.targets

@@ -341,9 +341,9 @@
           <!--EnvironmentVariables="@(EmscriptenEnvVars)" />-->
   <!--</Target>-->
 
-  <UsingTask TaskName="Microsoft.WebAssembly.Build.Tasks.EmitWasmBundleObjectFiles" AssemblyFile="$(WasmAppBuilderTasksAssemblyPath)" />
+  <UsingTask TaskName="EmitBundleObjectFiles" AssemblyFile="$(EmitBundleTasksAssemblyPath)" />
   <Target Name="_GenerateAssemblyObjectFiles" Returns="@(_WasiObjectFilesForBundle)">
-    <!-- Get the file hashes of everything in @(_WasmBundleFiles), then pass it all to EmitWasmBundleObjectFiles. This
+    <!-- Get the file hashes of everything in @(_WasmBundleFiles), then pass it all to EmitBundleObjectFiles. This
          will emit corresponding .o files for anything we don't already have on disk. -->
     <PropertyGroup>
       <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o</_WasmAssembliesBundleObjectFile>
@@ -357,7 +357,7 @@
       </WasmBundleAssembliesWithHashes>
     </ItemGroup>
     <!-- TODO make this incremental compilation -->
-    <EmitWasmBundleObjectFiles
+    <EmitBundleObjectFiles
       FilesToBundle="@(WasmBundleAssembliesWithHashes)"
       ClangExecutable="$(WasiClang)"
       BundleName="assemblies"

src/mono/wasi/runtime/driver.c

@@ -62,9 +62,9 @@ int32_t monoeg_g_hasenv(const char *variable);
 void mono_free (void*);
 int32_t mini_parse_debug_option (const char *option);
 char *mono_method_get_full_name (MonoMethod *method);
-extern void mono_wasm_register_timezones_bundle();
+extern void mono_register_timezones_bundle();
 #ifdef BUNDLED_ASSEMBLIES
-extern void mono_wasm_register_assemblies_bundle();
+extern void mono_register_assemblies_bundle();
 #endif
 
 extern const char* dotnet_wasi_getentrypointassemblyname();
@@ -432,9 +432,9 @@ mono_wasm_load_runtime (const char *unused, int debug_level)
 
 	mini_parse_debug_option ("top-runtime-invoke-unhandled");
 
-	mono_wasm_register_timezones_bundle();
+	mono_register_timezones_bundle();
 #ifdef BUNDLED_ASSEMBLIES
-	mono_wasm_register_assemblies_bundle();
+	mono_register_assemblies_bundle();
 #endif
 	mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL);
 	mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp);

src/mono/wasi/wasi.proj

@@ -65,7 +65,7 @@
     </ManagedToNativeGenerator>
   </Target>
 
-  <UsingTask TaskName="Microsoft.WebAssembly.Build.Tasks.EmitWasmBundleObjectFiles" AssemblyFile="$(WasmAppBuilderTasksAssemblyPath)" />
+  <UsingTask TaskName="EmitBundleObjectFiles" AssemblyFile="$(EmitBundleTasksAssemblyPath)" />
   <Target Name="GenerateTimezonesArchive" Returns="@(_WasmArchivedTimezones)">
     <PropertyGroup>
       <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data'))</_WasmTimezonesPath>
@@ -86,14 +86,14 @@
       </_WasmBundleTimezonesWithHashes>
     </ItemGroup>
     <!-- TODO make this incremental compilation -->
-    <EmitWasmBundleObjectFiles 
+    <EmitBundleObjectFiles
       FilesToBundle="@(_WasmBundleTimezonesWithHashes)" 
       ClangExecutable="$(WasiClang)"
       BundleName="timezones"
       BundleFile="$(_WasmTimezonesBundleObjectFile)"
       RegistrationCallbackFunctionName="mono_wasm_add_bundled_file"
       >
-    </EmitWasmBundleObjectFiles>
+    </EmitBundleObjectFiles>
     <ItemGroup>
       <WasmBundleTimezonesObjects Include="$([MSBuild]::MakeRelative($(WasiObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" />
       <WasmBundleTimezonesObjects Include="$([MSBuild]::MakeRelative($(WasiObjDir), $(_WasmTimezonesBundleObjectFile)).Replace('\','/'))" />

src/mono/wasm/runtime/driver.c

@@ -53,7 +53,7 @@ int monoeg_g_setenv(const char *variable, const char *value, int overwrite);
 int32_t mini_parse_debug_option (const char *option);
 char *mono_method_get_full_name (MonoMethod *method);
 
-extern void mono_wasm_register_timezones_bundle();
+extern void mono_register_timezones_bundle();
 extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token);
 static void mono_wasm_init_finalizer_thread (void);
 
@@ -512,7 +512,7 @@ mono_wasm_load_runtime (const char *unused, int debug_level)
 
 	mini_parse_debug_option ("top-runtime-invoke-unhandled");
 
-	mono_wasm_register_timezones_bundle();
+	mono_register_timezones_bundle();
 	mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL);
 	mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp);
 

src/mono/wasm/wasm.proj

@@ -80,7 +80,7 @@
     </ManagedToNativeGenerator>
   </Target>
 
-  <UsingTask TaskName="Microsoft.WebAssembly.Build.Tasks.EmitWasmBundleSourceFiles" AssemblyFile="$(WasmAppBuilderTasksAssemblyPath)" />
+  <UsingTask TaskName="EmitBundleSourceFiles" AssemblyFile="$(EmitBundleTasksAssemblyPath)" />
   <Target Name="GenerateTimezonesArchive" Returns="@(_WasmArchivedTimezones)">
     <PropertyGroup>
       <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data'))</_WasmTimezonesPath>
@@ -104,13 +104,13 @@
       </_WasmBundleTimezonesWithHashes>
     </ItemGroup>
     <!-- TODO make this incremental compilation -->
-    <EmitWasmBundleSourceFiles
+    <EmitBundleSourceFiles
       FilesToBundle="@(_WasmBundleTimezonesWithHashes)"
       BundleName="timezones"
       BundleFile="$(WasmObjDir)\wasm-bundled-timezones.c"
       RegistrationCallbackFunctionName="mono_wasm_add_bundled_file"
-      >
-    </EmitWasmBundleSourceFiles>
+      />
+
     <ItemGroup>
       <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" />
       <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" />

src/tasks/EmitBundleTask/EmitBundle.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>$(TargetFrameworkForNETCoreTasks)</TargetFramework>
+    <Nullable>enable</Nullable>
+    <NoWarn>$(NoWarn),CA1050,CA1850</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\Common\Utils.cs" />
+
+    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
+  </ItemGroup>
+
+  <Target Name="GetFilesToPackage" Returns="@(FilesToPackage)">
+    <ItemGroup>
+      <FilesToPackage Include="$(OutputPath)$(MSBuildProjectName)*"
+                      TargetPath="tasks\$(TargetFrameworkForNETCoreTasks)" />
+    </ItemGroup>
+  </Target>
+
+</Project>

src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs → src/tasks/EmitBundleTask/EmitBundleBase.cs

@@ -11,9 +11,7 @@
 using System.Threading.Tasks;
 using Microsoft.Build.Framework;
 
-namespace Microsoft.WebAssembly.Build.Tasks;
-
-public abstract class EmitWasmBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask
+public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask
 {
     private CancellationTokenSource BuildTaskCancelled { get; } = new();
 
@@ -65,7 +63,7 @@ public override bool Execute()
             return (registeredName, symbolName);
         }).ToList();
 
-        Log.LogMessage(MessageImportance.Low, "Bundling {numFiles} files for {bundleName}", files.Count, BundleName);
+        Log.LogMessage(MessageImportance.Low, "Bundling {0} files for {1}", files.Count, BundleName);
 
         if (remainingDestinationFilesToBundle.Length > 0)
         {
@@ -109,7 +107,7 @@ public override bool Execute()
         return Emit(BundleFile, (inputStream) =>
         {
             using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom);
-            GenerateRegisteredBundledObjects($"mono_wasm_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer);
+            GenerateRegisteredBundledObjects($"mono_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer);
         }) && !Log.HasLoggedErrors;
     }
 

src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs → src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs

@@ -6,9 +6,7 @@
 using System.IO;
 using Microsoft.Build.Framework;
 
-namespace Microsoft.WebAssembly.Build.Tasks;
-
-public class EmitWasmBundleObjectFiles : EmitWasmBundleBase
+public class EmitBundleObjectFiles : EmitBundleBase
 {
     [Required]
     public string ClangExecutable { get; set; } = default!;

src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs → src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs

@@ -5,12 +5,10 @@
 using System.IO;
 using Microsoft.Build.Framework;
 
-namespace Microsoft.WebAssembly.Build.Tasks;
-
-// It would be ideal that this Task would always produce object files as EmitWasmBundleObjectFiles does.
-// EmitWasmBundleObjectFiles could do it with clang by streaming code directly to clang input stream.
+// It would be ideal that this Task would always produce object files as EmitBundleObjectFiles does.
+// EmitBundleObjectFiles could do it with clang by streaming code directly to clang input stream.
 // For emcc it's not possible, so we need to write the code to disk first and then compile it in MSBuild.
-public class EmitWasmBundleSourceFiles : EmitWasmBundleBase
+public class EmitBundleSourceFiles : EmitBundleBase
 {
     public override bool Emit(string destinationFile, Action<Stream> inputProvider)
     {

src/tasks/LibraryBuilder/LibraryBuilder.cs

@@ -85,6 +85,11 @@ public bool IsSharedLibrary
     /// </summary>
     public string? AssembliesLocation { get; set; }
 
+    /// <summary>
+    /// Determines whether or not assemblies are bundled into the library
+    /// </summary>
+    public bool BundlesAssemblies { get; set; }
+
     public bool StripDebugSymbols { get; set; }
 
     /// <summary>
@@ -179,7 +184,7 @@ private void GatherAotSourcesObjects(StringBuilder aotSources, StringBuilder aot
 
                 if (symbolsAdded > 0)
                 {
-                    exportedAssemblies.Add(Path.GetFileNameWithoutExtension(compiledAssembly.Path));
+                    exportedAssemblies.Add(Path.GetFileName(compiledAssembly.Path));
                 }
             }
         }
@@ -309,6 +314,11 @@ private string GenerateExtraDefinitions()
             extraDefinitions.AppendLine("add_definitions(-DUSES_AOT_DATA=1)");
         }
 
+        if (BundlesAssemblies)
+        {
+            extraDefinitions.AppendLine("add_definitions(-DBUNDLED_ASSEMBLIES=1)");
+        }
+
         return extraDefinitions.ToString();
     }