add hooks to debug OpenSSL memory

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography.X509Certificates;
@@ -170,5 +171,117 @@
 
             return bytes;
         }
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SetMemoryTracking")]
+        private static unsafe partial int SetMemoryTracking(delegate* unmanaged<MemoryOperation, UIntPtr, UIntPtr, int, char*, int, void> trackingCallback);
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_GetMemoryUse")]
+        internal static partial int GetMemoryUse(ref int memoryUse, ref int allocationCount);
+
+        public static int GetOpenSslAllocatedMemory()
+        {
+            int used = 0;
+            int count = 0;
+            GetMemoryUse(ref used, ref count);
+            return used;
+        }
+
+        public static int GetOpenSslAllocationCount()
+        {
+            int used = 0;
+            int count = 0;
+            GetMemoryUse(ref used, ref count);
+            return count;
+        }
+#if DEBUG
+        [StructLayout(LayoutKind.Sequential)]
+        private unsafe struct MemoryEntry
+        {
+            public int Size;
+            public int Line;
+            public char* File;
+        }
+
+        private enum MemoryOperation
+        {
+            Malloc = 1,
+            Realloc = 2,
+            Free = 3,
+        }
+
+        private static readonly unsafe UIntPtr Offset = (UIntPtr)sizeof(MemoryEntry);
+        private static HashSet<UIntPtr>? _allocations;
+
+        [UnmanagedCallersOnly]
+        private static unsafe void MemoryTrackinCallback(MemoryOperation operation, UIntPtr ptr, UIntPtr oldPtr, int size, char* file, int line)
+        {
+            Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)ptr, 1);
+
+            Debug.Assert(entry[0].File != null);
+            Debug.Assert(ptr != UIntPtr.Zero);
+
+            switch (operation)
+            {
+                case MemoryOperation.Malloc:
+                    Debug.Assert(size == entry[0].Size);
+                    lock (_allocations!)
+                    {
+                        _allocations!.Add(ptr);
+                    }
+                    break;
+                case MemoryOperation.Realloc:
+                    lock (_allocations!)
+                    {
+                        if ((IntPtr)oldPtr != IntPtr.Zero)
+                        {
+                            _allocations!.Remove(oldPtr);
+                        }
+                        _allocations!.Add(ptr);
+                    }
+                    break;
+                case MemoryOperation.Free:
+                    lock (_allocations!)
+                    {
+                        _allocations!.Remove(ptr);
+                    }
+                    break;
+            }
+        }
+
+        public static unsafe void EnableTracking()
+        {
+            _allocations ??= new HashSet<UIntPtr>();
+            _allocations!.Clear();
+            SetMemoryTracking(&MemoryTrackinCallback);
+        }
+
+        public static unsafe void DisableTracking()
+        {
+            SetMemoryTracking(null);
+            _allocations!.Clear();
+        }
+
+        public static unsafe Tuple<UIntPtr, int, string>[] GetIncrementalAllocations()
+        {
+            if (_allocations == null || _allocations.Count == 0)
+            {
+                return Array.Empty<Tuple<UIntPtr, int, string>>();
+            }
+
+            lock (_allocations!)
+            {
+                Tuple<UIntPtr, int, string>[] allocations = new Tuple<UIntPtr, int, string>[_allocations.Count];
+                int index = 0;
+                foreach (UIntPtr ptr in _allocations)
+                {
+                    Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)ptr, 1);
+                    allocations[index] = new Tuple<UIntPtr, int, string>(ptr + Offset, entry[0].Size, $"{Marshal.PtrToStringAnsi((IntPtr)entry[0].File)}:{entry[0].Line}");
+                    index++;
+                }
+
+                return allocations;
+            }
+        }
+#endif
     }
 }

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Initialization.cs

@@ -33,39 +33,11 @@
 
     internal static unsafe partial class CryptoInitializer
     {
-        internal struct MemoryEntry
-        {
-            public char* File;
-            public int Size;
-            public int Line;
-        }
-
-        private static readonly bool DebugMemory = GetMemoryDebug("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_DEBUG");
-        private static readonly bool ValidateMemory = GetMemoryDebug("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_VALIDATE");
-        private static readonly IntPtr Offset = sizeof(MemoryEntry);
-        private static HashSet<IntPtr>? _allocations;
-        private static HashSet<IntPtr>? _allocationsDiff;
-        private static bool _trackIncrementalAllocations;
-
-        internal static long TotalAllocatedMemory;
-        internal static long TotalAllocations;
 
 #pragma warning disable CA1810
         static unsafe CryptoInitializer()
         {
-            if (DebugMemory)
-            {
-                // we need to prepare everything as some allocations do happen during initialization itself.
-                _allocations = new HashSet<IntPtr>();
-                _allocationsDiff = new HashSet<IntPtr>();
-
-                // prevent trimming
-                EnableTracking();
-                DisableTracking();
-                GetIncrementalAllocations();
-            }
-
-            if (EnsureOpenSslInitialized(DebugMemory ? &CryptoMalloc : null, DebugMemory ? &CryptoRealloc : null, DebugMemory ? &CryptoFree : null) != 0)
+            if (EnsureOpenSslInitialized() != 0)
             {
                 // Ideally this would be a CryptographicException, but we use
                 // OpenSSL in libraries lower than System.Security.Cryptography.
@@ -75,196 +47,27 @@
                 // these libraries will be unable to operate correctly.
                 throw new InvalidOperationException();
             }
+
         }
 #pragma warning restore CA1810
 
         internal static void Initialize()
         {
             // No-op that exists to provide a hook for other static constructors.
-        }
-
-        private static bool GetMemoryDebug(string name)
-        {
-            string? value = Environment.GetEnvironmentVariable(name);
-            if (int.TryParse(value, CultureInfo.InvariantCulture, out int enabled))
-            {
-                return enabled == 1;
-            }
-
-            return false;
-        }
-
-        internal static void EnableTracking()
-        {
-            _allocationsDiff!.Clear();
-            _trackIncrementalAllocations = true;
-        }
-
-        internal static Tuple<IntPtr, int, string>[] GetIncrementalAllocations()
-        {
-            lock (_allocationsDiff!)
+            string? value = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_DEBUG");
+            if (int.TryParse(value, CultureInfo.InvariantCulture, out int enabled) && enabled == 1)
             {
-                Tuple<IntPtr, int, string>[] allocations = new Tuple<IntPtr, int, string>[_allocationsDiff.Count];
-                int index = 0;
-                foreach (IntPtr ptr in _allocationsDiff)
-                {
-                    Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)ptr, 1);
-                    allocations[index] = new Tuple<IntPtr, int, string>(ptr+Offset, entry[0].Size, $"{Marshal.PtrToStringAnsi((IntPtr)entry[0].File)}:{entry[0].Line}");
-                    index++;
-                }
-
-                return allocations;
+                Crypto.GetOpenSslAllocationCount();
+                Crypto.GetOpenSslAllocatedMemory();
+#if DEBUG
+                Crypto.EnableTracking();
+                Crypto.GetIncrementalAllocations();
+                Crypto.DisableTracking();
+#endif
             }
         }
 
-        internal static void DisableTracking()
-        {
-            _trackIncrementalAllocations = false;
-            _allocationsDiff!.Clear();
-        }
-
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EnsureOpenSslInitialized")]
-        private static unsafe partial int EnsureOpenSslInitialized(delegate* unmanaged<UIntPtr, char*, int, void*> mallocFunction, delegate* unmanaged<void*, UIntPtr, char*, int, void*> reallocFunction, delegate* unmanaged<void*, void> freeFunction);
-
-        [UnmanagedCallersOnly]
-        internal static unsafe void* CryptoMalloc(UIntPtr size, char* file, int line)
-        {
-            void* ptr =  NativeMemory.Alloc(size + (UIntPtr)Offset);
-            Debug.Assert(ptr != null);
-
-            if (ptr == null)
-            {
-                return null;
-            }
-
-            Span<MemoryEntry> entry = new Span<MemoryEntry>(ptr, 1);
-            entry[0].Line = line;
-            entry[0].File = file;
-            entry[0].Size = (int)size;
-
-            if (ValidateMemory)
-            {
-                lock (_allocations!)
-                {
-                    Debug.Assert(_allocations!.Add((IntPtr)ptr));
-                }
-            }
-            if (_trackIncrementalAllocations)
-            {
-                lock (_allocationsDiff!)
-                {
-                    Debug.Assert(_allocationsDiff!.Add((IntPtr)ptr));
-                }
-            }
-            Interlocked.Add(ref TotalAllocatedMemory, (long)size);
-            Interlocked.Increment(ref TotalAllocations);
-
-            return (void*)((IntPtr)ptr + Offset);
-        }
-
-        [UnmanagedCallersOnly]
-        internal static unsafe void* CryptoRealloc(void* oldPtr, UIntPtr size, char* file, int line)
-        {
-            void * ptr;
-            Span<MemoryEntry> entry;
-
-            if (oldPtr != null)
-            {
-                IntPtr entryPtr = (IntPtr)oldPtr - Offset;
-                entry = new Span<MemoryEntry>((void*)entryPtr, 1);
-
-                if (ValidateMemory)
-                {
-                    lock (_allocations!)
-                    {
-                        if (!_allocations!.Remove(entryPtr))
-                        {
-                            Environment.FailFast($"Failed to find OpenSSL memory 0x{(IntPtr)oldPtr:x}");
-                        }
-                    }
-                }
-
-                if (_trackIncrementalAllocations)
-                {
-                    lock (_allocationsDiff!)
-                    {
-                        // this may fail as we may start tracking after given chunk was allocated
-                        _allocationsDiff!.Remove(entryPtr);
-                    }
-                }
-
-                Interlocked.Add(ref TotalAllocatedMemory, -((long)entry[0].Size));
-                ptr =  NativeMemory.Realloc((void*)entryPtr, size + (UIntPtr)Offset);
-            }
-            else
-            {
-                ptr =  NativeMemory.Alloc(size + (UIntPtr)Offset);
-            }
-
-
-            Debug.Assert(ptr != null);
-            if (ptr == null)
-            {
-                return null;
-            }
-
-            if (ValidateMemory)
-            {
-                lock (_allocations!)
-                {
-                    Debug.Assert(_allocations!.Add((IntPtr)ptr));
-                }
-            }
-            if (_trackIncrementalAllocations)
-            {
-                lock (_allocationsDiff!)
-                {
-                    Debug.Assert(_allocationsDiff!.Add((IntPtr)ptr));
-                }
-            }
-            Interlocked.Add(ref TotalAllocatedMemory, (long)size);
-            Interlocked.Increment(ref TotalAllocations);
-
-            entry = new Span<MemoryEntry>((void*)ptr, 1);
-            entry[0].Line = line;
-            entry[0].File = file;
-            entry[0].Size = (int)size;
-
-            return (void*)((IntPtr)ptr + Offset);
-        }
-
-        [UnmanagedCallersOnly]
-        internal static unsafe void CryptoFree(void* ptr)
-        {
-            if (ptr != null)
-            {
-                IntPtr entryPtr = (IntPtr)ptr - Offset;
-                if (ValidateMemory)
-                {
-                    lock (_allocations!)
-                    {
-                        {
-                            if (!_allocations!.Remove(entryPtr))
-                            {
-                                Environment.FailFast($"Failed to find OpenSSL memory 0x{(IntPtr)ptr:x}");
-                            }
-                        }
-                    }
-                }
-                if (_trackIncrementalAllocations)
-                {
-                    lock (_allocationsDiff!)
-                    {
-                        // this may fail as we may start tracking after given chunk was allocated
-                        _allocationsDiff!.Remove(entryPtr);
-                    }
-                }
-
-                Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)entryPtr, 1);
-                Interlocked.Add(ref TotalAllocatedMemory, -((long)entry[0].Size));
-
-                NativeMemory.Free((void*)entryPtr);
-            }
-        }
+        private static unsafe partial int EnsureOpenSslInitialized();
     }
 }

src/libraries/System.Net.Http/src/System.Net.Http.csproj

@@ -432,11 +432,6 @@
              Link="Common\System\Net\Security\CertificateHelper.Unix.cs" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(TargetPlatformIdentifier)' != '' and '$(TargetPlatformIdentifier)' != 'windows' and '$(TargetPlatformIdentifier)' != 'browser' and '$(TargetPlatformIdentifier)' != 'osx' and '$(TargetPlatformIdentifier)' != 'ios' and '$(TargetPlatformIdentifier)' != 'tvos'">
-    <Compile Include="$(CommonPath)Interop\Unix\System.Security.Cryptography.Native\Interop.Initialization.cs"
-             Link="Common\Interop\Unix\System.Security.Cryptography.Native\Interop.Initialization.cs" />
-  </ItemGroup>
-
   <ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'browser'">
     <Compile Include="$(CommonPath)\System\Net\HttpStatusDescription.cs"
              Link="Common\System\Net\HttpStatusDescription.cs" />
@@ -476,6 +471,7 @@
     <Reference Include="System.Collections" />
     <Reference Include="System.Collections.Concurrent" />
     <Reference Include="System.Diagnostics.Tracing" />
+    <Reference Include="System.Diagnostics.StackTrace" />
     <Reference Include="System.Memory" />
     <Reference Include="System.Runtime" />
     <Reference Include="System.Runtime.InteropServices" />