add hooks to debug OpenSSL memory

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Initialization.cs

@@ -2,7 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
 using System.Runtime.InteropServices;
+using System.Threading;
 
 internal static partial class Interop
 {
@@ -27,11 +31,41 @@ static OpenSsl()
         }
     }
 
-    internal static partial class CryptoInitializer
+    internal static unsafe partial class CryptoInitializer
     {
-        static CryptoInitializer()
+        internal struct MemoryEntry
         {
-            if (EnsureOpenSslInitialized() != 0)
+            public char* File;
+            public int Size;
+            public int Line;
+        }
+
+        private static readonly bool DebugMemory = GetMemoryDebug("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_DEBUG");
+        private static readonly bool ValidateMemory = GetMemoryDebug("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_VALIDATE");
+        private static readonly IntPtr Offset = sizeof(MemoryEntry);
+        private static HashSet<IntPtr>? _allocations;
+        private static HashSet<IntPtr>? _allocationsDiff;
+        private static bool _trackIncrementalAllocations;
+
+        internal static long TotalAllocatedMemory;
+        internal static long TotalAllocations;
+
+#pragma warning disable CA1810
+        static unsafe CryptoInitializer()
+        {
+            if (DebugMemory)
+            {
+                // we need to prepare everything as some allocations do happen during initialization itself.
+                _allocations = new HashSet<IntPtr>();
+                _allocationsDiff = new HashSet<IntPtr>();
+
+                // prevent trimming
+                EnableTracking();
+                DisableTracking();
+                GetIncrementalAllocations();
+            }
+
+            if (EnsureOpenSslInitialized(DebugMemory ? &CryptoMalloc : null, DebugMemory ? &CryptoRealloc : null, DebugMemory ? &CryptoFree : null) != 0)
             {
                 // Ideally this would be a CryptographicException, but we use
                 // OpenSSL in libraries lower than System.Security.Cryptography.
@@ -42,13 +76,195 @@ static CryptoInitializer()
                 throw new InvalidOperationException();
             }
         }
+#pragma warning restore CA1810
 
         internal static void Initialize()
         {
             // No-op that exists to provide a hook for other static constructors.
         }
 
+        private static bool GetMemoryDebug(string name)
+        {
+            string? value = Environment.GetEnvironmentVariable(name);
+            if (int.TryParse(value, CultureInfo.InvariantCulture, out int enabled))
+            {
+                return enabled == 1;
+            }
+
+            return false;
+        }
+
+        internal static void EnableTracking()
+        {
+            _allocationsDiff!.Clear();
+            _trackIncrementalAllocations = true;
+        }
+
+        internal static Tuple<IntPtr, int, string>[] GetIncrementalAllocations()
+        {
+            lock (_allocationsDiff!)
+            {
+                Tuple<IntPtr, int, string>[] allocations = new Tuple<IntPtr, int, string>[_allocationsDiff.Count];
+                int index = 0;
+                foreach (IntPtr ptr in _allocationsDiff)
+                {
+                    Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)ptr, 1);
+                    allocations[index] = new Tuple<IntPtr, int, string>(ptr+Offset, entry[0].Size, $"{Marshal.PtrToStringAnsi((IntPtr)entry[0].File)}:{entry[0].Line}");
+                    index++;
+                }
+
+                return allocations;
+            }
+        }
+
+        internal static void DisableTracking()
+        {
+            _trackIncrementalAllocations = false;
+            _allocationsDiff!.Clear();
+        }
+
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EnsureOpenSslInitialized")]
-        private static partial int EnsureOpenSslInitialized();
+        private static unsafe partial int EnsureOpenSslInitialized(delegate* unmanaged<UIntPtr, char*, int, void*> mallocFunction, delegate* unmanaged<void*, UIntPtr, char*, int, void*> reallocFunction, delegate* unmanaged<void*, void> freeFunction);
+
+        [UnmanagedCallersOnly]
+        internal static unsafe void* CryptoMalloc(UIntPtr size, char* file, int line)
+        {
+            void* ptr =  NativeMemory.Alloc(size + (UIntPtr)Offset);
+            Debug.Assert(ptr != null);
+
+            if (ptr == null)
+            {
+                return null;
+            }
+
+            Span<MemoryEntry> entry = new Span<MemoryEntry>(ptr, 1);
+            entry[0].Line = line;
+            entry[0].File = file;
+            entry[0].Size = (int)size;
+
+            if (ValidateMemory)
+            {
+                lock (_allocations!)
+                {
+                    Debug.Assert(_allocations!.Add((IntPtr)ptr));
+                }
+            }
+            if (_trackIncrementalAllocations)
+            {
+                lock (_allocationsDiff!)
+                {
+                    Debug.Assert(_allocationsDiff!.Add((IntPtr)ptr));
+                }
+            }
+            Interlocked.Add(ref TotalAllocatedMemory, (long)size);
+            Interlocked.Increment(ref TotalAllocations);
+
+            return (void*)((IntPtr)ptr + Offset);
+        }
+
+        [UnmanagedCallersOnly]
+        internal static unsafe void* CryptoRealloc(void* oldPtr, UIntPtr size, char* file, int line)
+        {
+            void * ptr;
+            Span<MemoryEntry> entry;
+
+            if (oldPtr != null)
+            {
+                IntPtr entryPtr = (IntPtr)oldPtr - Offset;
+                entry = new Span<MemoryEntry>((void*)entryPtr, 1);
+
+                if (ValidateMemory)
+                {
+                    lock (_allocations!)
+                    {
+                        if (!_allocations!.Remove(entryPtr))
+                        {
+                            Environment.FailFast($"Failed to find OpenSSL memory 0x{(IntPtr)oldPtr:x}");
+                        }
+                    }
+                }
+
+                if (_trackIncrementalAllocations)
+                {
+                    lock (_allocationsDiff!)
+                    {
+                        // this may fail as we may start tracking after given chunk was allocated
+                        _allocationsDiff!.Remove(entryPtr);
+                    }
+                }
+
+                Interlocked.Add(ref TotalAllocatedMemory, -((long)entry[0].Size));
+                ptr =  NativeMemory.Realloc((void*)entryPtr, size + (UIntPtr)Offset);
+            }
+            else
+            {
+                ptr =  NativeMemory.Alloc(size + (UIntPtr)Offset);
+            }
+
+
+            Debug.Assert(ptr != null);
+            if (ptr == null)
+            {
+                return null;
+            }
+
+            if (ValidateMemory)
+            {
+                lock (_allocations!)
+                {
+                    Debug.Assert(_allocations!.Add((IntPtr)ptr));
+                }
+            }
+            if (_trackIncrementalAllocations)
+            {
+                lock (_allocationsDiff!)
+                {
+                    Debug.Assert(_allocationsDiff!.Add((IntPtr)ptr));
+                }
+            }
+            Interlocked.Add(ref TotalAllocatedMemory, (long)size);
+            Interlocked.Increment(ref TotalAllocations);
+
+            entry = new Span<MemoryEntry>((void*)ptr, 1);
+            entry[0].Line = line;
+            entry[0].File = file;
+            entry[0].Size = (int)size;
+
+            return (void*)((IntPtr)ptr + Offset);
+        }
+
+        [UnmanagedCallersOnly]
+        internal static unsafe void CryptoFree(void* ptr)
+        {
+            if (ptr != null)
+            {
+                IntPtr entryPtr = (IntPtr)ptr - Offset;
+                if (ValidateMemory)
+                {
+                    lock (_allocations!)
+                    {
+                        {
+                            if (!_allocations!.Remove(entryPtr))
+                            {
+                                Environment.FailFast($"Failed to find OpenSSL memory 0x{(IntPtr)ptr:x}");
+                            }
+                        }
+                    }
+                }
+                if (_trackIncrementalAllocations)
+                {
+                    lock (_allocationsDiff!)
+                    {
+                        // this may fail as we may start tracking after given chunk was allocated
+                        _allocationsDiff!.Remove(entryPtr);
+                    }
+                }
+
+                Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)entryPtr, 1);
+                Interlocked.Add(ref TotalAllocatedMemory, -((long)entry[0].Size));
+
+                NativeMemory.Free((void*)entryPtr);
+            }
+        }
     }
 }

src/native/libs/System.Security.Cryptography.Native/apibridge_30.h

@@ -6,6 +6,10 @@
 #pragma once
 #include "pal_types.h"
 
+typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line);
+typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file, int line);
+typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line);
+
 typedef struct evp_mac_st EVP_MAC;
 typedef struct evp_mac_ctx_st EVP_MAC_CTX;
 
@@ -14,3 +18,5 @@ int local_EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX* ctx, const EVP_MD* md);
 int local_EVP_PKEY_CTX_set_rsa_padding(EVP_PKEY_CTX* ctx, int pad_mode);
 int local_EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX* ctx, int saltlen);
 int local_EVP_PKEY_CTX_set_signature_md(EVP_PKEY_CTX* ctx, const EVP_MD* md);
+
+int CRYPTO_set_mem_functions11(CRYPTO_malloc_fn malloc_fn, CRYPTO_realloc_fn realloc_fn, CRYPTO_free_fn free_fn);

src/native/libs/System.Security.Cryptography.Native/openssl.c

@@ -1467,7 +1467,7 @@ int32_t CryptoNative_OpenSslAvailable(void)
 static int32_t g_initStatus = 1;
 int g_x509_ocsp_index = -1;
 
-static int32_t EnsureOpenSslInitializedCore(void)
+static int32_t EnsureOpenSslInitializedCore(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction)
 {
     int ret = 0;
 
@@ -1476,7 +1476,28 @@ static int32_t EnsureOpenSslInitializedCore(void)
     // Otherwise call the 1.1 one.
 #ifdef FEATURE_DISTRO_AGNOSTIC_SSL
     InitializeOpenSSLShim();
+#endif
+
+    if (mallocFunction != NULL && reallocFunction != NULL && freefunction != NULL)
+    {
+        // This needs to be done before any allocation is done e.g. EnsureOpenSsl* is called.
+        // And it also needs to be after the pointers are loaded for DISTRO_AGNOSTIC_SSL
+#ifdef FEATURE_DISTRO_AGNOSTIC_SSL
+        if (!API_EXISTS(SSL_state))
+        {
+            // CRYPTO_set_mem_functions exists in OpenSSL 1.0.1 as well but it has different prototype
+            // and that makes it difficult to use with managed callbacks.
+            // Since 1.0 is long time out of support we use it only on 1.1.1+
+            CRYPTO_set_mem_functions11(mallocFunction, reallocFunction, freefunction);
+        }
+#elif OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_1_1_0_RTM
+        // OpenSSL 1.0 has different prototypes and it is out of support so we enable this only
+        // on 1.1.1+
+        CRYPTO_set_mem_functions(mallocFunction, reallocFunction, freefunction);
+#endif
+    }
 
+#ifdef FEATURE_DISTRO_AGNOSTIC_SSL
     if (API_EXISTS(SSL_state))
     {
         ret = EnsureOpenSsl10Initialized();
@@ -1501,15 +1522,22 @@ static int32_t EnsureOpenSslInitializedCore(void)
     return ret;
 }
 
+static CRYPTO_malloc_fn  _mallocFunction;
+static CRYPTO_realloc_fn _reallocFunction;
+static CRYPTO_free_fn _freefunction;
+
 static void EnsureOpenSslInitializedOnce(void)
 {
-    g_initStatus = EnsureOpenSslInitializedCore();
+    g_initStatus = EnsureOpenSslInitializedCore(_mallocFunction, _reallocFunction, _freefunction);
 }
 
 static pthread_once_t g_initializeShim = PTHREAD_ONCE_INIT;
 
-int32_t CryptoNative_EnsureOpenSslInitialized(void)
+int32_t CryptoNative_EnsureOpenSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction)
 {
+    _mallocFunction = mallocFunction;
+    _reallocFunction = reallocFunction;
+    _freefunction = freefunction;
     pthread_once(&g_initializeShim, EnsureOpenSslInitializedOnce);
     return g_initStatus;
 }

src/native/libs/System.Security.Cryptography.Native/openssl.h

@@ -69,7 +69,7 @@ PALEXPORT int32_t CryptoNative_GetRandomBytes(uint8_t* buf, int32_t num);
 
 PALEXPORT int32_t CryptoNative_LookupFriendlyNameByOid(const char* oidValue, const char** friendlyName);
 
-PALEXPORT int32_t CryptoNative_EnsureOpenSslInitialized(void);
+PALEXPORT int32_t CryptoNative_EnsureOpenSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction);
 
 PALEXPORT int64_t CryptoNative_OpenSslVersionNumber(void);
 

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -248,6 +248,7 @@ int EVP_DigestSqueeze(EVP_MD_CTX *ctx, unsigned char *out, size_t outlen);
     REQUIRED_FUNCTION(CRYPTO_malloc) \
     LEGACY_FUNCTION(CRYPTO_num_locks) \
     LEGACY_FUNCTION(CRYPTO_set_locking_callback) \
+    RENAMED_FUNCTION(CRYPTO_set_mem_functions11, CRYPTO_set_mem_functions) \
     REQUIRED_FUNCTION(d2i_ASN1_BIT_STRING) \
     REQUIRED_FUNCTION(d2i_BASIC_CONSTRAINTS) \
     REQUIRED_FUNCTION(d2i_EXTENDED_KEY_USAGE) \
@@ -775,6 +776,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define CRYPTO_malloc CRYPTO_malloc_ptr
 #define CRYPTO_num_locks CRYPTO_num_locks_ptr
 #define CRYPTO_set_locking_callback CRYPTO_set_locking_callback_ptr
+#define CRYPTO_set_mem_functions11 CRYPTO_set_mem_functions11_ptr
 #define d2i_ASN1_BIT_STRING d2i_ASN1_BIT_STRING_ptr
 #define d2i_BASIC_CONSTRAINTS d2i_BASIC_CONSTRAINTS_ptr
 #define d2i_EXTENDED_KEY_USAGE d2i_EXTENDED_KEY_USAGE_ptr

src/native/libs/System.Security.Cryptography.Native/pal_ssl.c

@@ -31,8 +31,6 @@ c_static_assert(TLSEXT_STATUSTYPE_ocsp == 1);
     "ECDHE-RSA-AES256-SHA384:" \
     "ECDHE-RSA-AES128-SHA256:" \
 
-int32_t CryptoNative_EnsureOpenSslInitialized(void);
-
 #ifdef NEED_OPENSSL_1_0
 static void EnsureLibSsl10Initialized(void)
 {
@@ -173,9 +171,11 @@ static void DetectCiphersuiteConfiguration(void)
 #endif
 }
 
-void CryptoNative_EnsureLibSslInitialized(void)
+
+void CryptoNative_EnsureLibSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction)
+
 {
-    CryptoNative_EnsureOpenSslInitialized();
+    CryptoNative_EnsureOpenSslInitialized(mallocFunction, reallocFunction, freefunction);
 
     // If portable, call the 1.0 initializer when needed.
     // If 1.0, call it statically.

src/native/libs/System.Security.Cryptography.Native/pal_ssl.h

@@ -130,7 +130,8 @@ typedef void (*SslCtxSetKeylogCallback)(const SSL* ssl, const char *line);
 /*
 Ensures that libssl is correctly initialized and ready to use.
 */
-PALEXPORT void CryptoNative_EnsureLibSslInitialized(void);
+
+PALEXPORT void CryptoNative_EnsureLibSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction);
 
 /*
 Shims the SSLv23_method method.