From 2f1fb09bfc408f1a821201ab3f5fe0883846c6d2 Mon Sep 17 00:00:00 2001 From: Roman Starkov Date: Tue, 29 Nov 2022 17:58:54 +0000 Subject: [PATCH 1/3] Dispose of ctypes to fix np.array holding on to memory for too long (fixes #108) --- src/Numpy/Manual/np.array.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Numpy/Manual/np.array.cs b/src/Numpy/Manual/np.array.cs index 11441ee..7157d25 100644 --- a/src/Numpy/Manual/np.array.cs +++ b/src/Numpy/Manual/np.array.cs @@ -52,7 +52,8 @@ public static NDarray array(T[] @object, Dtype dtype = null, bool? copy = var ndarray = np.empty(new Shape(@object.Length), dtype: type, order: order); if (@object.Length == 0) return new NDarray(ndarray); - long ptr = ndarray.PyObject.ctypes.data; + var ctypes = ndarray.PyObject.ctypes; + long ptr = ctypes.data; switch ((object)@object) { case char[] a: Marshal.Copy(a, 0, new IntPtr(ptr), a.Length); break; @@ -80,6 +81,7 @@ public static NDarray array(T[] @object, Dtype dtype = null, bool? copy = ndarray.imag = ndimag; break; } + ctypes.Dispose(); if (dtype !=null || subok != null || ndmin != null) return new NDarray(np.array(ndarray, dtype:dtype, copy: false, subok: subok, ndmin: ndmin)); return new NDarray(ndarray); From edaa8d69b07c141d120ebd983a0464e10ab01cf7 Mon Sep 17 00:00:00 2001 From: Roman Starkov Date: Tue, 29 Nov 2022 17:59:45 +0000 Subject: [PATCH 2/3] Unit test for np.array memory disposal --- .../NumPy_array_creation.tests.cs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/Numpy.UnitTest/NumPy_array_creation.tests.cs b/test/Numpy.UnitTest/NumPy_array_creation.tests.cs index 8b14d29..12f0e3a 100644 --- a/test/Numpy.UnitTest/NumPy_array_creation.tests.cs +++ b/test/Numpy.UnitTest/NumPy_array_creation.tests.cs @@ -355,6 +355,38 @@ public void full_likeTest() #endif } + [TestMethod] + public void arrayLeakTest() + { + var arr = new double[10_000_000]; + using (var process = System.Diagnostics.Process.GetCurrentProcess()) + { + long memStart; + long getMemAfterGc() + { + GC.Collect(2, GCCollectionMode.Forced, true, true); + process.Refresh(); + return process.PrivateMemorySize64; + } + long getOutstandingMem() => getMemAfterGc() - memStart; + + var ones = np.ones(10_000_000); // python runtime warmup + ones.Dispose(); + + memStart = getMemAfterGc(); + + ones = np.ones(10_000_000); + Assert.IsTrue(getOutstandingMem() > 70_000_000); + ones.Dispose(); + Assert.IsTrue(getOutstandingMem() < 1_000_000); + + var array = np.array(arr); + Assert.IsTrue(getOutstandingMem() > 70_000_000); + array.Dispose(); + Assert.IsTrue(getOutstandingMem() < 1_000_000); + } + } + [TestMethod] public void arrayTest() { From dc2670a85c44de4d0acc5d2d57aaa01d12f223cd Mon Sep 17 00:00:00 2001 From: Roman Starkov Date: Wed, 30 Nov 2022 09:26:14 +0000 Subject: [PATCH 3/3] Dispose of temporary array in the other path through np.array --- src/Numpy/Manual/np.array.cs | 9 +++++++-- test/Numpy.UnitTest/NumPy_array_creation.tests.cs | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Numpy/Manual/np.array.cs b/src/Numpy/Manual/np.array.cs index 7157d25..b1f4759 100644 --- a/src/Numpy/Manual/np.array.cs +++ b/src/Numpy/Manual/np.array.cs @@ -42,6 +42,7 @@ public static NDarray array(NDarray @object, Dtype dtype = null, bool? copy = nu if (subok != null) kwargs["subok"] = ToPython(subok); if (ndmin != null) kwargs["ndmin"] = ToPython(ndmin); dynamic py = self.InvokeMethod("array", args, kwargs); + args.Dispose(); return ToCsharp(py); } @@ -82,8 +83,12 @@ public static NDarray array(T[] @object, Dtype dtype = null, bool? copy = break; } ctypes.Dispose(); - if (dtype !=null || subok != null || ndmin != null) - return new NDarray(np.array(ndarray, dtype:dtype, copy: false, subok: subok, ndmin: ndmin)); + if (dtype != null || subok != null || ndmin != null) + { + var converted = np.array(ndarray, dtype: dtype, copy: false, subok: subok, ndmin: ndmin); + ndarray.Dispose(); + return new NDarray(converted); + } return new NDarray(ndarray); } diff --git a/test/Numpy.UnitTest/NumPy_array_creation.tests.cs b/test/Numpy.UnitTest/NumPy_array_creation.tests.cs index 12f0e3a..c4d81a1 100644 --- a/test/Numpy.UnitTest/NumPy_array_creation.tests.cs +++ b/test/Numpy.UnitTest/NumPy_array_creation.tests.cs @@ -384,6 +384,11 @@ long getMemAfterGc() Assert.IsTrue(getOutstandingMem() > 70_000_000); array.Dispose(); Assert.IsTrue(getOutstandingMem() < 1_000_000); + + array = np.array(arr, np.float32); + Assert.IsTrue(getOutstandingMem() > 30_000_000); + array.Dispose(); + Assert.IsTrue(getOutstandingMem() < 1_000_000); } }