Conversion from np.dtype int to .NET int fails

[myrthel]

Environment

• Pythonnet version: 3.0.2
• Python version: 3.10
• Operating System: Windows 10
• .NET Runtime: net7.0

Details

• I was trying to import some data from a np.Array() and got some error at conversation of np.dtype = int32 to Int32 in csharp and got an error. After this I debugged it and found a way how it works. I cloned the repo and edited the following line in Python.Runtime's Converter.cs:

    nint num = Runtime.PyLong_AsSignedSize_t(value);

to

    long? num = Runtime.PyLong_AsLongLong(value);

After this I wrote a test for all numpy int datatypes:

    using Python.Runtime;

namespace DataManager.Test;
[TestClass]
public class PythonnetTest
{
    [TestMethod]
    public void TestNumpyArray()
    {
        //Initialize();
        SByte ll_int8 = -128;
        SByte ul_int8 = 127;
        SByte[] int8 = { ll_int8, ul_int8 };
        Int16 ll_int16 = -32768;
        Int16 ul_int16 = 32767;
        Int16[] int16 = { ll_int16, ul_int16 };
        Int32 ll_int32 = -2147483648;
        Int32 ul_int32 = 2147483647;
        Int32[] int32 = { ll_int32, ul_int32 };
        Int64 ll_int64 = -9223372036854775808;
        Int64 ul_int64 = 9223372036854775807;
        Int64[] int64 = { ll_int64, ul_int64 };

        Byte ll_uint8 = 0;
        Byte ul_uint8 = 255;
        Byte[] uint8 = { ll_uint8, ul_uint8 };
        UInt16 ll_uint16 = 0;
        UInt16 ul_uint16 = 65535;
        UInt16[] uint16 = { ll_uint16, ul_uint16 };
        UInt32 ll_uint32 = 0;
        UInt32 ul_uint32 = 4294967295;
        UInt32[] uint32 = { ll_uint32, ul_uint32 };
        UInt64 ll_uint64 = 0;
        UInt64 ul_uint64 = 18446744073709551615;
        UInt64[] uint64 = { ll_uint64, ul_uint64 };
        SByte[] arr = new SByte[10];

        using (Py.GIL())
        {
            //dynamic lb = Py.Import("asammdf");
            //dynamic mdf = lb.MDF(filename);
            //dynamic iterator = mdf.iter_channels();


            using (PyModule scope = Py.CreateScope("numpytest"))
            {
                
                scope.Set("ll_int8", ll_int8.ToPython());
                scope.Set("ul_int8", ul_int8.ToPython());
                scope.Set("ll_int16", ll_int16.ToPython());
                scope.Set("ul_int16", ul_int16.ToPython());
                scope.Set("ll_int32", ll_int32.ToPython());
                scope.Set("ul_int32", ul_int32.ToPython());
                scope.Set("ll_int64", ll_int64.ToPython());
                scope.Set("ul_int64", ul_int64.ToPython());

                scope.Set("ll_uint8", ll_uint8.ToPython());
                scope.Set("ul_uint8", ul_uint8.ToPython());
                scope.Set("ll_uint16", ll_uint16.ToPython());
                scope.Set("ul_uint16", ul_uint16.ToPython());
                scope.Set("ll_uint32", ll_uint32.ToPython());
                scope.Set("ul_uint32", ul_uint32.ToPython());
                scope.Set("ll_uint64", ll_uint64.ToPython());
                scope.Set("ul_uint64", ul_uint64.ToPython());
                
                scope.Exec(
                    "import numpy as np\n" +
                    "np_int8   = np.array([ll_int8, ul_int8],     dtype = np.int8)\n" +
                    "np_int16  = np.array([ll_int16, ul_int16],   dtype = np.int16)\n" +
                    "np_int32  = np.array([ll_int32, ul_int32],   dtype = np.int32)\n" +
                    "np_int64  = np.array([ll_int64, ul_int64],   dtype = np.int64)\n" +
                    "np_uint8  = np.array([ll_uint8, ul_uint8],   dtype = np.uint8)\n" +
                    "np_uint16 = np.array([ll_uint16, ul_uint16], dtype = np.uint16)\n" +
                    "np_uint32 = np.array([ll_uint32, ul_uint32], dtype = np.uint32)\n" +
                    "np_uint64 = np.array([ll_uint64, ul_uint64], dtype = np.uint64)\n"
                    );
                dynamic np_int8   = scope.Get<object>("np_int8");
                dynamic np_int16  = scope.Get<object>("np_int16");
                dynamic np_int32  = scope.Get<object>("np_int32");
                dynamic np_int64  = scope.Get<object>("np_int64");
                dynamic np_uint8  = scope.Get<object>("np_uint8");
                dynamic np_uint16 = scope.Get<object>("np_uint16");
                dynamic np_uint32 = scope.Get<object>("np_uint32");
                dynamic np_uint64 = scope.Get<object>("np_uint64");

                SByte[] cs_np_int8  = np_int8. As<SByte[]>();
                Int16[] cs_np_int16 = np_int16.As<Int16[]>();
                Int32[] cs_np_int32 = np_int32.As<Int32[]>();
                Int64[] cs_np_int64 = np_int64.As<Int64[]>();

                Byte[]   cs_np_uint8  = np_uint8.As<Byte[]>();
                UInt16[] cs_np_uint16 = np_uint16.As<UInt16[]>();
                UInt32[] cs_np_uint32 = np_uint32.As<UInt32[]>();
                //UInt64[] cs_np_uint64 = np_uint64.As<UInt64[]>(); // Fails

                Assert.IsTrue(Enumerable.SequenceEqual(int8,cs_np_int8));
                Assert.IsTrue(Enumerable.SequenceEqual(int16, cs_np_int16));
                Assert.IsTrue(Enumerable.SequenceEqual(int32, cs_np_int32));
                Assert.IsTrue(Enumerable.SequenceEqual(int64, cs_np_int64));
                Assert.IsTrue(Enumerable.SequenceEqual(uint8, cs_np_uint8));
                Assert.IsTrue(Enumerable.SequenceEqual(uint16, cs_np_uint16));
                Assert.IsTrue(Enumerable.SequenceEqual(uint32, cs_np_uint32));
                //Assert.IsTrue(Enumerable.SequenceEqual(uint64, cs_np_uint64));
            }
        }
    }

    [TestInitialize]
    public void Initialize()
    {
        string pythonDll = @"C:\Users\m.huemer\AppData\Local\Programs\Python\Python310\python310.dll";
        Environment.SetEnvironmentVariable("PYTHONNET_PYDLL", pythonDll);
        PythonEngine.Initialize();
        
    }
}

I encountered, that it fails in all cases, and replaced all affected lines in the switch case of the ToPrimitive() function and casted the type accordingly.
This worked well, except for the UInt64[].

[filmor]

What was the error that you got before?

[myrthel]

Where can I see the message from python side? - Is this to find in Python.Runtime Exceptions Class?
I am getting this error in the debugger:
System.InvalidCastException: cannot convert object to target type ---> Python.Runtime.PythonException: an integer is required

[myrthel]

For everyone who struggles with numpy-arrays and its conversation.
I have found a good solution for that.

using PyBuffer pybuf_array = np_array.data.GetBuffer();
var len = pybuf_array.Length;
byte[] managedArray = new byte[len];
pybuf_array.Read(managedArray, 0, managedArray.Length, 0);
var values = MemoryMarshal.Cast<byte, double>(managedArray).ToArray();

But there are also other ways to get the array into your memory after this. For example Buffer.BlockCopy.
Main thing is, the data is contiguous. The method is much faster, than converting the array as it is not converting every value as individual.