Debug utility to load/save numpy matrices in C++ keeps crashing.

[Svalorzen]

Hello, I'm trying to write a small utility for debugging purposes, which I want to use to save/load numpy matrices into Eigen matrices. It is not supposed to be a permanent fixture, but something I want to use from time to time to debug certain parts of my code.

The problem is that both the loading and saving of the matrices crashes my program.

For loading, I did manage at some point to make it work, but I can't remember what I did differently from my current code. Now it crashes during PyTuple_New, and I think a cast is happening..

For saving, I never managed to make it work; the code crashes with a PyGILState_Ensure error, which I have seen is related to the lifetime of the interpreter, but I cannot see how it would be relevant here as the interpreter should last the entirety of the program.

Could you help me figure out what is wrong? Thanks in advance. My code is below.

#include <iostream>
#include <Eigen/Core>

#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/eigen.h>
#include <pybind11/numpy.h>

template <typename T>
using MatrixT = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor | Eigen::AutoAlign>;

namespace py = pybind11;

struct PythonHelper
{
    PythonHelper();
    ~PythonHelper();

    // Load numpy matrix saved via np.save()
    template <typename T>
    MatrixT<T> loadFromNumpyMatrix (const std::string& filename);

    template <typename T>
    void saveToNumpyMatrix (const std::string& filename, const MatrixT<T>&);

    struct Impl;
    std::unique_ptr<Impl> m_impl;
};

struct PythonHelper::Impl
{
    Impl();

    py::scoped_interpreter python;
    py::module np;
    py::object load;
    py::object save;
};

PythonHelper::Impl::Impl()
{
    np = py::module::import ("numpy");
    load = np.attr ("load");
    save = np.attr ("save");
}

PythonHelper::PythonHelper() = default;
PythonHelper::~PythonHelper() = default;

template <typename T>
MatrixT<T> PythonHelper::loadFromNumpyMatrix (const std::string& filename)
{
    MatrixT<T> matrix;
    try
    {
        matrix = m_impl->load (filename).template cast<MatrixT<T>>();
    }
    catch (const py::error_already_set& e)
    {
        std::cout << "Captured error_already_set: " << e.what() << '\n';
        std::abort();
    }
    catch ( const std::exception& e )
    {
        std::cout << "Captured exception: " << e.what() << '\n';
        std::abort();
    }
    catch ( ... )
    {
        std::cout << "Just failed...\n";
        std::abort();
    }
    return matrix;
}

template <typename T>
void PythonHelper::saveToNumpyMatrix (const std::string& filename, const MatrixT<T>& m)
{
    try
    {
        m_impl->save (filename, m);
    }
    catch (const py::error_already_set& e)
    {
        std::cout << "Captured error_already_set: " << e.what() << '\n';
        std::abort();
    }
    catch ( const std::exception& e )
    {
        std::cout << "Captured exception: " << e.what() << '\n';
        std::abort();
    }
    catch ( ... )
    {
        std::cout << "Just failed...\n";
        std::abort();
    }
}

int main() {
    PythonHelper p;
    
    // Fails on PyTuple_New (I think during a cast?)
    auto m = p.loadFromNumpyMatrix<double>("matrix.npy");
    std::cout << m << '\n';

    // Fails with PyGILState_Ensure error.
    MatrixT<double> x(3, 3);
    x << 1, 2, 3, 4, 5, 6, 7, 8, 9;
    p.saveToNumpyMatrix("test.npy", x);

    return 0;
}

[Svalorzen]

Sorry, I realized I forgot to initialize the Impl object...