Use Boost python with libtorch

Question

I am trying to create a python package from C++ code with Boost python. However when including libtorchin the code, the resulting python package shows strange errors such as

Boost.Python.ArgumentError: Python argument types in Test.init(Test, int, str) did not match C++ signature: init(_object*, int, std::string)"

Creating a C++ executable from the code works fine and creating python packages with Boost python without libtorch dependencies works also fine, so the problem seems to come from libtorchand boost pythonnot working properly together. Has anybody encountered the same and found a solutoin to this?

Here is my example:

test.h

#ifndef TEST_H
#define TEST_H

#include <string>

class Test{

    int tensorSize;
    std::string label;

    public:
        Test(int i, std::string label);
    void printTensor();
};

#endif

test.cpp

#include <boost/python.hpp>
#include <string>
#include "test.h"
#include <torch/torch.h>
#include <torch/script.h>


#include <iostream>

Test::Test(int i, std::string lbl) {
    tensorSize = i;
    label = lbl;
}

void Test::printTensor(){
    auto x = torch::randn({10, tensorSize});

    std::cout << label << '\n';
    std::cout << "x:\n" << x << '\n';
}


using namespace boost::python;
BOOST_PYTHON_MODULE(test)
{
    class_<Test>("Test", init<int, std::string>())
    .def("printTensor", &Test::printTensor)
    ;
}

main.cpp (for the executable)

#include "test.h"

int main() {
    Test t(3, "test");
    t.printTensor();
    return 0;
}

and the test file for testing the python package

import test

t = test.Test(3, "test")
t.printTensor()

for compiling I use CMAKE, here the CMakeLists.txt

cmake_minimum_required(VERSION 3.20)
project(boostTest)

set(BOOST_MIN_VERSION 1.86.0)
find_package(Torch REQUIRED)
find_package(PythonLibs 3 REQUIRED)

find_package(
    Boost ${BOOST_MIN_VERSION} REQUIRED
    COMPONENTS python REQUIRED
)
#C++ executable
include_directories(${PYTHON_INCLUDE_PATH})
add_executable(test.out main.cpp test.cpp  test.h)
target_link_libraries(test.out ${PYTHON_LIBRARIES} ${TORCH_LIBRARIES} Boost::python)

#boost python
set(CMAKE_SHARED_MODULE_PREFIX "")
add_library(test MODULE test.cpp)
target_link_libraries(test ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} ${TORCH_LIBRARIES} )
target_include_directories(test PRIVATE ${PYTHON_INCLUDE_DIRS})

I compiled the example above. The resulting executable works fine, the python package throws an error.

It just works for me. I have a slightly different CMake and use Nix package manager. Python312, Boost 1.86. It looks like you might be compiling against a different version of python than when running the python script? — sehe
– sehe, Commented Nov 2, 2024 at 20:31
You are probably right, that's also my assumption. I have two python installations on my computer (3.10 and 3.12), but using either one (by modifying my jam file) does not help, I am using libtorch version 2.4.0, btw, what version are you using? Another funny thing, when creating a python file with swig everything works fine together with libtorch. — ppeloton
– ppeloton, Commented Nov 4, 2024 at 18:25

sehe · Accepted Answer · 2024-11-02 20:45:34Z

I tried to repro. All I ran into was the missing dependency on Boost Graph:

File CMakeLists.txt

cmake_minimum_required(VERSION 3.19)
project(sotest)
#set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
set(CMAKE_EXPORT_COMPILE_COMMANDS On)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -march=native ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic")

find_package(Torch REQUIRED)
link_libraries("${TORCH_LIBRARIES}")

find_package(Threads REQUIRED)
link_libraries(Threads::Threads)

find_package(Boost CONFIG 1.64.0 COMPONENTS python graph)
link_libraries(Boost::python Boost::graph)

find_package(Python3 COMPONENTS Development)
link_libraries(Python3::Python)

add_executable(sotest test.cpp main.cpp)
add_library(sopytorch SHARED test.cpp)

#target_include_directories(sotest PUBLIC ${Python3_INCLUDE_DIRS})

File test.h

#include <string>

class Test{

    int tensorSize;
    std::string label;

    public:
        Test(int i, std::string label);
    void printTensor();
};

File test.cpp

#include "test.h"
#include <boost/python.hpp>
#include <string>
#include <torch/script.h>
#include <torch/torch.h>

#include <iostream>

Test::Test(int i, std::string lbl) {
    tensorSize = i;
    label = lbl;
}

void Test::printTensor(){
    auto x = torch::randn({10, tensorSize});

    std::cout << label << '\n';
    std::cout << "x:\n" << x << '\n';
}


using namespace boost::python;
BOOST_PYTHON_MODULE(test)
{
    class_<Test>("Test", init<int, std::string>())
    .def("printTensor", &Test::printTensor)
    ;
}

File main.cpp

#include "test.h"

int main() {
    Test t(3, "test");
    t.printTensor();
}

File test.py

import test

t = test.Test(3, "test")
t.printTensor()

PS

Oh yeah, I forgot to mention a step, but I assume you know you need to copy the shared library so python can find it, e.g. with

ln -sfv build/libsopytorch.so test.so

Collectives™ on Stack Overflow

Use Boost python with libtorch

1 Answer 1

PS

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

PS

Comments

Your Answer

Sign up or log in

Post as a guest

Related