Passing existing C++ objects to Lua

If you want truly independent binding, here is classic C-style boilerplate. Just replace "Object" with your class name and implement // { } blocks. Turning push_Object from static to public will give you universal existing Object pusher that also caches objects in metatable (otherwise multi-push would create many distinct userdata with all-obvious gc-trouble).

You may also C++ify or library-fy that if you want, but I personally do not do this, because adding e.g. __newindex-to-environment proxy and other quirks would not be so straightforward if it was library. Actually, all boilerplate is in push_mt, push_Object, forget_Object and check_Object, everything other is subject to fine-tuning.

Note that this only binds a single class, not all classes at once.

// { class Object { ... } }

static const char *tname = "Object";

static void push_Object(lua_State *L, Object *object);
static Object *check_Object(lua_State *L, int i);

static int
l_gc(lua_State *L)
{
    Object **ud = luaL_checkudata(L, 1, tname);

    if (*ud) {
        // { delete *ud }
        *ud = NULL;
    }
    return 0;
}

static int
l_tostring(lua_State *L)
{
    Object **ud = luaL_checkudata(L, 1, tname);

    lua_pushfstring(L, "%s: %p", tname, *ud);
    return 1;
}

static int
l_new(lua_State *L)
{
    Object *object = NULL; // { = new Object }

    push_Object(L, object);
    return 1;
}

static int
l_method(lua_State *L)
{
    Object *object = check_Object(L, 1);
    lua_Integer int_arg = luaL_checkinteger(L, 2);
    const char *str_arg = luaL_checklstring(L, 3, NULL);

    // { object->method(int_arg, str_arg) }

    return 0;
}

static const luaL_Reg lib[] = {
    // functions
    { "new",    l_new    }, // () -> object

    // methods
    { "method", l_method }, // (object, int, string) -> none

    { NULL, NULL },
};
static lua_CFunction first_m = l_method;

static void
push_mt(lua_State *L)
{
    if (luaL_newmetatable(L, tname)) {
        size_t m = 0; while (first_m != lib[m].func) m++;
        lua_createtable(L, 0, 0);
        luaL_register(L, NULL, &lib[m]);
        lua_setfield(L, -2, "__index");

        lua_pushcfunction(L, l_tostring);
        lua_setfield(L, -2, "__tostring");

        lua_pushcfunction(L, l_gc);
        lua_setfield(L, -2, "__gc");

        lua_pushstring(L, tname);
        lua_setfield(L, -2, "__metatable");

        // mt.objects = setmetatable({ }, { __mode = "v" })
        lua_createtable(L, 0, 0);
        lua_createtable(L, 0, 1);
        lua_pushstring(L, "v");
        lua_setfield(L, -2, "__mode");
        lua_setmetatable(L, -2);
        lua_setfield(L, -2, "objects");
    }
}

static void
push_Object(lua_State *L, Object *object)
{
    int top = lua_gettop(L);

    push_mt(L);
    lua_getfield(L, -1, "objects");
    // top+1 = mt
    // top+2 = mt.objects

    // ud = mt.objects[object]
    lua_pushlightuserdata(L, object);
    lua_gettable(L, top+2);

    if (lua_isnil(L, -1)) {
        lua_pop(L, 1);

        Object **ud = lua_newuserdata(L, sizeof(*ud));
        *ud = object;

        // setmetatable(ud, mt)
        lua_pushvalue(L, top+1);
        lua_setmetatable(L, -2);

        // mt.objects[object] = ud
        lua_pushlightuserdata(L, object);
        lua_pushvalue(L, -3);
        lua_pushvalue(L, top+2);
    }

    // return ud
    lua_replace(L, top+1);
    lua_settop(L, top+1);
    return; // ud at top
}

static void
forget_Object(lua_State *L, Object *object)
{
    int top = lua_gettop(L);

    push_mt(L);
    lua_getfield(L, -1, "objects");
    // top+1 = mt
    // top+2 = mt.objects

    // ud = mt.objects[object]
    lua_pushlightuserdata(L, object);
    lua_pushnil(L);
    lua_settable(L, top+2);

    lua_settop(L, top);
}

static Object *
check_Object(lua_State *L, int i)
{
    Object **ud = luaL_checkudata(L, i, tname);
    Object *object = *ud;

    if (object == NULL)
        luaL_error(L, "%s is finalized", tname);

    return object;
}

int
luaopen_Object(lua_State *L)
{
    push_mt(L); // register tname

    lua_createtable(L, 0, sizeof(lib)-1);
    luaL_register(L, NULL, lib);
    return 1;
}

https://github.com/Rapptz/sol

Sol is the perfect example of a library that can support C++11+-like semantics, including the passing of class types using templates. If I were you, I'd look into Sol's source code and try to replicate its methods of object and data passing--much of Lua's "meta" functionality is probably build on its userdata structure, so I'd suggest you start looking there.

By the way, Sol is dependent on only Lua, and is header-inclusion only. It's quite possible to just use it out of the box, provided you're using Lua 5.2 (and not 5.3, which is a big gripe of the community but shouldn't matter, as LuaJIT is also on 5.2, I believe).

Most of what I've found tends to answer the question "how to create an instance of a C++ class in Lua" rather than just passing it, which leads me to think whether is my idea even right or not.

Those are answers to your problem, too. You can't create a C++ class in Lua. Any class construction is going to be done in the C++ code.

If you're not going to use a binding library, then you're going to need to be an export (no fear, it's not hard) on Lua's C API and metatables.

You expose the C++ object to Lua as a "userdata". You then supply a metatable with an __index metamethod which allows you to write C++ handlers for when the object is indexed, such as when a property is being accessed (e.g. pt.x) or when a method is being called (e.g. entity:attack(target)). You also need a __gc metamethod so you can delete the C++ object when the corresponding Lua userdata is garbage collected.

There can be quite a bit of boilerplate code, which is what many C++ binding libraries are supposed to take care of. I've never used one, so I can't recommend one. However, due to the Law of Leaky Abstractions, I would strongly recommend taking a stab at this on your own so that you understand how it works.