Python extension in C - Metaclass

Question

I have the following python code:

class Meta(type):
    def __call__(cls, *args, **kwargs):
        obj = type.__call__(cls, *args, **kwargs)
        # Only do checks for subclasses
        if cls.__name__ == 'Parent':
            return obj
        required_attrs = ['x']
        for ra in required_attrs:
            if ra not in dir(obj):
                fmt = 'Subclasses of Parent must define the %s attribute'
                raise NotImplementedError(fmt % ra)
        return obj

class Parent(metaclass=Meta):
    pass

class Child(Parent):
    def __init__(self):
        self.x = True

Meta is used only to require that Child defines certain attributes. This class structure must remain as is because this is how my project is structured. Parent is actually called DefaultConfig and Child is actually a user-defined class derived from DefaultConfig.

I'm working on translating Meta and Parent into a C extension. This is the module:

#include <Python.h>
#include <structmember.h>

#define ARRLEN(x) sizeof(x)/sizeof(x[0])


typedef struct {
    PyObject_HEAD
} MetaObject;

typedef struct {
    PyObject_HEAD
} ParentObject;


static PyObject *Meta_call(MetaObject *type, PyObject *args, PyObject *kwargs) {
    PyObject *obj = PyType_GenericNew((PyTypeObject *) type, args, kwargs);

    // Only do checks for subclasses of Parent
    if (strcmp(obj->ob_type->tp_name, "Parent") == 0)
        return obj;

    // Get obj's attributes
    PyObject *obj_dir = PyObject_Dir(obj);
    if (obj_dir == NULL)
        return NULL;

    char *required_attrs[] = {"x"};

    // Raise an exception of obj doesn't define all required_attrs
    PyObject *attr_obj;
    int has_attr;
    for (int i=0; i<ARRLEN(required_attrs); i++) {
        attr_obj = PyUnicode_FromString(required_attrs[i]);
        has_attr = PySequence_Contains(obj_dir, attr_obj);
        if (has_attr == 0) {
            printf("Subclasses of Parent must define %s\n", required_attrs[i]);
            // raise NotImplementedError
            return NULL;
        } else if (has_attr == -1) {
            return NULL;
        }
    }

    return obj;
}


static PyTypeObject MetaType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "custom.Meta",
    .tp_basicsize = sizeof(MetaObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
    .tp_call = (ternaryfunc) Meta_call,
};

static PyTypeObject ParentType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "custom.Parent",
    .tp_basicsize = sizeof(ParentObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
};


static PyModuleDef custommodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "custom",
    .m_size = -1,
};


PyMODINIT_FUNC PyInit_custom(void) {
    PyObject *module = PyModule_Create(&custommodule);
    if (module == NULL)
        return NULL;

    // Should Parent inherit from Meta?
    ParentType.tp_base = &MetaType;

    if (PyType_Ready(&MetaType) < 0)
        return NULL;
    Py_INCREF(&MetaType);
    PyModule_AddObject(module, "Meta", (PyObject *) &MetaType);

    if (PyType_Ready(&ParentType) < 0)
        return NULL;
    Py_INCREF(&ParentType);
    PyModule_AddObject(module, "Parent", (PyObject *) &ParentType);

    return module;
}

This is the python code used to test module custom:

import custom

class Child(custom.Parent):
    def __init__(self):
        self.x = True

if __name__ == '__main__':
    c = Child()

Unfortunately, there is no .tp_meta member in the PyTypeObject struct, so how do I specify Meta as the metaclass of Parent?

EDIT:

Modified C code:

#include <Python.h>
#include <structmember.h>

#define ARRLEN(x) sizeof(x)/sizeof(x[0])


typedef struct {
    PyObject_HEAD
    PyTypeObject base;
} MetaObject;

typedef struct {
    PyObject_HEAD
} ParentObject;


static PyObject *Meta_call(MetaObject *type, PyObject *args, PyObject *kwargs) {
    PyObject *obj = PyType_GenericNew((PyTypeObject *) type, args, kwargs);

    // Only do checks for subclasses of Parent
    if (strcmp(obj->ob_type->tp_name, "Parent") == 0)
        return obj;

    // Get obj's attributes
    PyObject *obj_dir = PyObject_Dir(obj);
    if (obj_dir == NULL)
        return NULL;

    char *required_attrs[] = {"x"};

    // Raise an exception of obj doesn't define all required_attrs
    PyObject *attr_obj;
    int has_attr;
    for (int i=0; i<ARRLEN(required_attrs); i++) {
        attr_obj = PyUnicode_FromString(required_attrs[i]);
        has_attr = PySequence_Contains(obj_dir, attr_obj);
        if (has_attr == 0) {
            printf("Subclasses of Parent must define %s\n", required_attrs[i]);
            // raise NotImplementedError
            return NULL;
        } else if (has_attr == -1) {
            return NULL;
        }
    }

    return obj;
}


static PyTypeObject MetaType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "custom.Meta",
    .tp_basicsize = sizeof(MetaObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
    .tp_call = (ternaryfunc) Meta_call,
};

static PyTypeObject ParentType = {
    PyVarObject_HEAD_INIT(&MetaType, 0)
    .tp_name = "custom.Parent",
    .tp_basicsize = sizeof(ParentObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = PyType_GenericNew,
};


static PyModuleDef custommodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "custom",
    .m_size = -1,
};


PyMODINIT_FUNC PyInit_custom(void) {
    PyObject *module = PyModule_Create(&custommodule);
    if (module == NULL)
        return NULL;

    MetaType.tp_base = &PyType_Type;
    if (PyType_Ready(&MetaType) < 0)
        return NULL;
    Py_INCREF(&MetaType);
    PyModule_AddObject(module, "Meta", (PyObject *) &MetaType);

    if (PyType_Ready(&ParentType) < 0)
        return NULL;
    Py_INCREF(&ParentType);
    PyModule_AddObject(module, "Parent", (PyObject *) &ParentType);

    return module;
}

Antti Haapala · Accepted Answer · 2022-06-23 16:54:07Z

2

The metaclass is nothing but a type that is used as the type (ob_type!) of the class (type)... (clear, isn't it)... ParentType does not inherit from MetaType but is an instance of `MetaType.

Hence, the place where &MetaType should go (if anywhere), is ParentType.ob_type:

PyModule_AddObject(module, "Meta", (PyObject *) &MetaType);

ParentType.ob_type = &MetaType;

if (PyType_Ready(&ParentType) < 0)

PyType_Ready checks the ob_type field - if it is NULL, it takes the ob_type of the .tp_base; but if ob_type is set already, it is left as is.

Actually you can set it in the ParentType initializer:

PyVarObject_HEAD_INIT(&MetaType, 0)

The first argument goes to the ob_type field.

edited Jun 23, 2022 at 16:54

answered Oct 23, 2018 at 21:18

Antti Haapala

135k23 gold badges297 silver badges349 bronze badges

Sign up to request clarification or add additional context in comments.

6 Comments

Josh Weinstein Over a year ago

It seems though the python source code has a special attribute called PyId_metaclass that it uses to determine the meta class .... github.com/python/cpython/blob/…

Antti Haapala Over a year ago

@JoshWeinstein on the contrary, this is just the string "metaclass", used to get the metaclass=FooBar from the kwargs.

user2357112 Over a year ago

Setting ob_type isn't going to be enough, though, because there are other problems with the code in the question. For example, MetaObject doesn't include a PyTypeObject base member, and MetaType's tp_base needs to be set to PyType_Type.

user2357112 Over a year ago

Also, things like PyId_metaclass aren't quite just strings - they're wrappers used to manage interned, static strings. Python still ends up using it to fetch metaclass from the kwargs by string name, though. It's not any sort of special attribute, and it's not how the metaclass of an actual class object is stored.

user2357112 Over a year ago

@Nelson: From a quick glance, you don't need the PyObject_HEAD in MetaObject; the PyTypeObject's PyObject_HEAD takes care of that. There may be other problems. See the relevant section of the C API tutorial, and Modules/xxsubtype.c for an example of implementing subclasses of built-in types in C.

|

Josh Weinstein · Accepted Answer · 2018-10-23 21:35:22Z

-1

There is no direct way to do this. According to the py docs, there is no members or flags to directly indicate a class is a meta class of another. The attribute responsible for indicating a meta class is inside the class dictionary. You could implement something that modifies the .tp_dict member, but this is actually deemed unsafe if done through the dictionary C-API.

Warning It is not safe to use PyDict_SetItem() on or otherwise modify tp_dict with the dictionary C-API.

EDIT:

From the python source code, it seems meta class is accessed as an id via the C dictionary API, but the methods to do so are prefixed with an _, and don't appear in any documentation.

    meta = _PyDict_GetItemId(mkw, &PyId_metaclass);
    if (meta != NULL) {
        Py_INCREF(meta);
        if (_PyDict_DelItemId(mkw, &PyId_metaclass) < 0) {
            Py_DECREF(meta);
            Py_DECREF(mkw);
            Py_DECREF(bases);
            return NULL;
        }

These methods are apart of the "limited api", and can be used by defining the Py_LIMITED_API macro

PyAPI_FUNC(PyObject *) _PyDict_GetItemId(PyObject *dp, struct _Py_Identifier *key);
#endif /* !Py_LIMITED_API */
PyAPI_FUNC(int) PyDict_SetItemString(PyObject *dp, const char *key, PyObject *item);
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, struct _Py_Identifier *key, PyObject *item);
#endif /* !Py_LIMITED_API */

edited Oct 23, 2018 at 21:35

answered Oct 23, 2018 at 20:41

Josh Weinstein

2,9782 gold badges24 silver badges46 bronze badges

7 Comments

user2357112 Over a year ago

There is a member indicating a class's metaclass; it's part of PyObject_HEAD, and it's the same member used for any other object's class. The __metaclass__ dict entry isn't used in Python 3, and it was only ever used in Python 2 during class creation, not to determine the metaclass of an already-created class.

user2357112 Over a year ago

Setting __metaclass__ in the dict of an existing class won't affect its metaclass. Also, messing with tp_dict after PyType_Ready is called is still unsafe.

Josh Weinstein Over a year ago

Can you link to where that part of PyObject_HEAD is? not documented here docs.python.org/3/c-api/structures.html#c.PyObject

Nelson Over a year ago

/usr/include/python3.6/object.h:83 #define PyObject_HEAD PyObject ob_base;. I don't understand how a PyObject contains the information about a type's metaclass

Josh Weinstein Over a year ago

Updated my answer to include the place in the python source code that checks an objects meta class. It is not in PyObject_HEAD

|

Collectives™ on Stack Overflow

Python extension in C - Metaclass

2 Answers 2

6 Comments

7 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

6 Comments

7 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related