pybind11.h 50.5 KB
Newer Older
Wenzel Jakob's avatar
Wenzel Jakob committed
1
/*
2
    pybind11/pybind11.h: Main header file of the C++11 python binding generator library
Wenzel Jakob's avatar
Wenzel Jakob committed
3
4
5
6
7
8
9

    Copyright (c) 2015 Wenzel Jakob <wenzel@inf.ethz.ch>

    All rights reserved. Use of this source code is governed by a
    BSD-style license that can be found in the LICENSE file.
*/

10
#pragma once
Wenzel Jakob's avatar
Wenzel Jakob committed
11
12
13
14
15
16
17
18

#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable: 4127) // warning C4127: Conditional expression is constant
#pragma warning(disable: 4800) // warning C4800: 'int': forcing value to bool 'true' or 'false' (performance warning)
#pragma warning(disable: 4996) // warning C4996: The POSIX name for this item is deprecated. Instead, use the ISO C and C++ conformant name
#pragma warning(disable: 4100) // warning C4100: Unreferenced formal parameter
#pragma warning(disable: 4512) // warning C4512: Assignment operator was implicitly defined as deleted
19
#elif defined(__GNUG__) and !defined(__clang__)
20
21
22
23
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-but-set-parameter"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
Wenzel Jakob's avatar
Wenzel Jakob committed
24
25
#endif

26
#include "cast.h"
Wenzel Jakob's avatar
Wenzel Jakob committed
27

28
NAMESPACE_BEGIN(pybind11)
Wenzel Jakob's avatar
Wenzel Jakob committed
29

30
31
32
33
34
template <typename T> struct arg_t;

/// Annotation for keyword arguments
struct arg {
    arg(const char *name) : name(name) { }
Wenzel Jakob's avatar
Wenzel Jakob committed
35
    template <typename T> arg_t<T> operator=(const T &value);
36
37
38
39
40
    const char *name;
};

/// Annotation for keyword arguments with default values
template <typename T> struct arg_t : public arg {
41
42
    arg_t(const char *name, const T &value, const char *descr = nullptr)
        : arg(name), value(value), descr(descr) { }
43
    T value;
44
    const char *descr;
45
};
46

Wenzel Jakob's avatar
Wenzel Jakob committed
47
template <typename T> arg_t<T> arg::operator=(const T &value) { return arg_t<T>(name, value); }
48
49

/// Annotation for methods
Wenzel Jakob's avatar
Wenzel Jakob committed
50
struct is_method { PyObject *class_; is_method(object *o) : class_(o->ptr()) { } };
51
52
53
54
55
56
57

/// Annotation for documentation
struct doc { const char *value; doc(const char *value) : value(value) { } };

/// Annotation for function names
struct name { const char *value; name(const char *value) : value(value) { } };

58
/// Annotation indicating that a function is an overload associated with a given "sibling"
59
60
struct sibling { PyObject *value; sibling(handle value) : value(value.ptr()) { } };

61
62
63
64
/// Keep patient alive while nurse lives
template <int Nurse, int Patient> struct keep_alive { };

NAMESPACE_BEGIN(detail)
65
66

/// Partial template helper to invoke function call policies (e.g. keep_alive) when a function is called
67
template <typename... Args> struct process_dynamic;
68
69

/// Default implementation: do nothing
70
71
72
73
template <typename T> struct process_dynamic<T> {
    static void precall(PyObject *) { }
    static void postcall(PyObject *, PyObject *) { }
};
74
75

/// Recursively iterate over variadic template arguments
76
77
78
79
80
81
82
83
84
85
template <typename T, typename... Args> struct process_dynamic<T, Args...> {
    static void precall(PyObject *arg) {
        process_dynamic<T>::precall(arg);
        process_dynamic<Args...>::precall(arg);
    }
    static void postcall(PyObject *arg, PyObject *ret) {
        process_dynamic<T>::postcall(arg, ret);
        process_dynamic<Args...>::postcall(arg, ret);
    }
};
86
87
88

template <> struct process_dynamic<> : public process_dynamic<void> { };

89
90
NAMESPACE_END(detail)

Wenzel Jakob's avatar
Wenzel Jakob committed
91
/// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object
92
class cpp_function : public function {
93
94
protected:
    /// Special data structure which holds metadata about a bound function (signature, overloads, etc.)
Wenzel Jakob's avatar
Wenzel Jakob committed
95
    struct function_entry {
96
97
98
99
100
101
        /// Function name
        char *name = nullptr; /* why no C++ strings? They generate heavier code.. */

        // User-specified documentation string
        char *doc = nullptr;

102
103
        /// Human-readable version of the function signature
        char *signature = nullptr;
104

105
        /// List of registered keyword arguments
106
        std::vector<detail::argument_entry> args;
107

108
        /// Pointer to lambda function which converts arguments and performs the actual call
109
        PyObject * (*impl) (function_entry *, PyObject *, PyObject *) = nullptr;
110

111
        /// Storage for the wrapped function pointer and captured data, if any
112
        void *data = nullptr;
113

114
        /// Pointer to custom destructor for 'data' (if needed)
115
        void (*free_data) (void *ptr) = nullptr;
116

117
        /// Return value policy associated with this function
118
        return_value_policy policy = return_value_policy::automatic;
119

120
121
        /// True if name == '__init__'
        bool is_constructor = false;
122

123
124
        /// Python method object
        PyMethodDef *def = nullptr;
125

126
        /// Pointer to class (if this is method)
127
        PyObject *class_ = nullptr;
128

129
        /// Pointer to first registered function in overload chain
130
        PyObject *sibling = nullptr;
131

132
        /// Pointer to next overload
Wenzel Jakob's avatar
Wenzel Jakob committed
133
134
135
        function_entry *next = nullptr;
    };

136
137
    function_entry *m_entry;

Wenzel Jakob's avatar
Wenzel Jakob committed
138
139
140
    /// Picks a suitable return value converter from cast.h
    template <typename T> using return_value_caster =
        detail::type_caster<typename std::conditional<
141
            std::is_void<T>::value, detail::void_type, typename detail::intrinsic_type<T>::type>::type>;
Wenzel Jakob's avatar
Wenzel Jakob committed
142
143

    /// Picks a suitable argument value converter from cast.h
144
    template <typename... T> using arg_value_caster =
Wenzel Jakob's avatar
Wenzel Jakob committed
145
        detail::type_caster<typename std::tuple<T...>>;
146

147
    /// Deal with annotations that can be processed at function registration time
148
149
    template <typename... T> static void process_static(const std::tuple<T...> &args, function_entry *entry) {
        process_static(args, entry, typename detail::make_index_sequence<sizeof...(T)>::type());
150
151
    }

152
    /// contd.
153
    template <typename... T, size_t ... Index> static void process_static(const std::tuple<T...> &args,
154
            function_entry *entry, detail::index_sequence<Index...>) {
155
        int unused[] = { 0, (process_static(std::get<Index>(args), entry), 0)... };
156
157
158
        (void) unused;
    }

159
160
161
162
163
164
165
    /* The following overloads are used to process any annotations passed to
       cpp_function. They update the corresponding fields in m_entry */

    /// Process an annotation specifying the function's name
    static void process_static(const pybind11::name &n, function_entry *entry) { entry->name = (char *) n.value; }

    /// Process an annotation specifying function's docstring (provided as a C-style string)
166
    static void process_static(const char *doc, function_entry *entry) { entry->doc = (char *) doc; }
167
168

    /// Process an annotation specifying function's docstring
169
    static void process_static(const pybind11::doc &d, function_entry *entry) { entry->doc = (char *) d.value; }
170
171

    /// Process an annotation indicating the function's return value policy
172
    static void process_static(const pybind11::return_value_policy p, function_entry *entry) { entry->policy = p; }
173
174

    /// Process an annotation which indicates that this is an overloaded function associated with a given sibling
175
    static void process_static(const pybind11::sibling s, function_entry *entry) { entry->sibling = s.value; }
176
177

    /// Process an annotation which indicates that this function is a method
178
    static void process_static(const pybind11::is_method &m, function_entry *entry) { entry->class_ = m.class_; }
179
180

    /// Process a keyword argument annotation (*without* a default value)
181
    static void process_static(const pybind11::arg &a, function_entry *entry) {
182
        if (entry->class_ && entry->args.empty())
183
184
            entry->args.emplace_back("self", nullptr, nullptr);
        entry->args.emplace_back(a.name, nullptr, nullptr);
185
    }
186

187
    /// Process a keyword argument annotation (with a default value)
188
    template <typename T>
189
    static void process_static(const pybind11::arg_t<T> &a, function_entry *entry) {
190
        if (entry->class_ && entry->args.empty())
191
            entry->args.emplace_back("self", nullptr, nullptr);
192

193
        /* Convert keyword value into a Python object */
194
        PyObject *obj = detail::type_caster<typename detail::intrinsic_type<T>::type>::cast(
195
                a.value, return_value_policy::automatic, nullptr);
196

197
        if (obj == nullptr)
198
199
200
            pybind11_fail("arg(): could not convert default keyword "
                          "argument into a Python object (type not "
                          "registered yet?)");
201
202

        entry->args.emplace_back(a.name, a.descr, obj);
203
    }
204
205
206
207

    /// Process an annotation indicating a keep_alive call policy
    template <int Nurse, int Patient>
    static void process_static(const keep_alive<Nurse, Patient> &, function_entry *) { /* handled at call time */ }
Wenzel Jakob's avatar
Wenzel Jakob committed
208
public:
209
    cpp_function() { }
Wenzel Jakob's avatar
Wenzel Jakob committed
210
211

    /// Vanilla function pointers
212
213
214
    template <typename Return, typename... Args, typename... Extra>
    cpp_function(Return (*f)(Args...), Extra&&... extra) {
        using detail::descr;
215
        m_entry = new function_entry();
216
        m_entry->data = (void *) f;
Wenzel Jakob's avatar
Wenzel Jakob committed
217

218
        typedef arg_value_caster<Args...> cast_in;
219
220
        typedef return_value_caster<Return> cast_out;

221
        /* Dispatch code which converts function arguments and performs the actual function call */
222
        m_entry->impl = [](function_entry *entry, PyObject *pyArgs, PyObject *parent) -> PyObject * {
Wenzel Jakob's avatar
Wenzel Jakob committed
223
            cast_in args;
224
225

            /* Try to cast the function arguments into the C++ domain */
226
            if (!args.load(pyArgs, true))
227
228
229
230
231
232
233
234
235
236
                return PYBIND11_TRY_NEXT_OVERLOAD;

            detail::process_dynamic<Extra...>::precall(pyArgs); // call policy precall

            /* Do the call and convert the return value back into the Python domain */
            PyObject *result = cast_out::cast(
                args.template call<Return>((Return (*) (Args...)) entry->data),
                entry->policy, parent);

            detail::process_dynamic<Extra...>::postcall(pyArgs, result); // call policy postcall
237
            return result;
Wenzel Jakob's avatar
Wenzel Jakob committed
238
239
        };

240
        /* Process any user-provided function annotations */
241
        process_static(std::make_tuple(std::forward<Extra>(extra)...), m_entry);
242
243

        /* Generate a readable signature describing the function's arguments and return value types */
244
        PYBIND11_DESCR signature = cast_in::name() + detail::_(" -> ") + cast_out::name();
245
246

        /* Register the function with Python from generic (non-templated) code */
247
        initialize(signature.text(), signature.types(), sizeof...(Args));
Wenzel Jakob's avatar
Wenzel Jakob committed
248
249
250
    }

    /// Delegating helper constructor to deal with lambda functions
251
252
    template <typename Func, typename... Extra> cpp_function(Func &&f, Extra&&... extra) {
        initialize(std::forward<Func>(f),
Wenzel Jakob's avatar
Wenzel Jakob committed
253
                   (typename detail::remove_class<decltype(
254
255
                       &std::remove_reference<Func>::type::operator())>::type *) nullptr,
                   std::forward<Extra>(extra)...);
Wenzel Jakob's avatar
Wenzel Jakob committed
256
257
    }

258
    /// Delegating helper constructor to deal with class methods (non-const)
259
260
261
262
    template <typename Return, typename Class, typename... Arg, typename... Extra> cpp_function(
            Return (Class::*f)(Arg...), Extra&&... extra) {
        initialize([f](Class *c, Arg... args) -> Return { return (c->*f)(args...); },
                   (Return (*) (Class *, Arg...)) nullptr, std::forward<Extra>(extra)...);
Wenzel Jakob's avatar
Wenzel Jakob committed
263
    }
Wenzel Jakob's avatar
Wenzel Jakob committed
264

265
    /// Delegating helper constructor to deal with class methods (const)
266
267
268
269
    template <typename Return, typename Class, typename... Arg, typename... Extra> cpp_function(
            Return (Class::*f)(Arg...) const, Extra&&... extra) {
        initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(args...); },
                   (Return (*)(const Class *, Arg ...)) nullptr, std::forward<Extra>(extra)...);
Wenzel Jakob's avatar
Wenzel Jakob committed
270
271
    }

272
273
274
    /// Return the function name
    const char *name() const { return m_entry->name; }

275
276
protected:
    /// Special internal constructor for functors, lambda functions, etc.
277
278
279
280
    template <typename Func, typename Return, typename... Args, typename... Extra>
    void initialize(Func &&f, Return (*)(Args...), Extra&&... extra) {
        using detail::descr;
        struct capture { typename std::remove_reference<Func>::type f; };
Wenzel Jakob's avatar
Wenzel Jakob committed
281

282
        /* Store the function including any extra state it might have (e.g. a lambda capture object) */
283
        m_entry = new function_entry();
284
        m_entry->data = new capture { std::forward<Func>(f) };
285

286
        /* Create a cleanup handler, but only if we have to (less generated code) */
287
        if (!std::is_trivially_destructible<Func>::value)
288
            m_entry->free_data = [](void *ptr) { delete (capture *) ptr; };
289
        else
290
            m_entry->free_data = operator delete;
291

292
        typedef arg_value_caster<Args...> cast_in;
293
        typedef return_value_caster<Return> cast_out;
Wenzel Jakob's avatar
Wenzel Jakob committed
294

295
        /* Dispatch code which converts function arguments and performs the actual function call */
296
        m_entry->impl = [](function_entry *entry, PyObject *pyArgs, PyObject *parent) -> PyObject *{
Wenzel Jakob's avatar
Wenzel Jakob committed
297
            cast_in args;
298
299

            /* Try to cast the function arguments into the C++ domain */
300
            if (!args.load(pyArgs, true))
301
302
303
304
305
306
307
308
309
310
                return PYBIND11_TRY_NEXT_OVERLOAD;

            detail::process_dynamic<Extra...>::precall(pyArgs); // call policy precall

            /* Do the call and convert the return value back into the Python domain */
            PyObject *result = cast_out::cast(
                args.template call<Return>(((capture *) entry->data)->f),
                entry->policy, parent);

            detail::process_dynamic<Extra...>::postcall(pyArgs, result); // call policy postcall
311
            return result;
Wenzel Jakob's avatar
Wenzel Jakob committed
312
313
        };

314
        /* Process any user-provided function annotations */
315
        process_static(std::make_tuple(std::forward<Extra>(extra)...), m_entry);
316
317

        /* Generate a readable signature describing the function's arguments and return value types */
318
        PYBIND11_DESCR signature = cast_in::name() + detail::_(" -> ") + cast_out::name();
319
320

        /* Register the function with Python from generic (non-templated) code */
321
        initialize(signature.text(), signature.types(), sizeof...(Args));
Wenzel Jakob's avatar
Wenzel Jakob committed
322
323
    }

324
    /// Main dispatch logic for calls to functions bound using pybind11
Wenzel Jakob's avatar
Wenzel Jakob committed
325
    static PyObject *dispatcher(PyObject *self, PyObject *args, PyObject *kwargs) {
326
        /* Iterator over the list of potentially admissible overloads */
327
328
        function_entry *overloads = (function_entry *) PyCapsule_GetPointer(self, nullptr),
                       *it = overloads;
329
330

        /* Need to know how many arguments + keyword arguments there are to pick the right overload */
331
332
        int nargs = (int) PyTuple_Size(args),
            nkwargs = kwargs ? (int) PyDict_Size(kwargs) : 0;
333

334
        PyObject *parent = nargs > 0 ? PyTuple_GetItem(args, 0) : nullptr,
335
                 *result = PYBIND11_TRY_NEXT_OVERLOAD;
Wenzel Jakob's avatar
Wenzel Jakob committed
336
        try {
337
            for (; it != nullptr; it = it->next) {
338
                object args_(args, true);
339
                int kwargs_consumed = 0;
340

341
342
343
344
345
346
347
348
                /* For each overload:
                   1. If the required list of arguments is longer than the
                      actually provided amount, create a copy of the argument
                      list and fill in any available keyword/default arguments.
                   2. Ensure that all keyword arguments were "consumed"
                   3. Call the function call dispatcher (function_entry::impl)
                 */

349
                if (nargs < (int) it->args.size()) {
350
                    args_ = object(PyTuple_New(it->args.size()), false);
351
                    for (int i = 0; i < nargs; ++i) {
352
353
                        PyObject *item = PyTuple_GET_ITEM(args, i);
                        Py_INCREF(item);
354
                        PyTuple_SET_ITEM(args_.ptr(), i, item);
355
                    }
356
                    int arg_ctr = 0;
357
                    for (auto const &it2 : it->args) {
358
                        int index = arg_ctr++;
359
                        if (PyTuple_GET_ITEM(args_.ptr(), index))
360
361
362
                            continue;
                        PyObject *value = nullptr;
                        if (kwargs)
363
                            value = PyDict_GetItemString(kwargs, it2.name);
364
365
                        if (value)
                            kwargs_consumed++;
366
367
                        else if (it2.value)
                            value = it2.value;
368
369
                        if (value) {
                            Py_INCREF(value);
370
                            PyTuple_SET_ITEM(args_.ptr(), index, value);
371
372
373
374
375
                        } else {
                            kwargs_consumed = -1; /* definite failure */
                            break;
                        }
                    }
376
377
                }

378
                if (kwargs_consumed == nkwargs)
379
                    result = it->impl(it, args_.ptr(), parent);
380

381
                if (result != PYBIND11_TRY_NEXT_OVERLOAD)
Wenzel Jakob's avatar
Wenzel Jakob committed
382
383
                    break;
            }
Jonas Adler's avatar
Jonas Adler committed
384
385
386
387
388
389
390
391
392
393
        } catch (const error_already_set &)      {                                                 return nullptr;
        } catch (const index_error &e)           { PyErr_SetString(PyExc_IndexError,    e.what()); return nullptr;
        } catch (const stop_iteration &e)        { PyErr_SetString(PyExc_StopIteration, e.what()); return nullptr;
        } catch (const std::bad_alloc &e)        { PyErr_SetString(PyExc_MemoryError,   e.what()); return nullptr;
        } catch (const std::domain_error &e)     { PyErr_SetString(PyExc_ValueError,    e.what()); return nullptr;
        } catch (const std::invalid_argument &e) { PyErr_SetString(PyExc_ValueError,    e.what()); return nullptr;
        } catch (const std::length_error &e)     { PyErr_SetString(PyExc_ValueError,    e.what()); return nullptr;
        } catch (const std::out_of_range &e)     { PyErr_SetString(PyExc_IndexError,    e.what()); return nullptr;
        } catch (const std::range_error &e)      { PyErr_SetString(PyExc_ValueError,    e.what()); return nullptr;
        } catch (const std::exception &e)        { PyErr_SetString(PyExc_RuntimeError,  e.what()); return nullptr;
Wenzel Jakob's avatar
Wenzel Jakob committed
394
395
396
397
        } catch (...) {
            PyErr_SetString(PyExc_RuntimeError, "Caught an unknown exception!");
            return nullptr;
        }
398
399

        if (result == PYBIND11_TRY_NEXT_OVERLOAD) {
400
401
402
            std::string msg = "Incompatible function arguments. The "
                              "following argument types are supported:\n";
            int ctr = 0;
Wenzel Jakob's avatar
Wenzel Jakob committed
403
            for (function_entry *it2 = overloads; it2 != nullptr; it2 = it2->next) {
404
                msg += "    "+ std::to_string(++ctr) + ". ";
405
                msg += it2->signature;
406
407
408
409
410
411
412
                msg += "\n";
            }
            PyErr_SetString(PyExc_TypeError, msg.c_str());
            return nullptr;
        } else if (result == nullptr) {
            std::string msg = "Unable to convert function return value to a "
                              "Python type! The signature was\n\t";
413
            msg += it->signature;
414
415
416
            PyErr_SetString(PyExc_TypeError, msg.c_str());
            return nullptr;
        } else {
Wenzel Jakob's avatar
Wenzel Jakob committed
417
            if (overloads->is_constructor) {
418
419
                /* When a construtor ran successfully, the corresponding
                   holder type (e.g. std::unique_ptr) must still be initialized. */
Wenzel Jakob's avatar
Wenzel Jakob committed
420
                PyObject *inst = PyTuple_GetItem(args, 0);
421
422
                auto tinfo = detail::get_type_info(Py_TYPE(inst));
                tinfo->init_holder(inst, nullptr);
Wenzel Jakob's avatar
Wenzel Jakob committed
423
424
425
426
427
            }
            return result;
        }
    }

428
    /// When a cpp_function is GCed, release any memory allocated by pybind11
429
430
431
    static void destruct(function_entry *entry) {
        while (entry) {
            function_entry *next = entry->next;
432
433
434
435
436
437
438
439
440
441
            if (entry->free_data)
                entry->free_data(entry->data);
            std::free((char *) entry->name);
            std::free((char *) entry->doc);
            std::free((char *) entry->signature);
            for (auto &arg: entry->args) {
                std::free((char *) arg.name);
                std::free((char *) arg.descr);
                Py_XDECREF(arg.value);
            }
442
            if (entry->def) {
443
                std::free((char *) entry->def->ml_doc);
444
445
                delete entry->def;
            }
446
447
448
449
450
            delete entry;
            entry = next;
        }
    }

451
    /// Register a function call with Python (generic non-templated code goes here)
452
453
454
455
456
457
458
459
460
461
    void initialize(const char *text, const std::type_info * const * types, int args) {
        /* Create copies of all referenced C-style strings */
        m_entry->name = strdup(m_entry->name ? m_entry->name : "");
        if (m_entry->doc) m_entry->doc = strdup(m_entry->doc);
        for (auto &a: m_entry->args) {
            if (a.name)
                a.name = strdup(a.name);
            if (a.descr)
                a.descr = strdup(a.descr);
            else if (a.value)
462
                a.descr = strdup(((std::string) ((object) handle(a.value).attr("__repr__")).call().str()).c_str());
463
        }
464
        auto const &registered_types = detail::get_internals().registered_types_cpp;
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490

        /* Generate a proper function signature */
        std::string signature;
        size_t type_depth = 0, char_index = 0, type_index = 0, arg_index = 0;
        while (true) {
            char c = text[char_index++];
            if (c == '\0')
                break;

            if (c == '{') {
                if (type_depth == 1 && arg_index < m_entry->args.size()) {
                    signature += m_entry->args[arg_index].name;
                    signature += " : ";
                }
                ++type_depth;
            } else if (c == '}') {
                --type_depth;
                if (type_depth == 1 && arg_index < m_entry->args.size()) {
                    if (m_entry->args[arg_index].descr) {
                        signature += " = ";
                        signature += m_entry->args[arg_index].descr;
                    }
                    arg_index++;
                }
            } else if (c == '%') {
                const std::type_info *t = types[type_index++];
491
                if (!t)
492
                    pybind11_fail("Internal error while parsing type signature (1)");
493
494
                auto it = registered_types.find(t);
                if (it != registered_types.end()) {
495
                    signature += ((const detail::type_info *) it->second)->type->tp_name;
496
497
498
499
500
501
502
503
504
                } else {
                    std::string tname(t->name());
                    detail::clean_type_id(tname);
                    signature += tname;
                }
            } else {
                signature += c;
            }
        }
505
        if (type_depth != 0 || types[type_index] != nullptr)
506
            pybind11_fail("Internal error while parsing type signature (2)");
507
508
509
510
511

        #if !defined(PYBIND11_CPP14)
            delete[] types;
            delete[] text;
        #endif
512

513
#if PY_MAJOR_VERSION < 3
514
        if (strcmp(m_entry->name, "__next__") == 0) {
515
            std::free(m_entry->name);
516
517
            m_entry->name = strdup("next");
        }
518
519
#endif

520
        if (!m_entry->args.empty() && (int) m_entry->args.size() != args)
521
            pybind11_fail(
522
                "cpp_function(): function \"" + std::string(m_entry->name) + "\" takes " +
523
                std::to_string(args) + " arguments, but " + std::to_string(m_entry->args.size()) +
524
                " pybind11::arg entries were specified!");
525

526
        m_entry->is_constructor = !strcmp(m_entry->name, "__init__");
527
528
        m_entry->signature = strdup(signature.c_str());
        m_entry->args.shrink_to_fit();
529
530
531
532
533

#if PY_MAJOR_VERSION < 3
        if (m_entry->sibling && PyMethod_Check(m_entry->sibling))
            m_entry->sibling = PyMethod_GET_FUNCTION(m_entry->sibling);
#endif
534

535
536
537
538
        function_entry *s_entry = nullptr, *entry = m_entry;
        if (m_entry->sibling && PyCFunction_Check(m_entry->sibling)) {
            capsule entry_capsule(PyCFunction_GetSelf(m_entry->sibling), true);
            s_entry = (function_entry *) entry_capsule;
539
540
            /* Never append a method to an overload chain of a parent class;
               instead, hide the parent's overloads in this case */
541
            if (s_entry->class_ != m_entry->class_)
542
                s_entry = nullptr;
543
544
        }

545
546
        if (!s_entry) {
            /* No existing overload was found, create a new function object */
547
548
549
550
551
            m_entry->def = new PyMethodDef();
            memset(m_entry->def, 0, sizeof(PyMethodDef));
            m_entry->def->ml_name = m_entry->name;
            m_entry->def->ml_meth = reinterpret_cast<PyCFunction>(*dispatcher);
            m_entry->def->ml_flags = METH_VARARGS | METH_KEYWORDS;
552
553
554
            capsule entry_capsule(m_entry, [](PyObject *o) {
                destruct((function_entry *) PyCapsule_GetPointer(o, nullptr));
            });
555
            m_ptr = PyCFunction_New(m_entry->def, entry_capsule.ptr());
Wenzel Jakob's avatar
Wenzel Jakob committed
556
            if (!m_ptr)
557
                pybind11_fail("cpp_function::cpp_function(): Could not allocate function object");
Wenzel Jakob's avatar
Wenzel Jakob committed
558
        } else {
559
            /* Append at the end of the overload chain */
560
            m_ptr = m_entry->sibling;
Wenzel Jakob's avatar
Wenzel Jakob committed
561
            inc_ref();
562
563
564
565
            entry = s_entry;
            while (s_entry->next)
                s_entry = s_entry->next;
            s_entry->next = m_entry;
Wenzel Jakob's avatar
Wenzel Jakob committed
566
        }
567

Wenzel Jakob's avatar
Wenzel Jakob committed
568
        std::string signatures;
569
570
        int index = 0;
        function_entry *it = entry;
571
572
573
        /* Create a nice pydoc entry including all signatures and
           docstrings of the functions in the overload chain */
        while (it) {
574
            if (s_entry)
575
                signatures += std::to_string(++index) + ". ";
576
577
578
579
580
581
582
583
            signatures += "Signature : ";
            signatures += it->signature;
            signatures += "\n";
            if (it->doc && strlen(it->doc) > 0) {
                signatures += "\n";
                signatures += it->doc;
                signatures += "\n";
            }
584
            if (it->next)
Wenzel Jakob's avatar
Wenzel Jakob committed
585
                signatures += "\n";
586
            it = it->next;
Wenzel Jakob's avatar
Wenzel Jakob committed
587
588
589
        }
        PyCFunctionObject *func = (PyCFunctionObject *) m_ptr;
        if (func->m_ml->ml_doc)
590
            std::free((char *) func->m_ml->ml_doc);
Wenzel Jakob's avatar
Wenzel Jakob committed
591
        func->m_ml->ml_doc = strdup(signatures.c_str());
592
        if (entry->class_) {
593
#if PY_MAJOR_VERSION >= 3
Wenzel Jakob's avatar
Wenzel Jakob committed
594
            m_ptr = PyInstanceMethod_New(m_ptr);
595
596
597
#else
            m_ptr = PyMethod_New(m_ptr, nullptr, entry->class_);
#endif
Wenzel Jakob's avatar
Wenzel Jakob committed
598
            if (!m_ptr)
599
                pybind11_fail("cpp_function::cpp_function(): Could not allocate instance method object");
Wenzel Jakob's avatar
Wenzel Jakob committed
600
601
602
603
604
            Py_DECREF(func);
        }
    }
};

605
/// Wrapper for Python extension modules
Wenzel Jakob's avatar
Wenzel Jakob committed
606
607
class module : public object {
public:
608
    PYBIND11_OBJECT_DEFAULT(module, object, PyModule_Check)
Wenzel Jakob's avatar
Wenzel Jakob committed
609
610

    module(const char *name, const char *doc = nullptr) {
611
#if PY_MAJOR_VERSION >= 3
Wenzel Jakob's avatar
Wenzel Jakob committed
612
613
614
615
616
617
618
        PyModuleDef *def = new PyModuleDef();
        memset(def, 0, sizeof(PyModuleDef));
        def->m_name = name;
        def->m_doc = doc;
        def->m_size = -1;
        Py_INCREF(def);
        m_ptr = PyModule_Create(def);
619
620
621
#else
        m_ptr = Py_InitModule3(name, nullptr, doc);
#endif
Wenzel Jakob's avatar
Wenzel Jakob committed
622
        if (m_ptr == nullptr)
623
            pybind11_fail("Internal error in module::module()");
Wenzel Jakob's avatar
Wenzel Jakob committed
624
625
626
        inc_ref();
    }

627
628
629
630
    template <typename Func, typename... Extra>
    module &def(const char *name_, Func &&f, Extra&& ... extra) {
        cpp_function func(std::forward<Func>(f), name(name_),
                          sibling((handle) attr(name_)), std::forward<Extra>(extra)...);
Wenzel Jakob's avatar
Wenzel Jakob committed
631
        func.inc_ref(); /* The following line steals a reference to 'func' */
632
        PyModule_AddObject(ptr(), name_, func.ptr());
Wenzel Jakob's avatar
Wenzel Jakob committed
633
634
635
        return *this;
    }

636
    module def_submodule(const char *name, const char *doc = nullptr) {
Wenzel Jakob's avatar
Wenzel Jakob committed
637
638
639
        std::string full_name = std::string(PyModule_GetName(m_ptr))
            + std::string(".") + std::string(name);
        module result(PyImport_AddModule(full_name.c_str()), true);
640
        if (doc)
641
            result.attr("__doc__") = pybind11::str(doc);
Wenzel Jakob's avatar
Wenzel Jakob committed
642
643
644
        attr(name) = result;
        return result;
    }
Wenzel Jakob's avatar
Wenzel Jakob committed
645
646

    static module import(const char *name) {
647
648
        PyObject *obj = PyImport_ImportModule(name);
        if (!obj)
649
            pybind11_fail("Module \"" + std::string(name) + "\" not found!");
650
        return module(obj, false);
Wenzel Jakob's avatar
Wenzel Jakob committed
651
    }
Wenzel Jakob's avatar
Wenzel Jakob committed
652
653
654
655
};

NAMESPACE_BEGIN(detail)
/// Basic support for creating new Python heap types
656
class generic_type : public object {
Wenzel Jakob's avatar
Wenzel Jakob committed
657
public:
658
    PYBIND11_OBJECT_DEFAULT(generic_type, object, PyType_Check)
Wenzel Jakob's avatar
Wenzel Jakob committed
659

660
    generic_type(const object &scope, const char *name_, const std::type_info *type_cpp,
Wenzel Jakob's avatar
Wenzel Jakob committed
661
                size_t type_size, size_t instance_size,
662
663
664
665
666
667
668
669
                void (*init_holder)(PyObject *, const void *),
                const destructor &dealloc, object parent, const char *doc) {

        object type_holder(PyType_Type.tp_alloc(&PyType_Type, 0), false);
        object name(PYBIND11_FROM_STRING(name_), false);
        auto type = (PyHeapTypeObject*) type_holder.ptr();

        if (!type_holder || !name)
670
            pybind11_fail("generic_type: unable to create type object!");
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701

        /* Register supplemental type information in C++ dict */
        auto &internals = get_internals();
        detail::type_info *tinfo = new detail::type_info();
        tinfo->type = (PyTypeObject *) type;
        tinfo->type_size = type_size;
        tinfo->init_holder = init_holder;
        internals.registered_types_cpp[type_cpp] = tinfo;
        internals.registered_types_py[type] = tinfo;

        auto scope_module = (object) scope.attr("__module__");
        if (!scope_module)
            scope_module = (object) scope.attr("__name__");

        std::string full_name = (scope_module ? ((std::string) scope_module.str() + "." + name_)
                                              : std::string(name_));
        /* Basic type attributes */
        type->ht_type.tp_name = strdup(full_name.c_str());
        type->ht_type.tp_basicsize = instance_size;
        type->ht_type.tp_base = (PyTypeObject *) parent.release();

#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
        /* Qualified names for Python >= 3.3 */
        auto scope_qualname = (object) scope.attr("__qualname__");
        if (scope_qualname) {
            type->ht_qualname = PyUnicode_FromFormat(
                "%U.%U", scope_qualname.ptr(), name.ptr());
        } else {
            type->ht_qualname = name.ptr();
            name.inc_ref();
        }
702
#endif
703
        type->ht_name = name.release();
Wenzel Jakob's avatar
Wenzel Jakob committed
704

705
706
707
708
        /* Supported protocols */
        type->ht_type.tp_as_number = &type->as_number;
        type->ht_type.tp_as_sequence = &type->as_sequence;
        type->ht_type.tp_as_mapping = &type->as_mapping;
Wenzel Jakob's avatar
Wenzel Jakob committed
709

710
        /* Supported elementary operations */
Wenzel Jakob's avatar
Wenzel Jakob committed
711
712
713
        type->ht_type.tp_init = (initproc) init;
        type->ht_type.tp_new = (newfunc) new_instance;
        type->ht_type.tp_dealloc = dealloc;
714
715
716
717
718
719

        /* Support weak references (needed for the keep_alive feature) */
        type->ht_type.tp_weaklistoffset = offsetof(instance<void>, weakrefs);

        /* Flags */
        type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
720
721
722
#if PY_MAJOR_VERSION < 3
        type->ht_type.tp_flags |= Py_TPFLAGS_CHECKTYPES;
#endif
723
        type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC;
724

Wenzel Jakob's avatar
Wenzel Jakob committed
725
        if (doc) {
726
727
            /* Allocate memory for docstring (using PyObject_MALLOC, since
               Python will free this later on) */
728
            size_t size = strlen(doc) + 1;
729
            type->ht_type.tp_doc = (char *) PyObject_MALLOC(size);
Wenzel Jakob's avatar
Wenzel Jakob committed
730
731
            memcpy((void *) type->ht_type.tp_doc, doc, size);
        }
Wenzel Jakob's avatar
Wenzel Jakob committed
732
733

        if (PyType_Ready(&type->ht_type) < 0)
734
            pybind11_fail("generic_type: PyType_Ready failed!");
735
736

        m_ptr = type_holder.ptr();
Wenzel Jakob's avatar
Wenzel Jakob committed
737

738
739
        if (scope_module) // Needed by pydoc
            type_holder.attr("__module__") = scope_module;
Wenzel Jakob's avatar
Wenzel Jakob committed
740

741
742
        /* Register type with the parent scope */
        scope.attr(name_) = *this;
Wenzel Jakob's avatar
Wenzel Jakob committed
743

744
        type_holder.release();
Wenzel Jakob's avatar
Wenzel Jakob committed
745
746
747
    }

protected:
748
    /// Allocate a metaclass on demand (for static properties)
Wenzel Jakob's avatar
Wenzel Jakob committed
749
750
    handle metaclass() {
        auto &ht_type = ((PyHeapTypeObject *) m_ptr)->ht_type;
751
        auto &ob_type = PYBIND11_OB_TYPE(ht_type);
752

Wenzel Jakob's avatar
Wenzel Jakob committed
753
        if (ob_type == &PyType_Type) {
754
755
756
757
            std::string name_ = std::string(ht_type.tp_name) + "__Meta";
            object type_holder(PyType_Type.tp_alloc(&PyType_Type, 0), false);
            object name(PYBIND11_FROM_STRING(name_.c_str()), false);
            if (!type_holder || !name)
758
                pybind11_fail("generic_type::metaclass(): unable to create type object!");
759
760
761
762

            auto type = (PyHeapTypeObject*) type_holder.ptr();
            type->ht_name = name.release();

Wenzel Jakob's avatar
Wenzel Jakob committed
763
764
            type->ht_type.tp_name = strdup(name_.c_str());
            type->ht_type.tp_base = &PyType_Type;
765
766
767
            type->ht_type.tp_flags |= (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE) &
                                      ~Py_TPFLAGS_HAVE_GC;

Wenzel Jakob's avatar
Wenzel Jakob committed
768
            if (PyType_Ready(&type->ht_type) < 0)
769
                pybind11_fail("generic_type::metaclass(): PyType_Ready failed!");
770
771

            ob_type = (PyTypeObject *) type_holder.release();
Wenzel Jakob's avatar
Wenzel Jakob committed
772
773
774
775
776
777
778
779
780
781
782
        }
        return handle((PyObject *) ob_type);
    }

    static int init(void *self, PyObject *, PyObject *) {
        std::string msg = std::string(Py_TYPE(self)->tp_name) + ": No constructor defined!";
        PyErr_SetString(PyExc_TypeError, msg.c_str());
        return -1;
    }

    static PyObject *new_instance(PyTypeObject *type, PyObject *, PyObject *) {
783
784
785
        instance<void> *self = (instance<void> *) PyType_GenericAlloc((PyTypeObject *) type, 0);
        auto tinfo = detail::get_type_info(type);
        self->value = ::operator new(tinfo->type_size);
Wenzel Jakob's avatar
Wenzel Jakob committed
786
787
788
789
790
791
792
793
794
795
796
797
798
799
        self->owned = true;
        self->parent = nullptr;
        self->constructed = false;
        detail::get_internals().registered_instances[self->value] = (PyObject *) self;
        return (PyObject *) self;
    }

    static void dealloc(instance<void> *self) {
        if (self->value) {
            bool dont_cache = self->parent && ((instance<void> *) self->parent)->value == self->value;
            if (!dont_cache) { // avoid an issue with internal references matching their parent's address
                auto &registered_instances = detail::get_internals().registered_instances;
                auto it = registered_instances.find(self->value);
                if (it == registered_instances.end())
800
                    pybind11_fail("generic_type::dealloc(): Tried to deallocate unregistered instance!");
Wenzel Jakob's avatar
Wenzel Jakob committed
801
802
803
                registered_instances.erase(it);
            }
            Py_XDECREF(self->parent);
804
805
            if (self->weakrefs)
                PyObject_ClearWeakRefs((PyObject *) self);
Wenzel Jakob's avatar
Wenzel Jakob committed
806
807
808
809
        }
        Py_TYPE(self)->tp_free((PyObject*) self);
    }

Wenzel Jakob's avatar
Wenzel Jakob committed
810
811
812
    void install_buffer_funcs(
            buffer_info *(*get_buffer)(PyObject *, void *),
            void *get_buffer_data) {
Wenzel Jakob's avatar
Wenzel Jakob committed
813
814
        PyHeapTypeObject *type = (PyHeapTypeObject*) m_ptr;
        type->ht_type.tp_as_buffer = &type->as_buffer;
815
816
817
#if PY_MAJOR_VERSION < 3
        type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER;
#endif
Wenzel Jakob's avatar
Wenzel Jakob committed
818
819
        type->as_buffer.bf_getbuffer = getbuffer;
        type->as_buffer.bf_releasebuffer = releasebuffer;
820
821
822
        auto tinfo = detail::get_type_info(&type->ht_type);
        tinfo->get_buffer = get_buffer;
        tinfo->get_buffer_data = get_buffer_data;
Wenzel Jakob's avatar
Wenzel Jakob committed
823
824
825
    }

    static int getbuffer(PyObject *obj, Py_buffer *view, int flags) {
826
827
828
        auto tinfo = detail::get_type_info(Py_TYPE(obj));
        if (view == nullptr || obj == nullptr || !tinfo || !tinfo->get_buffer) {
            PyErr_SetString(PyExc_BufferError, "generic_type::getbuffer(): Internal error");
Wenzel Jakob's avatar
Wenzel Jakob committed
829
830
831
            return -1;
        }
        memset(view, 0, sizeof(Py_buffer));
832
        buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data);
Wenzel Jakob's avatar
Wenzel Jakob committed
833
834
835
836
837
838
839
840
841
842
843
844
        view->obj = obj;
        view->ndim = 1;
        view->internal = info;
        view->buf = info->ptr;
        view->itemsize = info->itemsize;
        view->len = view->itemsize;
        for (auto s : info->shape)
            view->len *= s;
        if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT)
            view->format = const_cast<char *>(info->format.c_str());
        if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
            view->ndim = info->ndim;
845
846
            view->strides = (ssize_t *) &info->strides[0];
            view->shape = (ssize_t *) &info->shape[0];
Wenzel Jakob's avatar
Wenzel Jakob committed
847
848
849
850
851
852
853
        }
        Py_INCREF(view->obj);
        return 0;
    }

    static void releasebuffer(PyObject *, Py_buffer *view) { delete (buffer_info *) view->internal; }
};
854
855
856
857
858
859
860

/* Forward declarations */
enum op_id : int;
enum op_type : int;
struct undefined_t;
template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_;
template <typename... Args> struct init;
Wenzel Jakob's avatar
Wenzel Jakob committed
861
862
NAMESPACE_END(detail)

863
template <typename type, typename holder_type = std::unique_ptr<type>> class class_ : public detail::generic_type {
Wenzel Jakob's avatar
Wenzel Jakob committed
864
865
866
public:
    typedef detail::instance<type, holder_type> instance_type;

867
    PYBIND11_OBJECT(class_, detail::generic_type, PyType_Check)
Wenzel Jakob's avatar
Wenzel Jakob committed
868
869

    class_(object &scope, const char *name, const char *doc = nullptr)
870
        : detail::generic_type(scope, name, &typeid(type), sizeof(type),
Wenzel Jakob's avatar
Wenzel Jakob committed
871
                              sizeof(instance_type), init_holder, dealloc,
872
                              object(), doc) { }
Wenzel Jakob's avatar
Wenzel Jakob committed
873
874
875

    class_(object &scope, const char *name, object &parent,
           const char *doc = nullptr)
876
        : detail::generic_type(scope, name, &typeid(type), sizeof(type),
Wenzel Jakob's avatar
Wenzel Jakob committed
877
                              sizeof(instance_type), init_holder, dealloc,
878
                              parent, doc) { }
Wenzel Jakob's avatar
Wenzel Jakob committed
879

880
881
    template <typename Func, typename... Extra>
    class_ &def(const char *name_, Func&& f, Extra&&... extra) {
882
883
884
885
        cpp_function cf(std::forward<Func>(f), name(name_),
                        sibling(attr(name_)), is_method(this),
                        std::forward<Extra>(extra)...);
        attr(cf.name()) = cf;
Wenzel Jakob's avatar
Wenzel Jakob committed
886
887
888
        return *this;
    }

889
890
    template <typename Func, typename... Extra> class_ &
    def_static(const char *name_, Func f, Extra&&... extra) {
891
892
893
894
        cpp_function cf(std::forward<Func>(f), name(name_),
                        sibling(attr(name_)),
                        std::forward<Extra>(extra)...);
        attr(cf.name()) = cf;
Wenzel Jakob's avatar
Wenzel Jakob committed
895
896
897
        return *this;
    }

898
899
900
    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
    class_ &def(const detail::op_<id, ot, L, R> &op, Extra&&... extra) {
        op.template execute<type>(*this, std::forward<Extra>(extra)...);