Commit 66c9a402 authored by Wenzel Jakob's avatar Wenzel Jakob
Browse files

Much more efficient generation of function signatures, updated docs

This modification taps into some newer C++14 features (if present) to
generate function signatures considerably more efficiently at compile
time rather than at run time.

With this change, pybind11 binaries are now *2.1 times* smaller compared
to the Boost.Python baseline in the benchmark. Compilation times get a
nice improvement as well.

Visual Studio 2015 unfortunately doesn't implement 'constexpr' well
enough yet to support this change and uses a runtime fallback.
parent 2ac5044a
......@@ -37,8 +37,16 @@ endif()
find_package(PythonInterp ${PYTHONLIBS_VERSION_STRING} EXACT REQUIRED)
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")
# Enable C++11 mode on C++ / Clang
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
CHECK_CXX_COMPILER_FLAG("-std=c++14" HAS_CPP14_FLAG)
CHECK_CXX_COMPILER_FLAG("-std=c++11" HAS_CPP11_FLAG)
if (HAS_CPP14_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
elseif (HAS_CPP11_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
else()
message(FATAL_ERROR "Unsupported compiler -- pybind11 requires C++11 support!")
endif()
# Enable link time optimization and set the default symbol
# visibility to hidden (very important to obtain small binaries)
......@@ -74,14 +82,15 @@ include_directories(include)
set(PYBIND11_HEADERS
include/pybind11/cast.h
include/pybind11/common.h
include/pybind11/complex.h
include/pybind11/descr.h
include/pybind11/functional.h
include/pybind11/numpy.h
include/pybind11/operators.h
include/pybind11/pybind11.h
include/pybind11/pytypes.h
include/pybind11/typeid.h
include/pybind11/numpy.h
include/pybind11/complex.h
include/pybind11/stl.h
include/pybind11/functional.h
include/pybind11/typeid.h
)
# Create the binding library
......
......@@ -23,12 +23,12 @@ become an excessively large and unnecessary dependency.
Think of this library as a tiny self-contained version of Boost.Python with
everything stripped away that isn't relevant for binding generation. The core
header files only require ~2K lines of code and depend on Python (2.7 or 3.x)
header files only require ~3K lines of code and depend on Python (2.7 or 3.x)
and the C++ standard library. This compact implementation was possible thanks
to some of the new C++11 language features (tuples, lambda functions and
variadic templates). Since its creation, this library has grown beyond
Boost.Python in many ways, leading to dramatically simpler binding code in many
common situations.
to some of the new C++11 language features (specifically: tuples, lambda
functions and variadic templates). Since its creation, this library has grown
beyond Boost.Python in many ways, leading to dramatically simpler binding code
in many common situations.
Tutorial and reference documentation is provided at
[http://pybind11.readthedocs.org/en/latest](http://pybind11.readthedocs.org/en/latest).
......@@ -71,6 +71,13 @@ In addition to the core functionality, pybind11 provides some extra goodies:
- Everything is contained in just a few header files; there is no need to link
against any additional libraries.
- Binaries are generally smaller by a factor of 2 or more compared to
equivalent bindings generated by Boost.Python.
- When supported by the compiler, two new C++14 features (relaxed constexpr,
return value deduction) such as are used to do additional work at compile
time, leading to smaller binaries.
### License
pybind11 is provided under a BSD-style license that can be found in the
......
......@@ -81,7 +81,7 @@ for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]:
f.write(codegen(nclasses))
n1 = dt.datetime.now()
os.system("g++ -Os -shared -rdynamic -undefined dynamic_lookup "
"-fvisibility=hidden -std=c++11 test.cpp -I include "
"-fvisibility=hidden -std=c++14 test.cpp -I include "
"-I /System/Library/Frameworks/Python.framework/Headers -o test.so")
n2 = dt.datetime.now()
elapsed = (n2 - n1).total_seconds()
......
......@@ -4,9 +4,12 @@ Benchmark
The following is the result of a synthetic benchmark comparing both compilation
time and module size of pybind11 against Boost.Python.
A python script (see the ``docs/benchmark.py`` file) was used to generate a
set of dummy classes whose count increases for each successive benchmark
(between 1 and 512 classes in powers of two). Each class has four methods with
Setup
-----
A python script (see the ``docs/benchmark.py`` file) was used to generate a set
of files with dummy classes whose count increases for each successive benchmark
(between 1 and 2048 classes in powers of two). Each class has four methods with
a randomly generated signature with a return value and four arguments. (There
was no particular reason for this setup other than the desire to generate many
unique function signatures whose count could be controlled in a simple way.)
......@@ -43,28 +46,38 @@ compilation was done with
.. code-block:: bash
Apple LLVM version 7.0.0 (clang-700.0.72)
Apple LLVM version 7.0.2 (clang-700.1.81)
and the following compilation flags
.. code-block:: bash
g++ -Os -shared -rdynamic -undefined dynamic_lookup -fvisibility=hidden -std=c++11
g++ -Os -shared -rdynamic -undefined dynamic_lookup -fvisibility=hidden -std=c++14
Compilation time
----------------
The following log-log plot shows how the compilation time grows for an
increasing number of class and function declarations. pybind11 includes fewer
headers, which initially leads to shorter compilation times, but the
performance is ultimately very similar (pybind11 is 1 second faster for the
largest file, which is less than 1% of the total compilation time).
increasing number of class and function declarations. pybind11 includes many
fewer headers, which initially leads to shorter compilation times, but the
performance is ultimately fairly similar (pybind11 is 19.8 seconds faster for
the largest largest file with 2048 classes and a total of 8192 methods -- a
modest **1.2x** speedup relative to Boost.Python, which required 116.35
seconds).
.. image:: pybind11_vs_boost_python1.svg
Differences between the two libraries become more pronounced when considering
the file size of the generated Python plugin. Note that the plot below does not
include the size of the Boost.Python shared library, hence Boost actually has a
slight advantage.
Module size
-----------
Differences between the two libraries become much more pronounced when
considering the file size of the generated Python plugin: for the largest file,
the binary generated by Boost.Python required 16.8 MiB, which was **2.17
times** / **9.1 megabytes** larger than the output generated by pybind11. For
very small inputs, Boost.Python has an edge in the plot below -- however, note
that it stores many definitions in an external library, whose size was not
included here, hence the comparison is slightly shifted in Boost.Python's
favor.
.. image:: pybind11_vs_boost_python2.svg
Despite this, the libraries procuced by Boost.Python for more than a few
functions are consistently larger by a factor of 1.75.
......@@ -39,8 +39,16 @@ and that the pybind11 repository is located in a subdirectory named :file:`pybin
# find_package(PythonInterp ${PYTHONLIBS_VERSION_STRING} EXACT REQUIRED)
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")
# Enable C++11 mode on C++ / Clang
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
CHECK_CXX_COMPILER_FLAG("-std=c++14" HAS_CPP14_FLAG)
CHECK_CXX_COMPILER_FLAG("-std=c++11" HAS_CPP11_FLAG)
if (HAS_CPP14_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
elseif (HAS_CPP11_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
else()
message(FATAL_ERROR "Unsupported compiler -- at least C++11 support is needed!")
endif()
# Enable link time optimization and set the default symbol
# visibility to hidden (very important to obtain small binaries)
......
......@@ -18,12 +18,12 @@ become an excessively large and unnecessary dependency.
Think of this library as a tiny self-contained version of Boost.Python with
everything stripped away that isn't relevant for binding generation. The core
header files only require ~2K lines of code and depend on Python (2.7 or 3.x)
header files only require ~3K lines of code and depend on Python (2.7 or 3.x)
and the C++ standard library. This compact implementation was possible thanks
to some of the new C++11 language features (tuples, lambda functions and
variadic templates). Since its creation, this library has grown beyond
Boost.Python in many ways, leading to dramatically simpler binding code in many
common situations.
to some of the new C++11 language features (specifically: tuples, lambda
functions and variadic templates). Since its creation, this library has grown
beyond Boost.Python in many ways, leading to dramatically simpler binding code
in many common situations.
Core features
*************
......@@ -64,3 +64,10 @@ In addition to the core functionality, pybind11 provides some extra goodies:
- Everything is contained in just a few header files; there is no need to link
against any additional libraries.
- Binaries are generally smaller by a factor of 2 or more compared to
equivalent bindings generated by Boost.Python.
- When supported by the compiler, two new C++14 features (relaxed constexpr,
return value deduction) such as are used to do additional work at compile
time, leading to smaller binaries.
This diff is collapsed.
This diff is collapsed.
......@@ -14,4 +14,5 @@ void kw_func(int x, int y) { std::cout << "kw_func(x=" << x << ", y=" << y << ")
void init_ex11(py::module &m) {
m.def("kw_func", &kw_func, py::arg("x"), py::arg("y"));
m.def("kw_func2", &kw_func, py::arg("x") = 100, py::arg("y") = 200);
m.def("kw_func3", [](const char *) { }, py::arg("data") = std::string("Hello world!"));
}
#!/usr/bin/env python
from __future__ import print_function
import sys, pydoc
sys.path.append('.')
import sys
import pydoc
import example
sys.path.append('.')
from example import kw_func
from example import kw_func2
from example import kw_func, kw_func2, kw_func3
print(pydoc.render_doc(kw_func, "Help on %s"))
print(pydoc.render_doc(kw_func2, "Help on %s"))
print(pydoc.render_doc(kw_func3, "Help on %s"))
kw_func(5, 10)
kw_func(5, y = 10)
kw_func(y = 10, x = 5)
kw_func(5, y=10)
kw_func(y=10, x=5)
kw_func2()
......@@ -24,3 +24,8 @@ kw_func2(y=10)
kw_func2(5, 10)
kw_func2(x=5, y=10)
try:
kw_func2(x=5, y=10, z=12)
except Exception as e:
print("Caught expected exception: " + str(e))
Help on built-in function kw_func
kkww__ffuunncc(...)
Signature : (x : int, y : int) -> None
Help on built-in function kw_func2
kkww__ffuunncc22(...)
Signature : (x : int = 100L, y : int = 200L) -> None
Help on built-in function kw_func3
kkww__ffuunncc33(...)
Signature : (data : str = u'Hello world!') -> None
kw_func(x=5, y=10)
kw_func(x=5, y=10)
kw_func(x=5, y=10)
......@@ -7,13 +22,6 @@ kw_func(x=5, y=200)
kw_func(x=100, y=10)
kw_func(x=5, y=10)
kw_func(x=5, y=10)
Help on built-in function kw_func
kkww__ffuunncc(...) method of builtins.PyCapsule instance
Signature : (x : int, y : int) -> None
Help on built-in function kw_func2
kkww__ffuunncc22(...) method of builtins.PyCapsule instance
Signature : (x : int = 100, y : int = 200) -> None
Caught expected exception: Incompatible function arguments. The following argument types are supported:
1. (x : int = 100L, y : int = 200L) -> None
......@@ -12,8 +12,8 @@
#include "pytypes.h"
#include "typeid.h"
#include "descr.h"
#include <array>
#include <list>
#include <limits>
NAMESPACE_BEGIN(pybind11)
......@@ -25,80 +25,6 @@ NAMESPACE_BEGIN(detail)
#define PYBIND11_AS_STRING PyString_AsString
#endif
/** Linked list descriptor type for function signatures (produces smaller binaries
compared to a previous solution using std::string and operator +=) */
class descr {
public:
struct entry {
const std::type_info *type = nullptr;
const char *str = nullptr;
entry *next = nullptr;
entry(const std::type_info *type) : type(type) { }
entry(const char *str) : str(str) { }
};
descr() { }
descr(descr &&d) : first(d.first), last(d.last) { d.first = d.last = nullptr; }
PYBIND11_NOINLINE descr(const char *str) { first = last = new entry { str }; }
PYBIND11_NOINLINE descr(const std::type_info &type) { first = last = new entry { &type }; }
PYBIND11_NOINLINE void operator+(const char *str) {
entry *next = new entry { str };
last->next = next;
last = next;
}
PYBIND11_NOINLINE void operator+(const std::type_info *type) {
entry *next = new entry { type };
last->next = next;
last = next;
}
PYBIND11_NOINLINE void operator+=(descr &&other) {
last->next = other.first;
while (last->next)
last = last->next;
other.first = other.last = nullptr;
}
PYBIND11_NOINLINE friend descr operator+(descr &&l, descr &&r) {
descr result(std::move(l));
result += std::move(r);
return result;
}
PYBIND11_NOINLINE std::string str() const {
std::string result;
auto const& registered_types = get_internals().registered_types;
for (entry *it = first; it != nullptr; it = it->next) {
if (it->type) {
auto it2 = registered_types.find(it->type);
if (it2 != registered_types.end()) {
result += it2->second.type->tp_name;
} else {
std::string tname(it->type->name());
detail::clean_type_id(tname);
result += tname;
}
} else {
result += it->str;
}
}
return result;
}
PYBIND11_NOINLINE ~descr() {
while (first) {
entry *tmp = first->next;
delete first;
first = tmp;
}
}
entry *first = nullptr;
entry *last = nullptr;
};
class type_caster_custom {
public:
PYBIND11_NOINLINE type_caster_custom(const std::type_info *type_info) {
......@@ -195,7 +121,7 @@ protected:
/// Generic type caster for objects stored on the heap
template <typename type, typename Enable = void> class type_caster : public type_caster_custom {
public:
static descr name() { return typeid(type); }
static PYBIND11_DESCR name() { return type_descr(_<type>()); }
type_caster() : type_caster_custom(&typeid(type)) { }
......@@ -224,7 +150,7 @@ protected:
protected: \
type value; \
public: \
static descr name() { return py_name; } \
static PYBIND11_DESCR name() { return type_descr(py_name); } \
static PyObject *cast(const type *src, return_value_policy policy, PyObject *parent) { \
return cast(*src, policy, parent); \
} \
......@@ -296,9 +222,10 @@ public:
return cast(*src, policy, parent);
}
static descr name() {
return std::is_floating_point<T>::value ? "float" : "int";
}
template <typename T2 = T, typename std::enable_if<std::is_integral<T2>::value, int>::type = 0>
static PYBIND11_DESCR name() { return type_descr(_("int")); }
template <typename T2 = T, typename std::enable_if<!std::is_integral<T2>::value, int>::type = 0>
static PYBIND11_DESCR name() { return type_descr(_("float")); }
operator T*() { return &value; }
operator T&() { return value; }
......@@ -314,7 +241,7 @@ public:
Py_INCREF(Py_None);
return Py_None;
}
PYBIND11_TYPE_CASTER(void_type, "None");
PYBIND11_TYPE_CASTER(void_type, _("None"));
};
template <> class type_caster<void> : public type_caster<void_type> { };
......@@ -332,7 +259,7 @@ public:
Py_INCREF(result);
return result;
}
PYBIND11_TYPE_CASTER(bool, "bool");
PYBIND11_TYPE_CASTER(bool, _("bool"));
};
template <> class type_caster<std::string> {
......@@ -352,7 +279,7 @@ public:
static PyObject *cast(const std::string &src, return_value_policy /* policy */, PyObject * /* parent */) {
return PyUnicode_FromString(src.c_str());
}
PYBIND11_TYPE_CASTER(std::string, "str");
PYBIND11_TYPE_CASTER(std::string, _("str"));
};
template <> class type_caster<char> {
......@@ -379,7 +306,7 @@ public:
return PyUnicode_DecodeLatin1(str, 1, nullptr);
}
static descr name() { return "str"; }
static PYBIND11_DESCR name() { return type_descr(_("str")); }
operator char*() { return (char *) value.c_str(); }
operator char() { if (value.length() > 0) return value[0]; else return '\0'; }
......@@ -411,13 +338,10 @@ public:
return tuple;
}
static descr name() {
class descr result("(");
result += std::move(type_caster<typename decay<T1>::type>::name());
result += ", ";
result += std::move(type_caster<typename decay<T2>::type>::name());
result += ")";
return result;
static PYBIND11_DESCR name() {
return type_descr(
_("(") + type_caster<typename decay<T1>::type>::name() +
_(", ") + type_caster<typename decay<T2>::type>::name() + _(")"));
}
operator type() {
......@@ -441,31 +365,11 @@ public:
return cast(src, policy, parent, typename make_index_sequence<size>::type());
}
static descr name(const std::list<argument_entry> &args = std::list<argument_entry>()) {
std::array<class descr, size> type_names {{
type_caster<typename decay<Tuple>::type>::name()...
}};
auto it = args.begin();
class descr result("(");
for (int i=0; i<size; ++i) {
if (it != args.end()) {
result += it->name;
result += " : ";
}
result += std::move(type_names[i]);
if (it != args.end()) {
if (it->descr) {
result += " = ";
result += it->descr;
}
++it;
}
if (i+1 < size)
result += ", ";
++it;
}
result += ")";
return result;
static PYBIND11_DESCR name() {
return type_descr(
_("(") +
detail::concat(type_caster<typename decay<Tuple>::type>::name()...) +
_(")"));
}
template <typename ReturnValue, typename Func> typename std::enable_if<!std::is_void<ReturnValue>::value, ReturnValue>::type call(Func &&f) {
......@@ -576,7 +480,7 @@ public:
src.inc_ref();
return (PyObject *) src.ptr();
}
PYBIND11_TYPE_CASTER(type, typeid(type));
PYBIND11_TYPE_CASTER(type, _<type>());
};
NAMESPACE_END(detail)
......
......@@ -167,16 +167,12 @@ struct overload_hash {
/// Stores information about a keyword argument
struct argument_entry {
char *name; ///< Argument name
char *descr; ///< Human-readable version of the argument value
PyObject *value; ///< Associated Python object
const char *name; ///< Argument name
const char *descr; ///< Human-readable version of the argument value
PyObject *value; ///< Associated Python object
argument_entry(char *name, char *descr, PyObject *value)
argument_entry(const char *name, const char *descr, PyObject *value)
: name(name), descr(descr), value(value) { }
~argument_entry() {
free(name); free(descr); Py_XDECREF(value);
}
};
/// Internal data struture used to track registered instances and types
......
......@@ -34,7 +34,7 @@ public:
return PyComplex_FromDoubles((double) src.real(), (double) src.imag());
}
PYBIND11_TYPE_CASTER(std::complex<T>, "complex");
PYBIND11_TYPE_CASTER(std::complex<T>, _("complex"));
};
NAMESPACE_END(detail)
NAMESPACE_END(pybind11)
/*
pybind11/descr.h: Helper type for concatenating type signatures
either at runtime (C++11) or compile time (C++14)
Copyright (c) 2015 Wenzel Jakob <wenzel@inf.ethz.ch>
All rights reserved. Use of this source code is governed by a
BSD-style license that can be found in the LICENSE file.
*/
#pragma once
#include "common.h"
NAMESPACE_BEGIN(pybind11)
NAMESPACE_BEGIN(detail)
#if defined(__clang__)
# if __has_feature(cxx_return_type_deduction) && __has_feature(cxx_relaxed_constexpr)
# define PYBIND11_CPP14
# endif
#elif defined(__GNUG__)
# if __cpp_constexpr >= 201304 && __cpp_decltype_auto >= 201304
# define PYBIND11_CPP14
# endif
#endif
#if defined(PYBIND11_CPP14) /* Concatenate type signatures at compile time using C++14 */
template <size_t Size1, size_t Size2> class descr {
template <size_t Size1_, size_t Size2_> friend class descr;
public:
constexpr descr(char const (&text) [Size1+1], const std::type_info * const (&types)[Size2+1])
: descr(text, types,
typename make_index_sequence<Size1>::type(),
typename make_index_sequence<Size2>::type()) { }
constexpr const char *text() const { return m_text; }
constexpr const std::type_info * const * types() const { return m_types; }
template <size_t OtherSize1, size_t OtherSize2>
constexpr descr<Size1 + OtherSize1, Size2 + OtherSize2> operator+(const descr<OtherSize1, OtherSize2> &other) const {
return concat(other,
typename make_index_sequence<Size1>::type(),
typename make_index_sequence<Size2>::type(),
typename make_index_sequence<OtherSize1>::type(),
typename make_index_sequence<OtherSize2>::type());
}
protected:
template <size_t... Indices1, size_t... Indices2>
constexpr descr(
char const (&text) [Size1+1],
const std::type_info * const (&types) [Size2+1],
index_sequence<Indices1...>, index_sequence<Indices2...>)
: m_text{text[Indices1]..., '\0'},
m_types{types[Indices2]..., nullptr } {}
template <size_t OtherSize1, size_t OtherSize2, size_t... Indices1,
size_t... Indices2, size_t... OtherIndices1, size_t... OtherIndices2>
constexpr descr<Size1 + OtherSize1, Size2 + OtherSize2>
concat(const descr<OtherSize1, OtherSize2> &other,
index_sequence<Indices1...>, index_sequence<Indices2...>,
index_sequence<OtherIndices1...>, index_sequence<OtherIndices2...>) const {
return descr<Size1 + OtherSize1, Size2 + OtherSize2>(
{ m_text[Indices1]..., other.m_text[OtherIndices1]..., '\0' },
{ m_types[Indices2]..., other.m_types[OtherIndices2]..., nullptr }
);
}
protected:
char m_text[Size1 + 1];
const std::type_info * m_types[Size2 + 1];
};
template <size_t Size> constexpr descr<Size - 1, 0> _(char const(&text)[Size]) {
return descr<Size - 1, 0>(text, { nullptr });
}
template <typename Type> constexpr descr<1, 1> _() {
return descr<1, 1>({ '%', '\0' }, { &typeid(Type), nullptr });
}
inline constexpr descr<0, 0> concat() { return _(""); }
template <size_t Size1, size_t Size2, typename... Args> auto constexpr concat(descr<Size1, Size2> descr) { return descr; }
template <size_t Size1, size_t Size2, typename... Args> auto constexpr concat(descr<Size1, Size2> descr, Args&&... args) { return descr + _(", ") + concat(args...); }
template <size_t Size1, size_t Size2> auto constexpr type_descr(descr<Size1, Size2> descr) { return _("{") + descr + _("}"); }
#define PYBIN