tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h
2index 71bc5902..6a148e74 100644
3--- a/include/pybind11/stl.h
4+++ b/include/pybind11/stl.h
5@@ -11,10 +11,14 @@
6
7 #include "pybind11.h"
8 #include "detail/common.h"
9+#include "detail/descr.h"
10+#include "detail/type_caster_base.h"
11
12 #include <deque>
13+#include <initializer_list>
14 #include <list>
15 #include <map>
16+#include <memory>
17 #include <ostream>
18 #include <set>
19 #include <unordered_map>
20@@ -35,6 +39,89 @@
21 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
22 PYBIND11_NAMESPACE_BEGIN(detail)
23
24+//
25+// Begin: Equivalent of
26+// https://github.com/google/clif/blob/ae4eee1de07cdf115c0c9bf9fec9ff28efce6f6c/clif/python/runtime.cc#L388-L438
27+/*
28+The three `PyObjectTypeIsConvertibleTo*()` functions below are
29+the result of converging the behaviors of pybind11 and PyCLIF
30+(http://github.com/google/clif).
31+
32+Originally PyCLIF was extremely far on the permissive side of the spectrum,
33+while pybind11 was very far on the strict side. Originally PyCLIF accepted any
34+Python iterable as input for a C++ `vector`/`set`/`map` argument, as long as
35+the elements were convertible. The obvious (in hindsight) problem was that
36+any empty Python iterable could be passed to any of these C++ types, e.g. `{}`
37+was accepted for C++ `vector`/`set` arguments, or `[]` for C++ `map` arguments.
38+
39+The functions below strike a practical permissive-vs-strict compromise,
40+informed by tens of thousands of use cases in the wild. A main objective is
41+to prevent accidents and improve readability:
42+
43+- Python literals must match the C++ types.
44+
45+- For C++ `set`: The potentially reducing conversion from a Python sequence
46+ (e.g. Python `list` or `tuple`) to a C++ `set` must be explicit, by going
47+ through a Python `set`.
48+
49+- However, a Python `set` can still be passed to a C++ `vector`. The rationale
50+ is that this conversion is not reducing. Implicit conversions of this kind
51+ are also fairly commonly used, therefore enforcing explicit conversions
52+ would have an unfavorable cost : benefit ratio; more sloppily speaking,
53+ such an enforcement would be more annoying than helpful.
54+*/
55+
56+inline bool PyObjectIsInstanceWithOneOfTpNames(PyObject *obj,
57+ std::initializer_list<const char *> tp_names) {
58+ if (PyType_Check(obj)) {
59+ return false;
60+ }
61+ const char *obj_tp_name = Py_TYPE(obj)->tp_name;
62+ for (const auto *tp_name : tp_names) {
63+ if (std::strcmp(obj_tp_name, tp_name) == 0) {
64+ return true;
65+ }
66+ }
67+ return false;
68+}
69+
70+inline bool PyObjectTypeIsConvertibleToStdVector(PyObject *obj) {
71+ if (PySequence_Check(obj) != 0) {
72+ return !PyUnicode_Check(obj) && !PyBytes_Check(obj);
73+ }
74+ return (PyGen_Check(obj) != 0) || (PyAnySet_Check(obj) != 0)
75+ || PyObjectIsInstanceWithOneOfTpNames(
76+ obj, {"dict_keys", "dict_values", "dict_items", "map", "zip"});
77+}
78+
79+inline bool PyObjectTypeIsConvertibleToStdSet(PyObject *obj) {
80+ return (PyAnySet_Check(obj) != 0) || PyObjectIsInstanceWithOneOfTpNames(obj, {"dict_keys"});
81+}
82+
83+inline bool PyObjectTypeIsConvertibleToStdMap(PyObject *obj) {
84+ if (PyDict_Check(obj)) {
85+ return true;
86+ }
87+ // Implicit requirement in the conditions below:
88+ // A type with `.__getitem__()` & `.items()` methods must implement these
89+ // to be compatible with https://docs.python.org/3/c-api/mapping.html
90+ if (PyMapping_Check(obj) == 0) {
91+ return false;
92+ }
93+ PyObject *items = PyObject_GetAttrString(obj, "items");
94+ if (items == nullptr) {
95+ PyErr_Clear();
96+ return false;
97+ }
98+ bool is_convertible = (PyCallable_Check(items) != 0);
99+ Py_DECREF(items);
100+ return is_convertible;
101+}
102+
103+//
104+// End: Equivalent of clif/python/runtime.cc
105+//
106+
107 /// Extracts an const lvalue reference or rvalue reference for U based on the type of T (e.g. for
108 /// forwarding a container element). Typically used indirect via forwarded_type(), below.
109 template <typename T, typename U>
110@@ -66,17 +153,10 @@ private:
111 }
112 void reserve_maybe(const anyset &, void *) {}
113
114-public:
115- bool load(handle src, bool convert) {
116- if (!isinstance<anyset>(src)) {
117- return false;
118- }
119- auto s = reinterpret_borrow<anyset>(src);
120- value.clear();
121- reserve_maybe(s, &value);
122- for (auto entry : s) {
123+ bool convert_iterable(const iterable &itbl, bool convert) {
124+ for (const auto &it : itbl) {
125 key_conv conv;
126- if (!conv.load(entry, convert)) {
127+ if (!conv.load(it, convert)) {
128 return false;
129 }
130 value.insert(cast_op<Key &&>(std::move(conv)));
131@@ -84,6 +164,29 @@ public:
132 return true;
133 }
134
135+ bool convert_anyset(anyset s, bool convert) {
136+ value.clear();
137+ reserve_maybe(s, &value);
138+ return convert_iterable(s, convert);
139+ }
140+
141+public:
142+ bool load(handle src, bool convert) {
143+ if (!PyObjectTypeIsConvertibleToStdSet(src.ptr())) {
144+ return false;
145+ }
146+ if (isinstance<anyset>(src)) {
147+ value.clear();
148+ return convert_anyset(reinterpret_borrow<anyset>(src), convert);
149+ }
150+ if (!convert) {
151+ return false;
152+ }
153+ assert(isinstance<iterable>(src));
154+ value.clear();
155+ return convert_iterable(reinterpret_borrow<iterable>(src), convert);
156+ }
157+
158 template <typename T>
159 static handle cast(T &&src, return_value_policy policy, handle parent) {
160 if (!std::is_lvalue_reference<T>::value) {
161@@ -115,15 +218,10 @@ private:
162 }
163 void reserve_maybe(const dict &, void *) {}
164
165-public:
166- bool load(handle src, bool convert) {
167- if (!isinstance<dict>(src)) {
168- return false;
169- }
170- auto d = reinterpret_borrow<dict>(src);
171+ bool convert_elements(const dict &d, bool convert) {
172 value.clear();
173 reserve_maybe(d, &value);
174- for (auto it : d) {
175+ for (const auto &it : d) {
176 key_conv kconv;
177 value_conv vconv;
178 if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) {
179@@ -134,6 +232,25 @@ public:
180 return true;
181 }
182
183+public:
184+ bool load(handle src, bool convert) {
185+ if (!PyObjectTypeIsConvertibleToStdMap(src.ptr())) {
186+ return false;
187+ }
188+ if (isinstance<dict>(src)) {
189+ return convert_elements(reinterpret_borrow<dict>(src), convert);
190+ }
191+ if (!convert) {
192+ return false;
193+ }
194+ auto items = reinterpret_steal<object>(PyMapping_Items(src.ptr()));
195+ if (!items) {
196+ throw error_already_set();
197+ }
198+ assert(isinstance<iterable>(items));
199+ return convert_elements(dict(reinterpret_borrow<iterable>(items)), convert);
200+ }
201+
202 template <typename T>
203 static handle cast(T &&src, return_value_policy policy, handle parent) {
204 dict d;
205@@ -166,13 +283,35 @@ struct list_caster {
206 using value_conv = make_caster<Value>;
207
208 bool load(handle src, bool convert) {
209- if (!isinstance<sequence>(src) || isinstance<bytes>(src) || isinstance<str>(src)) {
210+ if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) {
211 return false;
212 }
213- auto s = reinterpret_borrow<sequence>(src);
214+ if (isinstance<sequence>(src)) {
215+ return convert_elements(src, convert);
216+ }
217+ if (!convert) {
218+ return false;
219+ }
220+ // Designed to be behavior-equivalent to passing tuple(src) from Python:
221+ // The conversion to a tuple will first exhaust the generator object, to ensure that
222+ // the generator is not left in an unpredictable (to the caller) partially-consumed
223+ // state.
224+ assert(isinstance<iterable>(src));
225+ return convert_elements(tuple(reinterpret_borrow<iterable>(src)), convert);
226+ }
227+
228+private:
229+ template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
230+ void reserve_maybe(const sequence &s, Type *) {
231+ value.reserve(s.size());
232+ }
233+ void reserve_maybe(const sequence &, void *) {}
234+
235+ bool convert_elements(handle seq, bool convert) {
236+ auto s = reinterpret_borrow<sequence>(seq);
237 value.clear();
238 reserve_maybe(s, &value);
239- for (const auto &it : s) {
240+ for (const auto &it : seq) {
241 value_conv conv;
242 if (!conv.load(it, convert)) {
243 return false;
244@@ -182,13 +321,6 @@ struct list_caster {
245 return true;
246 }
247
248-private:
249- template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
250- void reserve_maybe(const sequence &s, Type *) {
251- value.reserve(s.size());
252- }
253- void reserve_maybe(const sequence &, void *) {}
254-
255 public:
256 template <typename T>
257 static handle cast(T &&src, return_value_policy policy, handle parent) {
258@@ -220,43 +352,87 @@ struct type_caster<std::deque<Type, Alloc>> : list_caster<std::deque<Type, Alloc
259 template <typename Type, typename Alloc>
260 struct type_caster<std::list<Type, Alloc>> : list_caster<std::list<Type, Alloc>, Type> {};
261
262+template <typename ArrayType, typename V, size_t... I>
263+ArrayType vector_to_array_impl(V &&v, index_sequence<I...>) {
264+ return {{std::move(v[I])...}};
265+}
266+
267+// Based on https://en.cppreference.com/w/cpp/container/array/to_array
268+template <typename ArrayType, size_t N, typename V>
269+ArrayType vector_to_array(V &&v) {
270+ return vector_to_array_impl<ArrayType, V>(std::forward<V>(v), make_index_sequence<N>{});
271+}
272+
273 template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0>
274 struct array_caster {
275 using value_conv = make_caster<Value>;
276
277 private:
278- template <bool R = Resizable>
279- bool require_size(enable_if_t<R, size_t> size) {
280- if (value.size() != size) {
281- value.resize(size);
282+ std::unique_ptr<ArrayType> value;
283+
284+ template <bool R = Resizable, enable_if_t<R, int> = 0>
285+ bool convert_elements(handle seq, bool convert) {
286+ auto l = reinterpret_borrow<sequence>(seq);
287+ value.reset(new ArrayType{});
288+ // Using `resize` to preserve the behavior exactly as it was before PR #5305
289+ // For the `resize` to work, `Value` must be default constructible.
290+ // For `std::valarray`, this is a requirement:
291+ // https://en.cppreference.com/w/cpp/named_req/NumericType
292+ value->resize(l.size());
293+ size_t ctr = 0;
294+ for (const auto &it : l) {
295+ value_conv conv;
296+ if (!conv.load(it, convert)) {
297+ return false;
298+ }
299+ (*value)[ctr++] = cast_op<Value &&>(std::move(conv));
300 }
301 return true;
302 }
303- template <bool R = Resizable>
304- bool require_size(enable_if_t<!R, size_t> size) {
305- return size == Size;
306- }
307
308-public:
309- bool load(handle src, bool convert) {
310- if (!isinstance<sequence>(src)) {
311+ template <bool R = Resizable, enable_if_t<!R, int> = 0>
312+ bool convert_elements(handle seq, bool convert) {
313+ auto l = reinterpret_borrow<sequence>(seq);
314+ if (l.size() != Size) {
315 return false;
316 }
317- auto l = reinterpret_borrow<sequence>(src);
318- if (!require_size(l.size())) {
319- return false;
320- }
321- size_t ctr = 0;
322- for (const auto &it : l) {
323+ // The `temp` storage is needed to support `Value` types that are not
324+ // default-constructible.
325+ // Deliberate choice: no template specializations, for simplicity, and
326+ // because the compile time overhead for the specializations is deemed
327+ // more significant than the runtime overhead for the `temp` storage.
328+ std::vector<Value> temp;
329+ temp.reserve(l.size());
330+ for (auto it : l) {
331 value_conv conv;
332 if (!conv.load(it, convert)) {
333 return false;
334 }
335- value[ctr++] = cast_op<Value &&>(std::move(conv));
336+ temp.emplace_back(cast_op<Value &&>(std::move(conv)));
337 }
338+ value.reset(new ArrayType(vector_to_array<ArrayType, Size>(std::move(temp))));
339 return true;
340 }
341
342+public:
343+ bool load(handle src, bool convert) {
344+ if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) {
345+ return false;
346+ }
347+ if (isinstance<sequence>(src)) {
348+ return convert_elements(src, convert);
349+ }
350+ if (!convert) {
351+ return false;
352+ }
353+ // Designed to be behavior-equivalent to passing tuple(src) from Python:
354+ // The conversion to a tuple will first exhaust the generator object, to ensure that
355+ // the generator is not left in an unpredictable (to the caller) partially-consumed
356+ // state.
357+ assert(isinstance<iterable>(src));
358+ return convert_elements(tuple(reinterpret_borrow<iterable>(src)), convert);
359+ }
360+
361 template <typename T>
362 static handle cast(T &&src, return_value_policy policy, handle parent) {
363 list l(src.size());
364@@ -272,12 +448,36 @@ public:
365 return l.release();
366 }
367
368- PYBIND11_TYPE_CASTER(ArrayType,
369- const_name<Resizable>(const_name(""), const_name("Annotated["))
370- + const_name("list[") + value_conv::name + const_name("]")
371- + const_name<Resizable>(const_name(""),
372- const_name(", FixedSize(")
373- + const_name<Size>() + const_name(")]")));
374+ // Code copied from PYBIND11_TYPE_CASTER macro.
375+ // Intentionally preserving the behavior exactly as it was before PR #5305
376+ template <typename T_, enable_if_t<std::is_same<ArrayType, remove_cv_t<T_>>::value, int> = 0>
377+ static handle cast(T_ *src, return_value_policy policy, handle parent) {
378+ if (!src) {
379+ return none().release();
380+ }
381+ if (policy == return_value_policy::take_ownership) {
382+ auto h = cast(std::move(*src), policy, parent);
383+ delete src; // WARNING: Assumes `src` was allocated with `new`.
384+ return h;
385+ }
386+ return cast(*src, policy, parent);
387+ }
388+
389+ // NOLINTNEXTLINE(google-explicit-constructor)
390+ operator ArrayType *() { return &(*value); }
391+ // NOLINTNEXTLINE(google-explicit-constructor)
392+ operator ArrayType &() { return *value; }
393+ // NOLINTNEXTLINE(google-explicit-constructor)
394+ operator ArrayType &&() && { return std::move(*value); }
395+
396+ template <typename T_>
397+ using cast_op_type = movable_cast_op_type<T_>;
398+
399+ static constexpr auto name
400+ = const_name<Resizable>(const_name(""), const_name("Annotated[")) + const_name("list[")
401+ + value_conv::name + const_name("]")
402+ + const_name<Resizable>(
403+ const_name(""), const_name(", FixedSize(") + const_name<Size>() + const_name(")]"));
404 };
405
406 template <typename Type, size_t Size>
407diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp
408index e7db8aaa..84bd4755 100644
409--- a/tests/test_stl.cpp
410+++ b/tests/test_stl.cpp
411@@ -193,6 +193,23 @@ TEST_SUBMODULE(stl, m) {
412 m.def("cast_array", []() { return std::array<int, 2>{{1, 2}}; });
413 m.def("load_array", [](const std::array<int, 2> &a) { return a[0] == 1 && a[1] == 2; });
414
415+ struct NoDefaultCtor {
416+ explicit constexpr NoDefaultCtor(int val) : val{val} {}
417+ int val;
418+ };
419+
420+ struct NoDefaultCtorArray {
421+ explicit constexpr NoDefaultCtorArray(int i)
422+ : arr{{NoDefaultCtor(10 + i), NoDefaultCtor(20 + i)}} {}
423+ std::array<NoDefaultCtor, 2> arr;
424+ };
425+
426+ // test_array_no_default_ctor
427+ py::class_<NoDefaultCtor>(m, "NoDefaultCtor").def_readonly("val", &NoDefaultCtor::val);
428+ py::class_<NoDefaultCtorArray>(m, "NoDefaultCtorArray")
429+ .def(py::init<int>())
430+ .def_readwrite("arr", &NoDefaultCtorArray::arr);
431+
432 // test_valarray
433 m.def("cast_valarray", []() { return std::valarray<int>{1, 4, 9}; });
434 m.def("load_valarray", [](const std::valarray<int> &v) {
435diff --git a/tests/test_stl.py b/tests/test_stl.py
436index 65fda54c..340cdc35 100644
437--- a/tests/test_stl.py
438+++ b/tests/test_stl.py
439@@ -48,6 +48,13 @@ def test_array(doc):
440 )
441
442
443+def test_array_no_default_ctor():
444+ lst = m.NoDefaultCtorArray(3)
445+ assert [e.val for e in lst.arr] == [13, 23]
446+ lst.arr = m.NoDefaultCtorArray(4).arr
447+ assert [e.val for e in lst.arr] == [14, 24]
448+
449+
450 def test_valarray(doc):
451 """std::valarray <-> list"""
452 lst = m.cast_valarray()