Source of truth for google3_iwyu_pragmas.patch

Merging and upstreaming this PR is deferred until Include Cleaner supports removing the "third_party/pybind11/include/" path name parts.

Author: rwgk

docs/type_caster_iwyu.rst

@@ -0,0 +1,68 @@
+pybind11 ``type_caster`` design & `clangd Include Cleaner`_
+===========================================================
+
+This document is purely to explain — in a nutshell — why pybind11 include
+files with ``type_caster`` specializations require this:
+
+.. code-block:: cpp
+
+  // IWYU pragma: always_keep
+
+For general technical information about the pybind11 ``type_caster`` mechanism
+see the `Custom type casters`_ section.
+
+The problem
+-----------
+
+The `clangd Include Cleaner`_ cannot possibly have the ability to detect which
+``type_caster`` specialization is the correct one to use in a given C++
+translation unit. Without IWYU pragmas, Include Cleaner is likely to suggest
+removing ``type_caster`` include files.
+
+The consequences
+----------------
+
+1. Incorrect runtime behavior.
+2. `ODR violations`_.
+
+Example for 1. Incorrect runtime behavior
+-----------------------------------------
+
+.. code-block:: cpp
+
+  #include <pybind11/pybind11.h>
+  #include <pybind11/stl.h>
+
+If the stl.h include is removed (as suggested by Include Cleaner), conversions
+between e.g. ``std::vector`` and Python ``list`` will no longer work. In most
+cases this will be very obvious at runtime, but there are less obvious
+situations, most notably for ``type_caster`` specializations that inherit from
+``type_caster_base`` (e.g. `pybind11_abseil/status_caster.h`_).
+Some conversions may still work correctly, while others will have unexpected
+behavior.
+
+Explanation for 2. ODR violations
+---------------------------------
+
+This problem only arises if two or more C++ translation units are statically
+linked (e.g. one ``.so`` Python extension built from multiple .cpp files), or
+all visibility-restricting features (e.g. ``-fvisibility=hidden``,
+``namespace pybind11`` ``__attribute__((visibility("hidden")))``,
+``RTLD_LOCAL``) are disabled.
+
+Consider the same simple code example as above: if the stl.h include is missing
+(as suggested by Include Cleaner) in one translation unit, but not in another
+(maybe because Include Cleaner was never applied), the resulting Python
+extension will have link incompatibilities, which is often referred to as
+ODR violations. The behavior is entirely undefined. For better or worse, the
+extension may perform as desired for years, until one day it suddenly does
+not, because of some unrelated change in the environment (e.g. new compiler
+version, a system library update), resulting in seemingly inexplicable failures
+
+See also: `pybind/pybind11#4022`_ (pybind11 smart_holder branch)
+
+.. _`clangd Include Cleaner`: https://clangd.llvm.org/design/include-cleaner
+.. _`Custom type casters`: https://pybind11.readthedocs.io/en/stable/advanced/cast/custom.html
+.. _`ODR violations`: https://en.cppreference.com/w/cpp/language/definition
+.. _`pybind11_abseil/status_caster.h`: https://github.com/pybind/pybind11_abseil/blob/4b883e48ae749ff984c220484d54fdeb0cb4302c/pybind11_abseil/status_caster.h#L52-L53).
+.. _`pybind/pybind11#4022`: https://github.com/pybind/pybind11/pull/4022

include/pybind11/attr.h

@@ -10,7 +10,9 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 #include "cast.h"
 
 #include <functional>

include/pybind11/buffer_info.h

@@ -9,7 +9,9 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 

include/pybind11/cast.h

@@ -8,15 +8,19 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
 #include "detail/descr.h"
 #include "detail/native_enum_data.h"
 #include "detail/smart_holder_sfinae_hooks_only.h"
 #include "detail/type_caster_base.h"
 #include "detail/type_caster_odr_guard.h"
 #include "detail/typeid.h"
+// IWYU pragma: end_exports
 #include "pytypes.h"
 
 #include <array>

include/pybind11/chrono.h

@@ -8,6 +8,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
 #include "pybind11.h"

include/pybind11/complex.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
 #include "pybind11.h"

include/pybind11/detail/abi_platform_id.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "common.h"

include/pybind11/detail/class.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "../attr.h"

include/pybind11/detail/common.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #define PYBIND11_VERSION_MAJOR 2

include/pybind11/detail/cross_extension_shared_state.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #define PYBIND11_HAS_CROSS_EXTENSION_SHARED_STATE

include/pybind11/detail/descr.h

@@ -10,6 +10,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "common.h"

include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/smart_holder.h"
+
 #pragma once
 
 #include "common.h"

include/pybind11/detail/init.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "class.h"

include/pybind11/detail/internals.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "../pytypes.h"

include/pybind11/detail/native_enum_data.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #define PYBIND11_HAS_NATIVE_ENUM

include/pybind11/detail/smart_holder_poc.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/smart_holder.h"
+
 /* Proof-of-Concept for smart pointer interoperability.
 
 High-level aspects:

include/pybind11/detail/smart_holder_sfinae_hooks_only.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "common.h"

include/pybind11/detail/smart_holder_type_casters.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/smart_holder.h"
+
 #pragma once
 
 #include "../gil.h"

include/pybind11/detail/type_caster_base.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "../pytypes.h"

include/pybind11/detail/type_caster_odr_guard.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "descr.h"

include/pybind11/detail/type_map.h

@@ -2,6 +2,8 @@
 // All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include "common.h"

include/pybind11/detail/typeid.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h"
+
 #pragma once
 
 #include <cstdio>

include/pybind11/eigen/matrix.h

@@ -7,10 +7,14 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
 #include "../numpy.h"
+// IWYU pragma: begin_exports
 #include "common.h"
+// IWYU pragma: end_exports
 
 /* HINT: To suppress warnings originating from the Eigen headers, use -isystem.
    See also:

include/pybind11/eigen/tensor.h

@@ -5,10 +5,14 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
 #include "../numpy.h"
+// IWYU pragma: begin_exports
 #include "common.h"
+// IWYU pragma: end_exports
 
 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
 static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0");

include/pybind11/functional.h

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
 #define PYBIND11_HAS_TYPE_CASTER_STD_FUNCTION_SPECIALIZATIONS

include/pybind11/gil.h

@@ -9,12 +9,16 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 
 #include <cassert>
 
 #if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+// IWYU pragma: begin_exports
 #    include "detail/internals.h"
+// IWYU pragma: end_exports
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)

include/pybind11/gil_safe_call_once.h

@@ -2,7 +2,9 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 #include "gil.h"
 
 #include <cassert>

include/pybind11/native_enum.h

@@ -4,9 +4,11 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
 #include "detail/native_enum_data.h"
 #include "detail/type_caster_base.h"
+// IWYU pragma: end_exports
 #include "cast.h"
 
 #include <limits>

include/pybind11/numpy.h

@@ -7,10 +7,14 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst
+
 #pragma once
 
 #include "pybind11.h"
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 #include "complex.h"
 #include "gil_safe_call_once.h"
 #include "pytypes.h"

include/pybind11/options.h

@@ -9,7 +9,9 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 

include/pybind11/pybind11.h

@@ -10,10 +10,12 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/class.h"
 #include "detail/init.h"
 #include "detail/native_enum_data.h"
 #include "detail/smart_holder_sfinae_hooks_only.h"
+// IWYU pragma: end_exports
 #include "attr.h"
 #include "gil.h"
 #include "gil_safe_call_once.h"

include/pybind11/pytypes.h

@@ -9,7 +9,9 @@
 
 #pragma once
 
+// IWYU pragma: begin_exports
 #include "detail/common.h"
+// IWYU pragma: end_exports
 #include "buffer_info.h"
 
 #include <assert.h>