python · vstinner · Oct 13, 2021 · Oct 13, 2021
@@ -598,3 +598,7 @@ Removed
  since Python 3.3. Use ``PyUnicode_CopyCharacters()`` or ``memcpy()``
  (``wchar_t*`` string), and ``PyUnicode_Fill()`` functions instead.
  (Contributed by Victor Stinner in :issue:`41123`.)
+
+* Remove the ``pystrhex.h`` header file. It only contains private functions.
+  C extensions should only include the main ``<Python.h>`` header file.
+  (Contributed by Victor Stinner in :issue:`45434`.)
diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h
@@ -0,0 +1,36 @@
+#ifndef Py_INTERNAL_STRHEX_H
+#define Py_INTERNAL_STRHEX_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+// Returns a str() containing the hex representation of argbuf.
+PyAPI_FUNC(PyObject*) _Py_strhex(const
+    char* argbuf,
+    const Py_ssize_t arglen);
+
+// Returns a bytes() containing the ASCII hex representation of argbuf.
+PyAPI_FUNC(PyObject*) _Py_strhex_bytes(
+    const char* argbuf,
+    const Py_ssize_t arglen);
+
+// These variants include support for a separator between every N bytes:
+PyAPI_FUNC(PyObject*) _Py_strhex_with_sep(
+    const char* argbuf,
+    const Py_ssize_t arglen,
+    const PyObject* sep,
+    const int bytes_per_group);
+PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep(
+    const char* argbuf,
+    const Py_ssize_t arglen,
+    const PyObject* sep,
+    const int bytes_per_group);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_STRHEX_H */
diff --git a/Include/pystrhex.h b/Include/pystrhex.h
@@ -1179,7 +1179,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/pyport.h \
 		$(srcdir)/Include/pystate.h \
 		$(srcdir)/Include/pystrcmp.h \
-		$(srcdir)/Include/pystrhex.h \
 		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pythonrun.h \
 		$(srcdir)/Include/pythread.h \
@@ -1271,6 +1270,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_pymem.h \
 		$(srcdir)/Include/internal/pycore_pystate.h \
 		$(srcdir)/Include/internal/pycore_runtime.h \
+		$(srcdir)/Include/internal/pycore_strhex.h \
 		$(srcdir)/Include/internal/pycore_structseq.h \
 		$(srcdir)/Include/internal/pycore_symtable.h \
 		$(srcdir)/Include/internal/pycore_sysmodule.h \

diff --git a/Misc/NEWS.d/next/C API/2021-10-13-14-42-46.bpo-45434.INNEEt.rst b/Misc/NEWS.d/next/C API/2021-10-13-14-42-46.bpo-45434.INNEEt.rst
@@ -0,0 +1,3 @@
+Remove the ``pystrhex.h`` header file. It only contains private functions. C
+extensions should only include the main ``<Python.h>`` header file. Patch by
+Victor Stinner.
@@ -212,19 +212,23 @@ _symtable symtablemodule.c
 # OPENSSL=/path/to/openssl/directory
 # _ssl _ssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
-#     -lssl -lcrypto
+#     -lssl -lcrypto \
+#     -DPy_BUILD_CORE_BUILTIN
 #_hashlib _hashopenssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
-#     -lcrypto
+#     -lcrypto \
+#     -DPy_BUILD_CORE_BUILTIN

 # To statically link OpenSSL:
 # _ssl _ssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
 #     -l:libssl.a -Wl,--exclude-libs,libssl.a \
-#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a
+#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a \
+#     -DPy_BUILD_CORE_BUILTIN
 #_hashlib _hashopenssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
-#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a
+#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a \
+#     -DPy_BUILD_CORE_BUILTIN

 # The crypt module is now disabled by default because it breaks builds
 # on many systems (where -lcrypt is needed), e.g. Linux (I believe).
@@ -255,18 +259,18 @@ _symtable symtablemodule.c
 # The _md5 module implements the RSA Data Security, Inc. MD5
 # Message-Digest Algorithm, described in RFC 1321.

-#_md5 md5module.c
+#_md5 md5module.c -DPy_BUILD_CORE_BUILTIN


 # The _sha module implements the SHA checksum algorithms.
 # (NIST's Secure Hash Algorithms.)
-#_sha1 sha1module.c
+#_sha1 sha1module.c -DPy_BUILD_CORE_BUILTIN
 #_sha256 sha256module.c -DPy_BUILD_CORE_BUILTIN
 #_sha512 sha512module.c -DPy_BUILD_CORE_BUILTIN
-#_sha3 _sha3/sha3module.c
+#_sha3 _sha3/sha3module.c -DPy_BUILD_CORE_BUILTIN

 # _blake module
-#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
+#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c -DPy_BUILD_CORE_BUILTIN

 # The _tkinter module.
 #
@@ -339,7 +343,7 @@ _symtable symtablemodule.c


 # Helper module for various ascii-encoders
-#binascii binascii.c
+#binascii binascii.c -DPy_BUILD_CORE_MODULE

 # Andrew Kuchling's zlib module.
 # This require zlib 1.1.3 (or later).

@@ -14,7 +14,7 @@
 */

 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"       // _Py_strhex()

 #include "../hashlib.h"
 #include "blake2ns.h"

@@ -14,7 +14,7 @@
 */

 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()

 #include "../hashlib.h"
 #include "blake2ns.h"

@@ -22,7 +22,7 @@

 #include "Python.h"
 #include "hashlib.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()

 /* EVP is the preferred interface to hashing in OpenSSL */
 #include <openssl/evp.h>

@@ -16,7 +16,7 @@
 */

 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()
 #include "../hashlib.h"

 /* **************************************************************************

diff --git a/Modules/binascii.c b/Modules/binascii.c
@@ -56,9 +56,9 @@
 #define PY_SSIZE_T_CLEAN

 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex_bytes_with_sep()
 #ifdef USE_ZLIB_CRC32
-#include "zlib.h"
+#  include "zlib.h"
 #endif

 typedef struct binascii_state {

@@ -18,7 +18,7 @@

 #include "Python.h"
 #include "hashlib.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()

 /*[clinic input]
 module _md5

@@ -18,7 +18,7 @@

 #include "Python.h"
 #include "hashlib.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()

 /*[clinic input]
 module _sha1

@@ -18,9 +18,9 @@

 #include "Python.h"
 #include "pycore_bitutils.h"      // _Py_bswap32()
+#include "pycore_strhex.h"        // _Py_strhex()
 #include "structmember.h"         // PyMemberDef
 #include "hashlib.h"
-#include "pystrhex.h"

 /*[clinic input]
 module _sha256

@@ -18,9 +18,9 @@

 #include "Python.h"
 #include "pycore_bitutils.h"      // _Py_bswap64()
+#include "pycore_strhex.h"        // _Py_strhex()
 #include "structmember.h"         // PyMemberDef
 #include "hashlib.h"
-#include "pystrhex.h"

 /*[clinic input]
 module _sha512

diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
@@ -4,9 +4,9 @@
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
 #include "pycore_bytes_methods.h"
-#include "pycore_object.h"
+#include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_strhex.h"        // _Py_strhex_with_sep()
 #include "bytesobject.h"
-#include "pystrhex.h"

 /*[clinic input]
 class bytearray "PyByteArrayObject *" "&PyByteArray_Type"

diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
@@ -10,8 +10,8 @@
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_object.h"        // _PyObject_GC_TRACK
 #include "pycore_pymem.h"         // PYMEM_CLEANBYTE
+#include "pycore_strhex.h"        // _Py_strhex_with_sep()

-#include "pystrhex.h"
 #include <stddef.h>

 /*[clinic input]

diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c
@@ -11,10 +11,10 @@
 */

 #include "Python.h"
-#include "pycore_abstract.h"   // _PyIndex_Check()
-#include "pycore_object.h"
-#include "pystrhex.h"
-#include <stddef.h>
+#include "pycore_abstract.h"      // _PyIndex_Check()
+#include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_strhex.h"        // _Py_strhex_with_sep()
+#include <stddef.h>               // offsetof()

 /*[clinic input]
 class memoryview "PyMemoryViewObject *" "&PyMemoryView_Type"

@@ -209,6 +209,7 @@
    <ClInclude Include="..\Include\internal\pycore_pymem.h" />
    <ClInclude Include="..\Include\internal\pycore_pystate.h" />
    <ClInclude Include="..\Include\internal\pycore_runtime.h" />
+    <ClInclude Include="..\Include\internal\pycore_strhex.h" />
    <ClInclude Include="..\Include\internal\pycore_structseq.h" />
    <ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
    <ClInclude Include="..\Include\internal\pycore_symtable.h" />
@@ -248,7 +249,6 @@
    <ClInclude Include="..\Include\pyport.h" />
    <ClInclude Include="..\Include\pystate.h" />
    <ClInclude Include="..\Include\pystrcmp.h" />
-    <ClInclude Include="..\Include\pystrhex.h" />
    <ClInclude Include="..\Include\pystrtod.h" />
    <ClInclude Include="..\Include\pythonrun.h" />
    <ClInclude Include="..\Include\pythread.h" />

@@ -201,9 +201,6 @@
    <ClInclude Include="..\Include\pystrtod.h">
      <Filter>Include</Filter>
    </ClInclude>
-    <ClInclude Include="..\Include\pystrhex.h">
-      <Filter>Include</Filter>
-    </ClInclude>
    <ClInclude Include="..\Include\Python.h">
      <Filter>Include</Filter>
    </ClInclude>
@@ -588,6 +585,9 @@
    <ClInclude Include="..\Include\internal\pycore_runtime.h">
      <Filter>Include\internal</Filter>
    </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_strhex.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
    <ClInclude Include="..\Include\internal\pycore_sysmodule.h">
      <Filter>Include\internal</Filter>
    </ClInclude>

diff --git a/Python/pystrhex.c b/Python/pystrhex.c
@@ -1,8 +1,8 @@
-/* bytes to hex implementation */
+/* Format bytes as hexadecimal */

 #include "Python.h"
+#include "pycore_strhex.h"       // _Py_strhex_with_sep()

-#include "pystrhex.h"

 static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen,
                                 const PyObject* sep, int bytes_per_sep_group,

diff --git a/setup.py b/setup.py
@@ -1709,12 +1709,12 @@ def detect_compress_exts(self):

        # Helper module for various ascii-encoders.  Uses zlib for an optimized
        # crc32 if we have it.  Otherwise binascii uses its own.
+        extra_compile_args = ['-DPy_BUILD_CORE_MODULE']
        if have_zlib:
-            extra_compile_args = ['-DUSE_ZLIB_CRC32']
+            extra_compile_args.append('-DUSE_ZLIB_CRC32')
            libraries = ['z']
            extra_link_args = zlib_extra_link_args
        else:
-            extra_compile_args = []
            libraries = []
            extra_link_args = []
        self.add(Extension('binascii', ['binascii.c'],
@@ -2469,6 +2469,7 @@ def split_var(name, sep):
            library_dirs=openssl_libdirs,
            libraries=openssl_libs,
            runtime_library_dirs=runtime_library_dirs,
+            extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
        )

        # This static linking is NOT OFFICIALLY SUPPORTED.
@@ -2530,27 +2531,29 @@ def detect_hash_builtins(self):
        if "sha256" in configured:
            self.add(Extension(
                '_sha256', ['sha256module.c'],
+                depends=['hashlib.h'],
                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
-                depends=['hashlib.h']
            ))

        if "sha512" in configured:
            self.add(Extension(
                '_sha512', ['sha512module.c'],
+                depends=['hashlib.h'],
                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
-                depends=['hashlib.h']
            ))

        if "md5" in configured:
            self.add(Extension(
                '_md5', ['md5module.c'],
-                depends=['hashlib.h']
+                depends=['hashlib.h'],
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
            ))

        if "sha1" in configured:
            self.add(Extension(
                '_sha1', ['sha1module.c'],
-                depends=['hashlib.h']
+                depends=['hashlib.h'],
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
            ))

        if "blake2" in configured:
@@ -2565,7 +2568,8 @@ def detect_hash_builtins(self):
                    '_blake2/blake2b_impl.c',
                    '_blake2/blake2s_impl.c'
                ],
-                depends=blake2_deps
+                depends=blake2_deps,
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
            ))

        if "sha3" in configured:
@@ -2576,7 +2580,8 @@ def detect_hash_builtins(self):
            self.add(Extension(
                '_sha3',
                ['_sha3/sha3module.c'],
-                depends=sha3_deps
+                depends=sha3_deps,
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
            ))

    def detect_nis(self):