[pull] main from python:main

Grammar/python.gram

@@ -789,6 +789,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]:
     | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok }) a=bitwise_or {
         _PyPegen_cmpop_expr_pair(p, NotEq, a) }
     | '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) }
+    | invalid_noteq
     | '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) }
     | '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) }
     | '>' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Gt, a) }
@@ -1615,3 +1616,10 @@ invalid_bitwise_or:
         ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax.  Maybe you meant 'or' or '|' instead of '||'?")
         : NULL
     }
+
+invalid_noteq:
+    | a='<' b='>' {
+      _PyPegen_tokens_are_adjacent(a, b)
+      ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax.  Maybe you meant '!=' instead of '<>'?")
+      : NULL
+    }

Include/cpython/pystate.h

@@ -118,8 +118,7 @@ struct _ts {
 
     int _whence;
 
-    /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED).
-       See Include/internal/pycore_pystate.h for more details. */
+    /* Thread state. See Include/internal/pycore_pystate.h for details. */
     int state;
 
     int py_recursion_remaining;

Include/internal/pycore_pystate.h

@@ -21,32 +21,32 @@ extern "C" {
 // interpreter at the same time. Only the "bound" thread may perform the
 // transitions between "attached" and "detached" on its own PyThreadState.
 //
-// The "suspended" state is used to implement stop-the-world pauses, such as
-// for cyclic garbage collection. It is only used in `--disable-gil` builds.
-// The "suspended" state is similar to the "detached" state in that in both
-// states the thread is not allowed to call most Python APIs. However, unlike
-// the "detached" state, a thread may not transition itself out from the
-// "suspended" state. Only the thread performing a stop-the-world pause may
-// transition a thread from the "suspended" state back to the "detached" state.
+// The "suspended" states are used to implement stop-the-world pauses and to
+// merge biased reference counts on behalf of detached threads. They are only
+// used in `--disable-gil` builds.
+// They are similar to the "detached" state in that the thread is not allowed
+// to call most Python APIs. A suspended thread trying to attach marks itself
+// as "suspended-waiting". Only the thread responsible for suspending it may
+// resume it, moving it to "detached" or "detached-waiting".
+// A "detached-waiting" thread must attach before it can be suspended again.
 //
 // The "shutting down" state is used when the interpreter is being finalized.
 // Threads in this state can't do anything other than block the OS thread.
 // (See _PyThreadState_HangThread).
 //
-// State transition diagram:
-//
-//            (bound thread)        (stop-the-world thread)
-// [attached]       <->       [detached]       <->       [suspended]
-//   |                                                        ^
-//   +---------------------------->---------------------------+
-//                          (bound thread)
-//
-// The (bound thread) and (stop-the-world thread) labels indicate which thread
-// is allowed to perform the transition.
-#define _Py_THREAD_DETACHED         0
-#define _Py_THREAD_ATTACHED         1
-#define _Py_THREAD_SUSPENDED        2
-#define _Py_THREAD_SHUTTING_DOWN    3
+// State transitions:
+//   Bound thread:          attached <-> detached
+//                         attached  -> suspended
+//                         suspended -> suspended-waiting
+//                         detached-waiting -> attached
+//   Suspending thread:     detached <-> suspended
+//                         suspended-waiting -> detached-waiting
+#define _Py_THREAD_DETACHED             0
+#define _Py_THREAD_ATTACHED             1
+#define _Py_THREAD_SUSPENDED            2
+#define _Py_THREAD_SHUTTING_DOWN        3
+#define _Py_THREAD_SUSPENDED_WAITING    4
+#define _Py_THREAD_DETACHED_WAITING     5
 
 
 /* Check if the current thread is the main thread.
@@ -162,8 +162,9 @@ extern void _PyThreadState_Suspend(PyThreadState *tstate);
 // Returns 1 on success, 0 if the thread was not in the "detached" state.
 extern int _PyThreadState_TrySuspendDetached(PyThreadState *tstate);
 
-// Undo a successful _PyThreadState_TrySuspendDetached(): switch the thread
-// back to "detached" and wake it if it is waiting to attach.
+// Resume a thread suspended by _PyThreadState_TrySuspendDetached() or a
+// stop-the-world pause: switch it back to "detached" or "detached-waiting"
+// and wake it if it is waiting to attach.
 extern void _PyThreadState_ResumeDetached(PyThreadState *tstate);
 #endif
 

Lib/test/pythoninfo.py

@@ -917,18 +917,17 @@ def collect_support_threading_helper(info_add):
     copy_attributes(info_add, threading_helper, 'support_threading_helper.%s', attributes)
 
 
-def collect_cc(info_add):
+def get_compiler_version(sysconfig_var):
     import sysconfig
-
-    CC = sysconfig.get_config_var('CC')
-    if not CC:
+    program = sysconfig.get_config_var(sysconfig_var)
+    if not program:
         return
 
     try:
         import shlex
-        args = shlex.split(CC)
+        args = shlex.split(program)
     except ImportError:
-        args = CC.split()
+        args = program.split()
     args.append('--version')
 
     stdout = run_command(args)
@@ -942,7 +941,21 @@ def collect_cc(info_add):
 
     text = first_line(stdout)
     text = normalize_text(text)
-    info_add('CC.version', text)
+    if text:
+        text = f'[{program}] {text}'
+    return text
+
+
+def collect_cc(info_add):
+    # C compiler
+    version = get_compiler_version('CC')
+    if version:
+        info_add('CC.version', version)
+
+    # C++ compiler
+    version = get_compiler_version('CXX')
+    if version:
+        info_add('CXX.version', version)
 
 
 def collect_gdbm(info_add):

Lib/test/test_cext/__init__.py

@@ -1,28 +1,37 @@
-# gh-116869: Build a basic C test extension to check that the Python C API
-# does not emit C compiler warnings.
+# gh-116869: Build a C/C++ test extension to check that the Python C API does
+# not emit compiler warnings.
 #
 # The Python C API must be compatible with building
 # with the -Werror=declaration-after-statement compiler flag.
 
 import os.path
+import platform
 import shlex
 import shutil
 import subprocess
-import sysconfig
 import sys
+import sysconfig
 import unittest
 from test import support
+from test.support import os_helper
+
+
+if not support.has_subprocess_support:
+    raise unittest.SkipTest("requires subprocess support")
 
 
+SOURCE_DIR = os.path.dirname(__file__)
 SOURCES = [
-    os.path.join(os.path.dirname(__file__), 'extension.c'),
+    os.path.join(SOURCE_DIR, 'extension.c'),
+    os.path.join(SOURCE_DIR, 'extension.cpp'),
+    os.path.join(SOURCE_DIR, 'setup.py'),
 ]
-SETUP = os.path.join(os.path.dirname(__file__), 'setup.py')
+MSVC = support.MS_WINDOWS
 
 
 # With MSVC on a debug build, the linker fails with: cannot open file
 # 'python311.lib', it should look 'python311_d.lib'.
-@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG,
+@unittest.skipIf(MSVC and support.Py_DEBUG,
                  'test fails on Windows debug build')
 # Building and running an extension in clang sanitizing mode is not
 # straightforward
@@ -34,41 +43,38 @@
 @support.requires_resource('cpu')
 class BaseTests:
     TEST_INTERNAL_C_API = False
-
-    # Default build with no options
-    def test_build(self):
-        self.check_build('_test_cext')
+    LANGUAGE = None
 
     def check_build(self, extension_name, std=None, limited=False,
-                    abi3t=False):
-        venv_dir = 'env'
-        with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
-            self._check_build(extension_name, python_exe,
-                              std=std, limited=limited,
-                              abi3t=abi3t)
-
-    def _check_build(self, extension_name, python_exe, std, limited,
-                     abi3t):
+                    abi3t=False, extra_cflags=None):
+        if self.LANGUAGE == 'C++' and not std and sys.platform == 'darwin':
+            # Old Apple clang++ default C++ std is gnu++98, use C++11 instead
+            std = 'c++11'
+
         pkg_dir = 'pkg'
         os.mkdir(pkg_dir)
-        shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP)))
+        self.addCleanup(os_helper.rmtree, pkg_dir)
+
         for source in SOURCES:
             dest = os.path.join(pkg_dir, os.path.basename(source))
             shutil.copy(source, dest)
 
         def run_cmd(operation, cmd):
             env = os.environ.copy()
+            env['CPYTHON_TEST_EXT_NAME'] = extension_name
+            env['CPYTHON_TEST_LANG'] = self.LANGUAGE
             if std:
                 env['CPYTHON_TEST_STD'] = std
             if limited:
                 env['CPYTHON_TEST_LIMITED'] = '1'
             if abi3t:
                 env['CPYTHON_TEST_ABI3T'] = '1'
-            if support.MS_WINDOWS and sysconfig.is_python_build():
-                env['CPYTHON_EXTRA_INCDIRS'] = os.path.split(sysconfig.get_config_h_filename())[0]
-                env['CPYTHON_EXTRA_LIBDIRS'] = os.path.split(sys.executable)[0]
-            env['CPYTHON_TEST_EXT_NAME'] = extension_name
-            env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
+            if MSVC and sysconfig.is_python_build():
+                env['CPYTHON_TEST_EXTRA_INCDIRS'] = os.path.split(sysconfig.get_config_h_filename())[0]
+                env['CPYTHON_TEST_EXTRA_LIBDIRS'] = os.path.split(sys.executable)[0]
+            env['CPYTHON_TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
+            if extra_cflags:
+                env['CPYTHON_TEST_EXTRA_CFLAGS'] = extra_cflags
             if support.verbose:
                 print('Run:', ' '.join(map(shlex.quote, cmd)))
                 subprocess.run(cmd, check=True, env=env)
@@ -85,6 +91,7 @@ def run_cmd(operation, cmd):
                         f"{operation} failed with exit code {proc.returncode}")
 
         # Build and install the C extension
+        python_exe = PYTHON_EXE
         cmd = [python_exe, '-X', 'dev',
                '-m', 'pip', 'install', '--no-build-isolation',
                os.path.abspath(pkg_dir)]
@@ -101,39 +108,109 @@ def run_cmd(operation, cmd):
                '-c', 'pass']
         run_cmd('Reference run', cmd)
 
-        # Import the C extension
+        # Import the C/C++ extension
         cmd = [python_exe,
                '-X', 'dev',
                '-X', 'showrefcount',
                '-c', f"import {extension_name}"]
         run_cmd('Import', cmd)
 
 
-class TestPublicCAPI(BaseTests, unittest.TestCase):
+class TestPublicC(BaseTests, unittest.TestCase):
+    LANGUAGE = 'C'
+
+    # Default build with no options
+    def test_build(self):
+        self.check_build('_test_cext')
+
+    @unittest.skipIf(MSVC, "MSVC doesn't support /std:c99")
+    def test_build_c99(self):
+        # In public docs, we say C API is compatible with C11. However,
+        # in practice we do maintain C99 compatibility in public headers.
+        # Please ask the C API WG before adding a new C11-only feature.
+        self.check_build('_test_cext_c99', std='c99')
+
+    def test_build_c11(self):
+        self.check_build('_test_cext_c11', std='c11')
+
     def test_build_limited(self):
-        self.check_build('_test_limited_cext', limited=True)
+        self.check_build('_test_cext_limited', limited=True)
 
     def test_build_limited_c11(self):
-        self.check_build('_test_limited_c11_cext', limited=True, std='c11')
+        self.check_build('_test_cext_limited_c11', limited=True, std='c11')
 
-    def test_build_c11(self):
-        self.check_build('_test_c11_cext', std='c11')
+    def test_build_abi3t(self):
+        # Test with Py_TARGET_ABI3T
+        self.check_build('_test_cext_abi3t', abi3t=True)
+
+
+class TestPublicCpp(BaseTests, unittest.TestCase):
+    LANGUAGE = 'C++'
+
+    def test_build(self):
+        self.check_build('_test_cppext')
+
+    def test_build_cpp03(self):
+        # In public docs, we say C API is compatible with C++11. However,
+        # in practice we do maintain C++03 compatibility in public headers.
+        # Please ask the C API WG before adding a new C++11-only feature.
+        self.check_build('_test_cppext_cpp03', std='c++03')
+
+    @unittest.skipIf(MSVC, "MSVC doesn't support /std:c++11")
+    def test_build_cpp11(self):
+        self.check_build('_test_cppext_cpp11', std='c++11')
+
+    # Only test C++14 on MSVC.
+    # On s390x RHEL7, GCC 4.8.5 doesn't support C++14.
+    @unittest.skipIf(not MSVC, "need MSVC")
+    def test_build_cpp14(self):
+        self.check_build('_test_cppext_cpp14', std='c++14')
+
+    # Test that headers compile with Intel asm syntax, which may conflict
+    # with inline assembly in free-threading headers that use AT&T syntax.
+    @unittest.skipIf(MSVC, "MSVC doesn't support -masm=intel")
+    @unittest.skipUnless(platform.machine() in ('x86_64', 'i686', 'AMD64'),
+                         "x86-specific flag")
+    def test_build_intel_asm(self):
+        self.check_build('_test_cppext_intel_asm', extra_cflags='-masm=intel')
+
+    def test_build_limited(self):
+        self.check_build('_test_cppext_limited', limited=True)
+
+    def test_build_limited_cpp03(self):
+        self.check_build('_test_cppext_limited_cpp03', std='c++03', limited=True)
 
     def test_build_abi3t(self):
         # Test with Py_TARGET_ABI3T
-        self.check_build('_test_abi3t', abi3t=True)
+        self.check_build('_test_cppext_abi3t', abi3t=True)
 
-    @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
-    def test_build_c99(self):
-        # In public docs, we say C API is compatible with C11. However,
-        # in practice we do maintain C99 compatibility in public headers.
-        # Please ask the C API WG before adding a new C11-only feature.
-        self.check_build('_test_c99_cext', std='c99')
+
+class TestInteralC(BaseTests, unittest.TestCase):
+    LANGUAGE = 'C'
+    TEST_INTERNAL_C_API = True
+
+    # Default build with no options
+    def test_build(self):
+        self.check_build('_test_cext_internal')
 
 
-class TestInteralCAPI(BaseTests, unittest.TestCase):
+class TestInteralCpp(BaseTests, unittest.TestCase):
+    LANGUAGE = 'C++'
     TEST_INTERNAL_C_API = True
 
+    def test_build(self):
+        self.check_build('_test_cppext_internal')
+
+
+def setUpModule():
+    global VENV_CONTEXT, PYTHON_EXE
+    VENV_CONTEXT = support.setup_venv_with_pip_setuptools('env')
+    PYTHON_EXE = VENV_CONTEXT.__enter__()
+
+
+def tearDownModule():
+    VENV_CONTEXT.__exit__(None, None, None)
+
 
 if __name__ == "__main__":
     unittest.main()