diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 6f832b7a34b10b9..bd4125cacdd0e66 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -15,8 +15,7 @@ Before Python initialization In an application embedding Python, the :c:func:`Py_Initialize` function must be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. +a few functions. The following functions can be safely called before Python is initialized: @@ -75,277 +74,6 @@ The following functions can be safely called before Python is initialized: been initialized: :c:func:`Py_EncodeLocale`, and :c:func:`Py_RunMain`. -.. _global-conf-vars: - -Global configuration variables ------------------------------- - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - - -.. c:var:: int Py_BytesWarningFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python - Initialization Configuration `. - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_DebugFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python - Initialization Configuration `. - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_DontWriteBytecodeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python - Initialization Configuration `. - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_FrozenFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.pathconfig_warnings` should be used instead, see - :ref:`Python Initialization Configuration `. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_HashRandomizationFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should - be used instead, see :ref:`Python Initialization Configuration - `. - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_IgnoreEnvironmentFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.use_environment` should be used instead, see - :ref:`Python Initialization Configuration `. - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_InspectFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.inspect` should be used instead, see - :ref:`Python Initialization Configuration `. - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_InteractiveFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.interactive` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-i` option. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_IsolatedFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.isolated` should be used instead, see - :ref:`Python Initialization Configuration `. - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - This API is kept for backward compatibility: setting - :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_LegacyWindowsStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_NoSiteFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.site_import` should be used instead, see - :ref:`Python Initialization Configuration `. - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_NoUserSiteDirectory - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.user_site_directory` should be used instead, see - :ref:`Python Initialization Configuration `. - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_OptimizeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.optimization_level` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_QuietFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python - Initialization Configuration `. - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_UnbufferedStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python - Initialization Configuration `. - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - - .. deprecated-removed:: 3.12 3.16 - - -.. c:var:: int Py_VerboseFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python - Initialization Configuration `. - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - .. deprecated-removed:: 3.12 3.16 - - Initializing and finalizing the interpreter ------------------------------------------- diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index e4664ef815f567c..cf04e4c1ed63a10 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -128,9 +128,8 @@ Module Objects The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute is reassigned or the module is destroyed. - .. deprecated:: 3.2 - :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on - unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. + :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on + unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. .. _c_module_slots: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 662158323c5d305..82447cc98cd8d03 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -951,8 +951,6 @@ func,Py_EndInterpreter,3.2,, func,Py_EnterRecursiveCall,3.9,, func,Py_Exit,3.2,, func,Py_FatalError,3.2,, -data,Py_FileSystemDefaultEncodeErrors,3.10,, -data,Py_FileSystemDefaultEncoding,3.2,, func,Py_Finalize,3.2,, func,Py_FinalizeEx,3.6,, func,Py_GenericAlias,3.9,, @@ -965,7 +963,6 @@ func,Py_GetCopyright,3.2,, func,Py_GetPlatform,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, -data,Py_HasFileSystemDefaultEncoding,3.2,, func,Py_HashBuffer,3.16,, func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, @@ -1020,7 +1017,6 @@ macro,Py_T_ULONGLONG,3.12,, macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, -data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst index fe2d91cf316b18f..dd840fac9b151b3 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.16.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -17,56 +17,56 @@ Pending removal in Python 3.16 * Global configuration variables: - * :c:var:`Py_DebugFlag`: + * :c:var:`!Py_DebugFlag`: Use :c:member:`PyConfig.parser_debug` or :c:func:`PyConfig_Get("parser_debug") ` instead. - * :c:var:`Py_VerboseFlag`: + * :c:var:`!Py_VerboseFlag`: Use :c:member:`PyConfig.verbose` or :c:func:`PyConfig_Get("verbose") ` instead. - * :c:var:`Py_QuietFlag`: + * :c:var:`!Py_QuietFlag`: Use :c:member:`PyConfig.quiet` or :c:func:`PyConfig_Get("quiet") ` instead. - * :c:var:`Py_InteractiveFlag`: + * :c:var:`!Py_InteractiveFlag`: Use :c:member:`PyConfig.interactive` or :c:func:`PyConfig_Get("interactive") ` instead. - * :c:var:`Py_InspectFlag`: + * :c:var:`!Py_InspectFlag`: Use :c:member:`PyConfig.inspect` or :c:func:`PyConfig_Get("inspect") ` instead. - * :c:var:`Py_OptimizeFlag`: + * :c:var:`!Py_OptimizeFlag`: Use :c:member:`PyConfig.optimization_level` or :c:func:`PyConfig_Get("optimization_level") ` instead. - * :c:var:`Py_NoSiteFlag`: + * :c:var:`!Py_NoSiteFlag`: Use :c:member:`PyConfig.site_import` or :c:func:`PyConfig_Get("site_import") ` instead. - * :c:var:`Py_BytesWarningFlag`: + * :c:var:`!Py_BytesWarningFlag`: Use :c:member:`PyConfig.bytes_warning` or :c:func:`PyConfig_Get("bytes_warning") ` instead. - * :c:var:`Py_FrozenFlag`: + * :c:var:`!Py_FrozenFlag`: Use :c:member:`PyConfig.pathconfig_warnings` or :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: + * :c:var:`!Py_IgnoreEnvironmentFlag`: Use :c:member:`PyConfig.use_environment` or :c:func:`PyConfig_Get("use_environment") ` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: + * :c:var:`!Py_DontWriteBytecodeFlag`: Use :c:member:`PyConfig.write_bytecode` or :c:func:`PyConfig_Get("write_bytecode") ` instead. - * :c:var:`Py_NoUserSiteDirectory`: + * :c:var:`!Py_NoUserSiteDirectory`: Use :c:member:`PyConfig.user_site_directory` or :c:func:`PyConfig_Get("user_site_directory") ` instead. - * :c:var:`Py_UnbufferedStdioFlag`: + * :c:var:`!Py_UnbufferedStdioFlag`: Use :c:member:`PyConfig.buffered_stdio` or :c:func:`PyConfig_Get("buffered_stdio") ` instead. - * :c:var:`Py_HashRandomizationFlag`: + * :c:var:`!Py_HashRandomizationFlag`: Use :c:member:`PyConfig.use_hash_seed` and :c:member:`PyConfig.hash_seed` or :c:func:`PyConfig_Get("hash_seed") ` instead. - * :c:var:`Py_IsolatedFlag`: + * :c:var:`!Py_IsolatedFlag`: Use :c:member:`PyConfig.isolated` or :c:func:`PyConfig_Get("isolated") ` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + * :c:var:`!Py_LegacyWindowsFSEncodingFlag`: Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: + * :c:var:`!Py_LegacyWindowsStdioFlag`: Use :c:member:`PyConfig.legacy_windows_stdio` or :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst index 841d1b455b6bec6..273d85bd9332455 100644 --- a/Doc/deprecations/c-api-pending-removal-in-future.rst +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -12,8 +12,6 @@ although there is currently no date scheduled for their removal. Use :c:func:`PyErr_GetRaisedException` instead. * :c:func:`PyErr_Restore`: Use :c:func:`PyErr_SetRaisedException` instead. -* :c:func:`PyModule_GetFilename`: - Use :c:func:`PyModule_GetFilenameObject` instead. * :c:func:`PyOS_AfterFork`: Use :c:func:`PyOS_AfterFork_Child` instead. * :c:func:`PySlice_GetIndicesEx`: diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index 1d5cf24d0628432..1a8e8593e8a60a9 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -6,7 +6,8 @@ Remote debugging attachment protocol This protocol enables external tools to attach to a running CPython process and execute Python code remotely. -Most platforms require elevated privileges to attach to another Python process. +Attaching to another Python process may require additional permissions or +configuration, depending on the platform. Disabling remote debugging -------------------------- @@ -23,44 +24,91 @@ To disable remote debugging support, use any of the following: Permission requirements ======================= -Attaching to a running Python process for remote debugging requires elevated -privileges on most platforms. The specific requirements and troubleshooting +Attaching to a running Python process for remote debugging requires special +configuration on most platforms. The specific requirements and troubleshooting steps depend on your operating system: .. rubric:: Linux -The tracer process must have the ``CAP_SYS_PTRACE`` capability or equivalent -privileges. You can only trace processes you own and can signal. Tracing may -fail if the process is already being traced, or if it is running with -set-user-ID or set-group-ID. Security modules like Yama may further restrict -tracing. +In general, you can debug your own processes, but there are several common +configurations that may disable this. Some Linux distributions enable **ptrace +restrictions**, aka "Yama," as a form of system hardening. Recent versions of +the ``setpriv`` command (util-linux 2.41, released June 2025) let you loosen +ptrace restrictions on a per-process basis: -To temporarily relax ptrace restrictions (until reboot), run: + ``setpriv --ptracer any python3`` + +(This is configured on the process *being debugged*.) You can also turn off +ptrace restrictions for all processes until reboot with: ``echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope`` +This can also be configured persistently, usually in ``/etc/sysctl.d``. + .. note:: Disabling ``ptrace_scope`` reduces system hardening and should only be done - in trusted environments. - -If running inside a container, use ``--cap-add=SYS_PTRACE`` or -``--privileged``, and run as root if needed. + in low-security environments. + +It is also possible that the ``ptrace`` system call is disabled because of a +security filter. In particular, this was common with older versions of some +container software. Docker 19.03 or newer (released 2019) and containerd 1.6.7 +or newer (released 2022) will automatically allow usage of the ``ptrace`` +system call inside containers, when running on Linux kernel 4.8 or higher. If +you cannot upgrade to these versions, you can create your container with an +option like ``--security-opt seccomp=unconfined`` to disable the system call +security filter for that container. This weakens the container's isolation and +should only be done in low-security environments. + +If you need to trace a process that you *do not* own, you will need superuser +access or equivalent. This also applies to processes that have changed their +security credentials, e.g., set-user-ID or set-group-ID processes (though this +is unusual for Python). Try running the debugging command with ``sudo -E``. -Try re-running the command with elevated privileges: +.. note:: - ``sudo -E !!`` + The ``CAP_SYS_PTRACE`` capability is equivalent to superuser access, in + that it allows debugging *any* process, not just your own. You may see + advice on the internet suggesting using it to work around ptrace + restrictions or system call filters. This may work in practice, as would + ``sudo``, but this gives the debugging process much more access than it + needs and should only be done in low-security environments. +Finally, note that a process can only have one tracer at a time. If you have +already attached to a Python process under ``strace``, ``gdb``, etc., you +won't be able to simultaneously use remote debugging. (Superuser access cannot +get around this restriction.) .. rubric:: macOS -To attach to another process, you typically need to run your debugging tool -with elevated privileges. This can be done by using ``sudo`` or running as -root. +By default, macOS disables the ability to debug other processes. + +You can modify your Python binary to opt in to being debugged by giving it an +**ad-hoc code signature** with an **entitlement** enabling it to be debugged. +(An ad-hoc "signature" is just a configuration without any actual cryptographic +signature or a need for a certificate or anything else such as an Apple +developer program membership.) + +The following commands will create a file ``get-task-allow.plist`` with the +necessary entitlement and add it to the Python binary: + +.. code-block:: sh + + echo '{"com.apple.security.get-task-allow": true}' | plutil -convert xml1 -o get-task-allow.plist - + codesign --sign - --entitlements get-task-allow.plist path/to/bin/python3 + +where ``path/to/bin/python3`` is the path to your Python binary, which you can +find by e.g. running ``which python3`` or evaluating ``sys.base_executable`` at +the Python REPL. (These instructions are for a non-framework build of Python. +Framework builds may need to be configured differently.) -Even when attaching to processes you own, macOS may block debugging unless -the debugger is run with root privileges due to system security restrictions. +You should then be able to debug your own Python processes started with that +binary. +Alternatively, much as with Linux, processes with superuser privileges e.g. ``sudo`` +are not subject to this check and can debug any user's process on the system +(though there are additional checks on specific binaries, such as OS-provided +commands, due to System Integrity Protection). .. rubric:: Windows diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst index 262e73f2eaa09fb..d37f18227f73725 100644 --- a/Doc/includes/optional-module.rst +++ b/Doc/includes/optional-module.rst @@ -7,3 +7,4 @@ If you are the distributor, see :ref:`optional-module-requirements`. .. Similar notes appear in the docs of the modules: - zipfile - tarfile + - turtle diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index f28c4230f3d6b89..8affcc9ef79defd 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -24,25 +24,6 @@ -------------- -Introduction -============ - -Turtle graphics is an implementation of `the popular geometric drawing tools -introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon -in 1967. - -.. include:: ../includes/optional-module.rst - - -Get started -=========== - -Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the -command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the -direction it is facing, drawing a line as it moves. Give it the command -``turtle.right(25)``, and it rotates in-place 25 degrees clockwise. - .. sidebar:: Turtle star Turtle can draw intricate shapes using programs that repeat simple @@ -51,21 +32,40 @@ direction it is facing, drawing a line as it moves. Give it the command .. image:: turtle-star.png :align: center -In Python, turtle graphics provides a representation of a physical "turtle" -(a little robot with a pen) that draws on a sheet of paper on the floor. +Imagine a robotic turtle starting at (0, 0) in the x-y plane. +After an ``import turtle``, give it the command ``turtle.forward(15)``, and it +moves (on-screen!) 15 pixels in the direction it is facing, drawing a line as +it moves. Give it the command ``turtle.right(25)``, and it rotates in-place 25 +degrees clockwise. + +Turtle graphics is an implementation of `the drawing tools introduced in Logo +`_ in 1967. It was created as an +educational tool, and its instant, visible feedback makes it an effective way +for learners to encounter programming concepts. It is also a convenient way to +produce simple graphical output without bringing in external libraries. + +This document includes four main sections: -It's an effective and well-proven way for learners to encounter -programming concepts and interaction with software, as it provides instant, -visible feedback. It also provides convenient access to graphical output -in general. +* :ref:`turtle-tutorial` teaches the basics of turtle drawing. +* :ref:`turtle-reference` describes the functions, methods and classes this + module defines. +* :ref:`turtle-howtos` details how to handle specific tasks. +* :ref:`turtle-explanation` provides background on the object-oriented + interface. + +.. note:: -Turtle drawing was originally created as an educational tool, to be used by -teachers in the classroom. For the programmer who needs to produce some -graphical output it can be a way to do that without the overhead of -introducing more complex or external libraries into their work. + Turtle graphics requires the :mod:`tkinter` :term:`optional module`. + The python.org installers for Windows and macOS include it, but some + Linux distributions and other platforms may package it separately. If + ``import turtle`` fails with an error mentioning ``_tkinter``, look for + documentation from your distributor (that is, whoever provided Python to you). + Check this in advance if you're planning to use turtle graphics with a learner. .. _turtle-tutorial: +.. _get-started: +.. _get-started-as-quickly-as-possible: Tutorial ======== @@ -108,7 +108,8 @@ Notice how the turtle, represented by an arrow, points in different directions as you steer it. Experiment with those commands, and also with ``backward()`` and -``right()``. +``right()``. Many commands also have terser aliases, such as ``fd()`` for +:func:`forward`. Pen control @@ -188,38 +189,16 @@ Finally, complete the filling:: ``end_fill()`` command.) +.. _turtle-howtos: .. _turtle-how-to: +.. _how-to: -How to... -========= +How-to guides +============= This section covers some typical turtle use-cases and approaches. -Get started as quickly as possible ----------------------------------- - -One of the joys of turtle graphics is the immediate, visual feedback that's -available from simple commands - it's an excellent way to introduce children -to programming ideas, with a minimum of overhead (not just children, of -course). - -The turtle module makes this possible by exposing all its basic functionality -as functions, available with ``from turtle import *``. The :ref:`turtle -graphics tutorial ` covers this approach. - -It's worth noting that many of the turtle commands also have even more terse -equivalents, such as ``fd()`` for :func:`forward`. These are especially -useful when working with learners for whom typing is not a skill. - -.. _note: - - You'll need to have the :mod:`Tk interface package ` installed on - your system for turtle graphics to work. Be warned that this is not - always straightforward, so check this in advance if you're planning to - use turtle graphics with a learner. - - Automatically begin and end filling ----------------------------------- @@ -323,8 +302,11 @@ The turtle's screen can be customised, for example:: t.screen.bgcolor("orange") -Turtle graphics reference -========================= +.. _turtle-reference: +.. _turtle-graphics-reference: + +Reference +========= .. note:: diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index 3f7906ee84ec148..325cf006d0507e4 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -5,8 +5,27 @@ c-api/allocation.html: deprecated-aliases c-api/file.html: deprecated-api c-api/interp-lifecycle.html: c.PySys_SetArgv c-api/interp-lifecycle.html: c.PySys_SetArgvEx +c-api/interp-lifecycle.html: c.Py_BytesWarningFlag +c-api/interp-lifecycle.html: c.Py_DebugFlag +c-api/interp-lifecycle.html: c.Py_DontWriteBytecodeFlag +c-api/interp-lifecycle.html: c.Py_FrozenFlag +c-api/interp-lifecycle.html: c.Py_HashRandomizationFlag +c-api/interp-lifecycle.html: c.Py_IgnoreEnvironmentFlag +c-api/interp-lifecycle.html: c.Py_InspectFlag +c-api/interp-lifecycle.html: c.Py_InteractiveFlag +c-api/interp-lifecycle.html: c.Py_IsolatedFlag +c-api/interp-lifecycle.html: c.Py_LegacyWindowsFSEncodingFlag +c-api/interp-lifecycle.html: c.Py_LegacyWindowsStdioFlag +c-api/interp-lifecycle.html: c.Py_NoSiteFlag +c-api/interp-lifecycle.html: c.Py_NoUserSiteDirectory +c-api/interp-lifecycle.html: c.Py_OptimizeFlag +c-api/interp-lifecycle.html: c.Py_QuietFlag c-api/interp-lifecycle.html: c.Py_SetProgramName c-api/interp-lifecycle.html: c.Py_SetPythonHome +c-api/interp-lifecycle.html: c.Py_UnbufferedStdioFlag +c-api/interp-lifecycle.html: c.Py_VerboseFlag +c-api/interp-lifecycle.html: global-conf-vars +c-api/interp-lifecycle.html: global-configuration-variables c-api/threads.html: c.PyEval_InitThreads # Removed sections @@ -91,3 +110,7 @@ reference/expressions.html: generator.throw # Obsolete sections in 'turtle' docs library/turtle.html: changes-since-python-2-6 library/turtle.html: changes-since-python-3-0 + +# 'turtle' documentation reorganisation (gh-157847) +library/turtle.html: note +library/turtle.html: introduction diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 9e48b3ab7393c87..78ddc627fcf4d70 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -2170,24 +2170,24 @@ Deprecated * Deprecate global configuration variable: - * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` - * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` - * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` - * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` - * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` - * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` - * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` - * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` - * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` - * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` - * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` - * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` - * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` - * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` + * :c:var:`!Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` + * :c:var:`!Py_VerboseFlag`: use :c:member:`PyConfig.verbose` + * :c:var:`!Py_QuietFlag`: use :c:member:`PyConfig.quiet` + * :c:var:`!Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` + * :c:var:`!Py_InspectFlag`: use :c:member:`PyConfig.inspect` + * :c:var:`!Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` + * :c:var:`!Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` + * :c:var:`!Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` + * :c:var:`!Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` + * :c:var:`!Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` + * :c:var:`!Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` + * :c:var:`!Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` + * :c:var:`!Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` + * :c:var:`!Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` and :c:member:`PyConfig.hash_seed` - * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` - * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` + * :c:var:`!Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` + * :c:var:`!Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` + * :c:var:`!Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index c76681261f6746d..3d3125ad17b126b 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1128,6 +1128,10 @@ Deprecated C APIs and :c:func:`PyAsyncGen_New` are deprecated. They are scheduled for removal in 3.18. +* :c:func:`PyModule_GetFilename` is no longer deprecated, but using + :c:func:`PyModule_GetFilenameObject` instead is still recommended. + (Contributed by Victor Stinner in :gh:`154757`.) + .. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst @@ -1141,6 +1145,37 @@ Deprecated C APIs Removed C APIs -------------- +* Remove 21 global configuration variables, deprecated in Python 3.12. + To initialize Python, the :ref:`PyInitConfig C API ` can be + used instead. + To read these variables at runtime, :c:func:`PyConfig_Get` can be used + instead. + Removed variables with their :ref:`PyInitConfig C API ` and + :c:func:`PyConfig_Get` replacement: + + * :c:var:`!Py_DebugFlag`: Use ``"parser_debug"`` instead. + * :c:var:`!Py_VerboseFlag`: Use ``"verbose"`` instead. + * :c:var:`!Py_QuietFlag`: Use ``"quiet"`` instead. + * :c:var:`!Py_InteractiveFlag`: Use ``"interactive"`` instead. + * :c:var:`!Py_InspectFlag`: Use ``"inspect"`` instead. + * :c:var:`!Py_OptimizeFlag`: Use ``"optimization_level"`` instead. + * :c:var:`!Py_NoSiteFlag`: Use ``"site_import"`` instead (opposite value). + * :c:var:`!Py_BytesWarningFlag`: Use ``"bytes_warning"`` instead. + * :c:var:`!Py_FrozenFlag`: Use ``"pathconfig_warnings"`` instead (opposite value). + * :c:var:`!Py_IgnoreEnvironmentFlag`: Use ``"use_environment"`` instead (opposite value). + * :c:var:`!Py_DontWriteBytecodeFlag`: Use ``"write_bytecode"`` instead (opposite value). + * :c:var:`!Py_NoUserSiteDirectory`: Use ``"user_site_directory"`` instead (opposite value). + * :c:var:`!Py_UnbufferedStdioFlag`: Use ``"buffered_stdio"`` instead (opposite value). + * :c:var:`!Py_HashRandomizationFlag`: Use ``"hash_seed"`` instead. + * :c:var:`!Py_IsolatedFlag`: Use ``"isolated"`` instead. + * :c:var:`!Py_LegacyWindowsFSEncodingFlag`: Use ``"legacy_windows_fs_encoding"`` instead. + * :c:var:`!Py_LegacyWindowsStdioFlag`: Use ``"legacy_windows_stdio"`` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: Use ``"filesystem_encoding"`` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: Use ``"filesystem_errors"`` instead. + * :c:var:`!Py_UTF8Mode`: Use ``"utf8_mode"`` instead. + + (Contributed by Victor Stinner in :gh:`158001`.) + * Remove :c:func:`!PyEval_InitThreads` function which did nothing since Python 3.7 and was deprecated since Python 3.9. (Contributed by Victor Stinner in :gh:`154757`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 3af3e6ec9cac08f..6900c97e9546c9a 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2524,7 +2524,7 @@ number of other issues). Some known details affected: over the default filters set by the interpreter Due to changes in the way the default warnings filters are configured, -setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer +setting :c:data:`!Py_BytesWarningFlag` to a value greater than one is no longer sufficient to both emit :exc:`BytesWarning` messages and have them converted to exceptions. Instead, the flag must be set (to cause the warnings to be emitted in the first place), and an explicit ``error::BytesWarning`` @@ -2547,7 +2547,7 @@ Starting in 3.7.1, :c:func:`Py_Initialize` now consistently reads and respects all of the same environment settings as :c:func:`Py_Main` (in earlier Python versions, it respected an ill-defined subset of those environment variables, while in Python 3.7.0 it didn't read any of them due to :issue:`34247`). If -this behavior is unwanted, set :c:data:`Py_IgnoreEnvironmentFlag` to 1 before +this behavior is unwanted, set :c:data:`!Py_IgnoreEnvironmentFlag` to 1 before calling :c:func:`Py_Initialize`. In 3.7.1 the C API for Context Variables diff --git a/Include/cpython/pydebug.h b/Include/cpython/pydebug.h index f6ebd99ed7e2ff2..d27f992f4601099 100644 --- a/Include/cpython/pydebug.h +++ b/Include/cpython/pydebug.h @@ -5,27 +5,6 @@ extern "C" { #endif -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_DebugFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_VerboseFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_QuietFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_InteractiveFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_InspectFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_OptimizeFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_NoSiteFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_BytesWarningFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_FrozenFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_DontWriteBytecodeFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_NoUserSiteDirectory; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UnbufferedStdioFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_HashRandomizationFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_IsolatedFlag; - -#ifdef MS_WINDOWS -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_LegacyWindowsFSEncodingFlag; -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; -#endif - /* this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like PYTHONPATH and PYTHONHOME from the environment */ diff --git a/Include/fileobject.h b/Include/fileobject.h index 6a6d11409497fab..464bc3b29f483ad 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -16,19 +16,6 @@ PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int); PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *); PyAPI_FUNC(int) PyObject_AsFileDescriptor(PyObject *); -/* The default encoding used by the platform file system APIs - If non-NULL, this is different than the default encoding for strings -*/ -Py_DEPRECATED(3.12) PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 -Py_DEPRECATED(3.12) PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; -#endif -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; - -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 -Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UTF8Mode; -#endif - #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEOBJECT_H # include "cpython/fileobject.h" diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 83cdc4f3dfdd44a..128790823aa8794 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -190,8 +190,6 @@ extern int _Py_open_osfhandle(void *handle, int flags); ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY() -extern int _Py_HasFileSystemDefaultEncodeErrors; - extern int _Py_DecodeUTF8Ex( const char *arg, Py_ssize_t arglen, diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 88c66672ff164a8..3e12839ec509811 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -25,7 +25,7 @@ PyAPI_FUNC(PyObject *) PyModule_GetDict(PyObject *); PyAPI_FUNC(PyObject *) PyModule_GetNameObject(PyObject *); #endif PyAPI_FUNC(const char *) PyModule_GetName(PyObject *); -Py_DEPRECATED(3.2) PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *); +PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *); PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *); PyAPI_FUNC(PyModuleDef*) PyModule_GetDef(PyObject*); PyAPI_FUNC(void*) PyModule_GetState(PyObject*); diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 0749a56d86fa4a6..3e15a3401356697 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -486,6 +486,7 @@ def prepare(self) -> None: raw.cc[termios.VMIN] = b"\x01" raw.cc[termios.VTIME] = b"\x00" self.__input_fd_set(raw) + self.__rawtermstate = raw # Apple Terminal will re-wrap lines for us unless we preempt the # damage. @@ -726,7 +727,19 @@ def input_hook(self): # avoid inline imports here so the repl doesn't get flooded # with import logging from -X importtime=2 if posix is not None and posix._is_inputhook_installed(): - return posix._inputhook + return self.__run_input_hook + + def __run_input_hook(self): + # gh-152907: input hooks expect cooked output, but pyrepl runs with + # OPOST disabled. Restore the saved output flags around the hook + # (only oflag; input must stay raw at the prompt). + cooked = self.__rawtermstate.copy() + cooked.oflag = self.__svtermstate.oflag + self.__input_fd_set(cooked) + try: + return posix._inputhook() + finally: + self.__input_fd_set(self.__rawtermstate) def __enable_bracketed_paste(self) -> None: os.write(self.output_fd, b"\x1b[?2004h") diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 7132cffd58f094a..0f31d93582b83e7 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -1,7 +1,7 @@ import collections import marshal import pstats -lazy from _colorize import ANSIColors +lazy from _colorize import get_colors from .collector import Collector, extract_lineno from .constants import MICROSECONDS_PER_SECOND, PROFILING_MODE_CPU @@ -178,6 +178,8 @@ def print_stats(self, sort=-1, limit=None, show_summary=True, mode=None): } # Print header with colors and proper alignment + ANSIColors = get_colors() + print(f"{ANSIColors.BOLD_BLUE}Profile Stats:{ANSIColors.RESET}") header_nsamples = f"{ANSIColors.BOLD_BLUE}{'nsamples':>{col_widths['nsamples']}}{ANSIColors.RESET}" @@ -269,6 +271,8 @@ def _determine_best_unit(max_value): def _print_summary(self, stats_list, total_samples): """Print summary of interesting functions.""" + ANSIColors = get_colors() + print( f"\n{ANSIColors.BOLD_BLUE}Summary of Interesting Functions:{ANSIColors.RESET}" ) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 6cb6dd483520887..1dff6529ade0408 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -6,7 +6,7 @@ import sysconfig import time from collections import deque -lazy from _colorize import ANSIColors +lazy from _colorize import get_colors from .binary_collector import BinaryCollector @@ -272,6 +272,8 @@ def _print_realtime_stats(self): ) # Max time = Min Hz # Build cache stats string if stats collection is enabled + ANSIColors = get_colors() + cache_stats_str = "" if self.collect_stats: try: @@ -305,6 +307,8 @@ def _print_unwinder_stats(self): except RuntimeError: return # Stats not enabled + ANSIColors = get_colors() + print(f"\n{ANSIColors.BOLD_BLUE}{'='*50}{ANSIColors.RESET}") print(f"{ANSIColors.BOLD_BLUE}Unwinder Statistics:{ANSIColors.RESET}") @@ -399,6 +403,8 @@ def _print_binary_stats(self, collector): except (ValueError, RuntimeError): return # Collector closed or stats unavailable + ANSIColors = get_colors() + print(f" {ANSIColors.CYAN}Binary Encoding:{ANSIColors.RESET}") repeat_records = stats.get('repeat_records', 0) diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 290126343618381..789f652b004bffb 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -379,46 +379,6 @@ def expect_bool_not(value): finally: config_set(name, old_value) - def test_config_set_global_vars(self): - # Test PyConfig_Set() with global configuration variables - config_get = _testcapi.config_get - config_set = _testcapi.config_set - get_configs = _testinternalcapi.get_configs - new_values = (0, 1, 5) - - for name, global_name, not_value in ( - ('bytes_warning', 'Py_BytesWarningFlag', False), - ('inspect', 'Py_InspectFlag', False), - ('interactive', 'Py_InteractiveFlag', False), - ('optimization_level', 'Py_OptimizeFlag', False), - ('parser_debug', 'Py_DebugFlag', False), - ('quiet', 'Py_QuietFlag', False), - ('use_environment', 'Py_IgnoreEnvironmentFlag', True), - ('verbose', 'Py_VerboseFlag', False), - ('write_bytecode', 'Py_DontWriteBytecodeFlag', True), - - # Read-only variables - #('buffered_stdio', 'Py_UnbufferedStdioFlag', True), - #('isolated', 'Py_IsolatedFlag', False), - #('pathconfig_warnings', 'Py_FrozenFlag', True), - #('site_import', 'Py_NoSiteFlag', True), - #('user_site_directory', 'Py_NoUserSiteDirectory', True), - # Windows only - #('legacy_windows_stdio', 'Py_LegacyWindowsStdioFlag', False) - ): - with self.subTest(name=name): - old_value = config_get(name) - try: - for value in new_values: - config_set(name, value) - global_config = get_configs()['global_config'] - expected = value - if not_value: - expected = int(not value) - self.assertEqual(global_config[global_name], expected) - finally: - config_set(name, old_value) - def test_config_set_cpu_count(self): config_get = _testcapi.config_get config_set = _testcapi.config_set diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py index eb04f6c0136022d..57d0a3c2f2e99e8 100644 --- a/Lib/test/test_capi/test_tokenizer.py +++ b/Lib/test/test_capi/test_tokenizer.py @@ -12,9 +12,6 @@ def test_source(self): def test_source_discard(self): _testinternalcapi.test_tokenizer_source_discard() - def test_cursor(self): - _testinternalcapi.test_tokenizer_cursor() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index d57452602ce5574..c75c0f8627bf084 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -113,6 +113,17 @@ def test_valid(self, compiler): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") + @subTests('symbol', ('single', 'exec')) + @subTests('prefix', ('', 'f', 't')) + def test_incomplete_string_diagnostics(self, symbol, prefix): + opening = f' รก = {prefix}"""first\n' + source = 'if True:\n' + opening + 'second' + with self.assertRaises(_IncompleteInputError) as cm: + Compile()(source, '', symbol) + text = opening + 'second' + ('\n' if symbol == 'exec' else '') + self.assertEqual(cm.exception.args, ( + 'incomplete input', ('', 2, 9, text, 2, -1))) + @subTests('compiler', COMPILERS) def test_incomplete(self, compiler): ai = functools.partial(self.assertIncomplete, compiler=compiler) diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index 82c928c9f6406c2..0d62702c0c0fd35 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -73,12 +73,12 @@ def test_undefined(self): class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" - def test_optimizeflag(self): - # This test accesses the Py_OptimizeFlag integer, which is - # exported by the Python dll and should match the sys.flags value + def test_version_var(self): + # This test accesses the Py_Version integer, which is + # exported by the Python dll and should match the sys.hexversion value - opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value - self.assertEqual(opt, sys.flags.optimize) + version = c_int.in_dll(pythonapi, "Py_Version").value + self.assertEqual(version, sys.hexversion) @thread_unsafe('overrides frozen modules') def test_frozentable(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 7ac22455541fe2c..a6ebda5bf4e5df7 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3405,7 +3405,7 @@ def test_descrdoc(self): from _io import FileIO def check(descr, what): self.assertEqual(descr.__doc__, what) - check(FileIO.closed, "True if the file is closed") # getset descriptor + check(FileIO.closed, "True if the file is closed.") # getset descriptor check(complex.real, "the real part of a complex number") # member descriptor def test_doc_descriptor(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 81241ea1f337336..77b231323cc2757 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -823,7 +823,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # global config DEFAULT_GLOBAL_CONFIG = { 'Py_HasFileSystemDefaultEncoding': 0, - 'Py_HashRandomizationFlag': 1, '_Py_HasFileSystemDefaultEncodeErrors': 0, } COPY_GLOBAL_PRE_CONFIG = [ @@ -831,31 +830,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ] COPY_GLOBAL_CONFIG = [ # Copy core config to global config for expected values - # True means that the core config value is inverted (0 => 1 and 1 => 0) - ('Py_BytesWarningFlag', 'bytes_warning'), - ('Py_DebugFlag', 'parser_debug'), - ('Py_DontWriteBytecodeFlag', 'write_bytecode', True), ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), - ('Py_FrozenFlag', 'pathconfig_warnings', True), - ('Py_IgnoreEnvironmentFlag', 'use_environment', True), - ('Py_InspectFlag', 'inspect'), - ('Py_InteractiveFlag', 'interactive'), - ('Py_IsolatedFlag', 'isolated'), - ('Py_NoSiteFlag', 'site_import', True), - ('Py_NoUserSiteDirectory', 'user_site_directory', True), - ('Py_OptimizeFlag', 'optimization_level'), - ('Py_QuietFlag', 'quiet'), - ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), - ('Py_VerboseFlag', 'verbose'), ] - if MS_WINDOWS: - COPY_GLOBAL_PRE_CONFIG.extend(( - ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), - )) - COPY_GLOBAL_CONFIG.extend(( - ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), - )) EXPECTED_CONFIG = None @@ -1013,20 +990,10 @@ def check_global_config(self, configs): config = configs['config'] expected = dict(self.DEFAULT_GLOBAL_CONFIG) - for item in self.COPY_GLOBAL_CONFIG: - if len(item) == 3: - global_key, core_key, opposite = item - expected[global_key] = 0 if config[core_key] else 1 - else: - global_key, core_key = item - expected[global_key] = config[core_key] - for item in self.COPY_GLOBAL_PRE_CONFIG: - if len(item) == 3: - global_key, core_key, opposite = item - expected[global_key] = 0 if pre_config[core_key] else 1 - else: - global_key, core_key = item - expected[global_key] = pre_config[core_key] + for global_key, core_key in self.COPY_GLOBAL_CONFIG: + expected[global_key] = config[core_key] + for global_key, core_key in self.COPY_GLOBAL_PRE_CONFIG: + expected[global_key] = pre_config[core_key] self.assertEqual(configs['global_config'], expected) diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index acd8d76dc90a293..5243d24a9ecf266 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -3,7 +3,7 @@ import __future__ import ast import unittest -from test.support import force_not_colorized, import_helper +from test.support import force_not_colorized, import_helper, subTests from test.support.script_helper import spawn_python, kill_python from textwrap import dedent import os @@ -87,8 +87,44 @@ def test_unknown_future_flag(self): from __future__ import rested_snopes # typo error here: nested => rested """ self.assertSyntaxError( - code, lineno=2, - message='future feature rested_snopes is not defined', offset=24, + code, + lineno=2, + message=( + "future feature 'rested_snopes' is not defined. " + "Did you mean: 'nested_scopes'?" + ), + offset=24, + ) + + @subTests("typo, origin", [ + ("nest_scopes", "nested_scopes"), + ("gneretors", "generators"), + ("divicion", "division"), + ("absolute_imports", "absolute_import"), + ("print_func", "print_function"), + ("unicode_literal", "unicode_literals"), + ("barry_as_bdfl", "barry_as_FLUFL"), + ("generatorstop", "generator_stop"), + ("anotations", "annotations"), + ]) + def test_typos_in_future_imports(self, typo, origin): + self.assertSyntaxError( + f"from __future__ import {typo}", + lineno=1, + message=( + f"future feature '{typo}' is not defined. " + f"Did you mean: '{origin}'?" + ), + offset=24, + ) + + @subTests("name", ["missing_name", "brces", "brace"]) + def test_no_suggestion_on_missing_name(self, name): + self.assertSyntaxError( + f"from __future__ import {name}", + lineno=1, + message=f"future feature '{name}' is not defined", + offset=24, ) def test_future_import_not_on_top(self): @@ -137,7 +173,7 @@ def test_future_import_star(self): code = """ from __future__ import * """ - self.assertSyntaxError(code, message='future feature * is not defined', offset=24) + self.assertSyntaxError(code, message="future feature '*' is not defined", offset=24) def test_future_import_braces(self): code = """ diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 65c425c1d29721b..17b1e03470dd048 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -789,7 +789,7 @@ def test_dll_dependency_import(self): pydname = importlib.util.find_spec("_sqlite3").origin depname = os.path.join( os.path.dirname(pydname), - "sqlite3{}.dll".format("_d" if "_d" in pydname else "")) + "sqlite3{}.dll".format("_d" if "_d" in os.path.basename(pydname) else "")) with os_helper.temp_dir() as tmp: tmp2 = os.path.join(tmp, "DLLs") diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index a4381f2c0233954..d3760ac3c266a8f 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6280,11 +6280,7 @@ def test_gc_module_has_signatures(self): self._test_module_has_signatures(gc, no_signature) def test_io_module_has_signatures(self): - methods_no_signature = { - 'BufferedRWPair': {'read', 'peek', 'read1', 'readinto', 'readinto1', 'write'}, - } - self._test_module_has_signatures(io, - methods_no_signature=methods_no_signature) + self._test_module_has_signatures(io) def test_itertools_module_has_signatures(self): import itertools diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index a210b81d877dc1e..f9fb84101d4a2eb 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -1458,13 +1458,13 @@ def test_chunk_size(self): t._CHUNK_SIZE = 0 with self.assertRaises(TypeError): t._CHUNK_SIZE = 'x' - with self.assertRaises(ValueError): + with self.assertRaises(OverflowError): t._CHUNK_SIZE = sys.maxsize + 1 - with self.assertRaises(ValueError): + with self.assertRaises(OverflowError): t._CHUNK_SIZE = -sys.maxsize - 2 - with self.assertRaises(ValueError): + with self.assertRaises(OverflowError): t._CHUNK_SIZE = 2**1000 - with self.assertRaises(ValueError): + with self.assertRaises(OverflowError): t._CHUNK_SIZE = -2**1000 with self.assertRaisesRegex(AttributeError, 'cannot be deleted'): del t._CHUNK_SIZE diff --git a/Lib/test/test_peg_generator/test_c_generator.py b/Lib/test/test_peg_generator/test_c_generator.py new file mode 100644 index 000000000000000..5b48738c2620a99 --- /dev/null +++ b/Lib/test/test_peg_generator/test_c_generator.py @@ -0,0 +1,281 @@ +import io +import unittest +from unittest import mock + +from test import test_tools + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen import grammar as grammar_module + from pegen.c_generator import CParserGenerator + from pegen.c_generator_file import CParserEmitter + from pegen.grammar import NamedItem, RuleKind + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, parse_string + + +class TestCGenerator(unittest.TestCase): + def make_generator(self, source): + grammar = parse_string(source, GrammarParser) + return CParserGenerator( + grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, io.StringIO() + ) + + def emit_parser(self, parser): + output = io.StringIO() + CParserEmitter(parser, output).emit() + return output.getvalue() + + def test_rule_types_distinguish_implicit_and_explicit_void_pointer(self): + generator = self.make_generator(""" + start: &implicit implicit explicit + implicit: NAME + explicit[void*]: NAME + """) + generator.rules["explicit"].type = "void *" + start, implicit, explicit = generator.prepare("example.gram").rules + + self.assertIsNone(implicit.signature.return_type) + self.assertEqual(explicit.signature.return_type, "void *") + self.assertEqual(implicit.signature.c_return_type, "void *") + self.assertEqual(explicit.signature.c_return_type, "void *") + self.assertEqual( + implicit.signature.declaration(), "static void *implicit_rule(Parser *p);" + ) + self.assertEqual( + explicit.signature.declaration(), "static void * explicit_rule(Parser *p);" + ) + call = start.alternatives[0].calls[0] + self.assertEqual(call.function, "_PyPegen_lookahead") + generator = self.make_generator("start: &explicit\nexplicit[void*]: NAME\n") + with self.assertRaisesRegex(RuntimeError, "return type is incompatible"): + generator.prepare("example.gram") + + def test_parser_plan_does_not_depend_on_compilation_state(self): + generator = self.make_generator(""" + @header 'CUSTOM HEADER' + @subheader 'CUSTOM SUBHEADER' + @trailer 'CUSTOM TRAILER %(modulename)s %(mode)d' + @modulename 'sample' + @bytecode '1' + start[mod_ty]: expr_without_invalid 'pass' "zsoft" "asoft" ('bb' | 'aa')* ENDMARKER + expr_without_invalid[expr_ty] (memo): name=expr [NUMBER] { name } + expr[expr_ty]: expr '+' NAME | NAME + """) + generator.debug = True + parser = generator.prepare("some/path/example.gram") + expected = self.emit_parser(parser) + + self.assertEqual(generator.file.getvalue(), "") + self.assertEqual(parser.source_name, "example.gram") + self.assertEqual(parser.headers, ("CUSTOM HEADER", "CUSTOM SUBHEADER")) + self.assertEqual(parser.trailer, "CUSTOM TRAILER sample 2") + self.assertEqual(parser.soft_keywords, ("asoft", "zsoft")) + self.assertEqual([word for word, _ in parser.keyword_groups[2]], ["bb", "aa"]) + self.assertTrue(any(rule.signature.kind is RuleKind.LOOP0 for rule in parser.rules)) + with self.assertRaises(AttributeError): + parser.rules[0].alternatives[0].calls[0].assigned_variable = "changed" + + generator.grammar.metas.clear() + generator.grammar.metas["trailer"] = "%(missing)s" + for rule in generator.all_rules.values(): + rule.name = "changed" + rule.type = "changed_type" + rule.flags = frozenset() + rule.rhs.alts[0].action = "changed_action" + rule.rhs.alts.clear() + generator.rules.clear() + generator.all_rules.clear() + generator.keywords.clear() + generator.soft_keywords.clear() + generator.debug = False + generator.skip_actions = True + self.assertEqual(self.emit_parser(parser), expected) + + def test_repeated_preparation_keeps_variable_names_local(self): + source = """ + start: 'run' expr term bindings other ENDMARKER + expr: expr '+' NAME | NAME + term: term '*' NUMBER | NUMBER + bindings: (name_var=NUMBER) name_var[expr_ty]=(NAME) [NUMBER] (NAME | NUMBER) { name_var_1 } + other: name_var=NUMBER name_var=NAME { name_var_1 } + """ + generator = self.make_generator(source) + parser = generator.prepare("example.gram") + expected = self.emit_parser(parser) + + self.assertEqual(self.emit_parser(parser), expected) + self.assertEqual(generator.prepare("example.gram"), parser) + other = self.make_generator(source).prepare("example.gram") + self.assertEqual(other, parser) + self.assertEqual(self.emit_parser(other), expected) + self.assertEqual(expected.count("expr_ty name_var_1;"), 2) + self.assertEqual(expected.count("_res = name_var_1;"), 2) + self.assertNotIn("name_var_2", expected) + + def test_prepared_prefixes_preserve_reuse_and_repeatability(self): + generator = self.make_generator(""" + start: prefix ':' NAME | prefix ':' NUMBER | NAME | prefix '=' NAME + prefix[expr_ty] (memo): NAME + """) + parser = generator.prepare("example.gram") + start = parser.rules[0] + prefix, = start.prefixes + self.assertEqual(prefix.type, "expr_ty") + for alt in start.alternatives[:2]: + self.assertIn("!p->call_invalid_rules", alt.calls[0].expression()) + self.assertIn(prefix.result, alt.calls[0].expression()) + self.assertEqual(start.alternatives[3].calls[0].expression(), "prefix_rule(p)") + self.assertEqual(generator.prepare("example.gram"), parser) + expected = self.emit_parser(parser) + generator.rules.clear() + generator.all_rules.clear() + self.assertEqual(self.emit_parser(parser), expected) + + def test_nullable_prefix_is_not_reused(self): + generator = self.make_generator(""" + start: prefix ':' NAME | prefix ':' NUMBER + prefix (memo): NAME? + """) + start = generator.prepare("example.gram").rules[0] + self.assertEqual(start.prefixes, ()) + for alt in start.alternatives: + self.assertEqual(alt.calls[0].expression(), "prefix_rule(p)") + + def test_invalid_trailer_fails_before_output(self): + generator = self.make_generator(""" + @trailer '%(missing)s' + start: NAME ENDMARKER + """) + with self.assertRaisesRegex(KeyError, "missing"): + generator.generate("example.gram") + self.assertEqual(generator.file.getvalue(), "") + + def test_empty_keyword_tables(self): + parser = self.make_generator("start: NAME ENDMARKER\n").prepare("example.gram") + source = self.emit_parser(parser) + + self.assertEqual(parser.keyword_groups, ()) + self.assertEqual(parser.soft_keywords, ()) + self.assertIn("static const int n_keyword_lists = 0;", source) + self.assertIn( + "static KeywordToken *reserved_keywords[] = {\n" + " (KeywordToken[]) {{NULL, -1}},\n" + "};", + source, + ) + self.assertIn("static char *soft_keywords[] = {\n NULL,\n};", source) + + def test_lowering_rejects_undiscovered_items(self): + for replacement in ("missing", "(NAME NUMBER)", None): + with self.subTest(replacement=replacement): + generator = self.make_generator("start: NAME ENDMARKER\n") + generator.collect_rules() + lowerer = generator.callmakervisitor.make_lowerer() + inventory = tuple(generator.all_rules) + counter = generator.counter + rule = generator.rules["start"] + items = rule.rhs.alts[0].items + if replacement is None: + items[0] = NamedItem(None, items[0].item) + else: + grammar = parse_string(f"start: {replacement}\n", GrammarParser) + items[0].item = grammar.rules["start"].rhs.alts[0].items[0].item + with self.assertRaisesRegex(RuntimeError, "not discovered"): + lowerer.prepare_rule(rule) + self.assertEqual(tuple(generator.all_rules), inventory) + self.assertEqual(generator.counter, counter) + + def test_helper_resolution_does_not_depend_on_display_settings(self): + source = """ + start: NAME (a=NAME { a }) NAME* NAME+ ','.NAME+ ENDMARKER + """ + for simple in (True, False): + with self.subTest(simple=simple), mock.patch.object( + grammar_module, "SIMPLE_STR", simple + ): + generator = self.make_generator(source) + expected = generator.prepare("example.gram").rules + with mock.patch.object(grammar_module, "SIMPLE_STR", not simple): + actual = generator.prepare("example.gram").rules + self.assertEqual(len(actual), len(expected)) + for old, new in zip(expected, actual): + self.assertEqual(old.signature, new.signature) + self.assertEqual( + [alt.calls for alt in old.alternatives], + [alt.calls for alt in new.alternatives], + ) + + def test_invalid_rule_gating_uses_references(self): + cases = ( + ("invalid_example", True), + ("value=invalid_example", True), + ("[invalid_example]", True), + ("invalid_example?", True), + ("invalid_example*", True), + ("invalid_example+", True), + ("invalid_example.NAME+", True), + ("[invalid_example.NAME+]", True), + ("[invalid_example.(NAME NAME)+]", False), + ("[[invalid_example.(NAME NAME)+]]", False), + ("[invalid_example.(NAME | NUMBER)+]", False), + ("&invalid_example", False), + ("[invalid_example | NAME]", False), + ("invalid_name=NAME", False), + ) + for item, requires_invalid_rules in cases: + for simple in (True, False): + with self.subTest(item=item, simple=simple), mock.patch.object( + grammar_module, "SIMPLE_STR", simple + ): + generator = self.make_generator(f""" + start: {item} {{ _PyPegen_dummy_name(p) }} + invalid_example: NAME + """) + start = generator.prepare("example.gram").rules[0] + self.assertEqual( + start.alternatives[0].requires_invalid_rules, + requires_invalid_rules, + ) + + def test_lowering_preserves_legacy_named_call_types(self): + generator = self.make_generator(""" + start: Mixed LPAR ENDMARKER + Mixed[expr_ty]: NAME + """) + start, mixed = generator.prepare("example.gram").rules + self.assertEqual(mixed.signature.return_type, "expr_ty") + for call, name in zip(start.alternatives[0].calls, ("Mixed", "LPAR")): + with self.subTest(name=name): + self.assertEqual(call.function, f"{name}_rule") + self.assertIsNone(call.return_type) + + def test_lowering_snapshots_symbols_and_tokens(self): + grammar = parse_string(""" + start: 'pass' '+' atom ENDMARKER + atom[expr_ty]: NAME + """, GrammarParser) + exact_tokens = dict(EXACT_TOKENS) + non_exact_tokens = set(NON_EXACT_TOKENS) + generator = CParserGenerator( + grammar, ALL_TOKENS, exact_tokens, non_exact_tokens, io.StringIO() + ) + generator.collect_rules() + lowerer = generator.callmakervisitor.make_lowerer() + start = generator.rules["start"] + atom = generator.rules["atom"] + expected = lowerer.prepare_rule(start) + + atom.type = "stmt_ty" + generator.all_rules.clear() + generator.tokens.clear() + generator.keywords.clear() + exact_tokens.clear() + non_exact_tokens.clear() + + self.assertEqual(lowerer.prepare_rule(atom).signature.return_type, "expr_ty") + self.assertEqual(lowerer.prepare_rule(start), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index cd0b907667e4a27..c430f648bc1563f 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -239,6 +239,21 @@ def test_negative_lookahead(self) -> None: """ self.run_test(grammar_source, test_source) + def test_optional_gather_with_invalid_separator(self) -> None: + grammar_source = """ + start: 'prefix' guard_without_invalid NAME NEWLINE ENDMARKER + guard_without_invalid: + | [invalid_separator.(NAME NAME)+] { _PyPegen_dummy_name(p) } + invalid_separator: '+' + """ + test_source = """ + self.check_input_strings_for_grammar( + valid_cases=["prefix hello", "prefix a b hello", "prefix a b + c d hello"], + invalid_cases=["prefix", "prefix a b"], + ) + """ + self.run_test(grammar_source, test_source) + def test_cut(self) -> None: grammar_source = """ start: X ~ Y Z | X Q S @@ -404,6 +419,132 @@ def test_same_name_different_types(self) -> None: """ self.run_test(grammar_source, test_source) + def test_alternative_variable_bindings(self) -> None: + grammar_source = """ + start[mod_ty]: a=stmt NEWLINE ENDMARKER { + _PyAST_Module((asdl_stmt_seq *)_PyPegen_singleton_seq(p, a), NULL, p->arena) } + stmt[stmt_ty]: + | &NAME NAME name_var[expr_ty]=NAME NUMBER? { + _PyAST_Expr(name_var_1, EXTRA) } + | &NUMBER name_var=NUMBER name_var[expr_ty]=NAME { + _PyAST_Expr(name_var_1, EXTRA) } + """ + test_source = """ + for source in ("first second", "first second 42", "42 second"): + actual = parse.parse_string(source, mode=1) + self.assertEqual(len(actual.body), 1) + self.assertIsInstance(actual.body[0], ast.Expr) + self.assertIsInstance(actual.body[0].value, ast.Name) + self.assertEqual(actual.body[0].value.id, "second") + """ + self.run_test(grammar_source, test_source) + + def test_rule_cleanup(self) -> None: + grammar_source = """ + @subheader ''' + #define CHECK_INVALID(expected) \\ + (assert(p->call_invalid_rules == (expected)), _PyPegen_dummy_name(p)) + ''' + start: enable (checked_without_invalid '+' | checked_without_invalid after | after) NEWLINE ENDMARKER + enable: 'enable' { (p->call_invalid_rules = 1, _PyPegen_dummy_name(p)) } + checked_without_invalid (memo): "value" ~ NAME { CHECK_INVALID(0) } + after: NAME { CHECK_INVALID(1) } + """ + test_source = """ + self.check_input_strings_for_grammar([ + "enable value name +", # Successful rule return. + "enable value name tail", # Memoized return after backtracking. + "enable fallback", # Failed rule return. + "enable value", # Early return through a cut. + ]) + """ + self.run_test(grammar_source, test_source) + + def test_left_recursive_rule_cleanup(self) -> None: + grammar_source = """ + @subheader ''' + #define CHECK_INVALID(expected) \\ + (assert(p->call_invalid_rules == (expected)), _PyPegen_dummy_name(p)) + ''' + start: enable (expr_without_invalid after | after) NEWLINE ENDMARKER + enable: 'enable' { (p->call_invalid_rules = 1, _PyPegen_dummy_name(p)) } + expr_without_invalid: + | expr_without_invalid '+' NAME { CHECK_INVALID(0) } + | NAME { CHECK_INVALID(0) } + after: NAME { CHECK_INVALID(1) } | NUMBER { CHECK_INVALID(1) } + """ + test_source = """ + self.check_input_strings_for_grammar([ + "enable name tail", + "enable name + other + last tail", + "enable fallback", # Backtrack past a successful recursive rule. + "enable 42", # The recursive rule has no successful alternative. + ]) + """ + self.run_test(grammar_source, test_source) + + def test_nested_rule_cleanup(self) -> None: + grammar_source = """ + @subheader ''' + #define CHECK_INVALID(expected) \\ + (assert(p->call_invalid_rules == (expected)), _PyPegen_dummy_name(p)) + ''' + start: enable outer_without_invalid after NEWLINE ENDMARKER + enable: 'enable' { (p->call_invalid_rules = 1, _PyPegen_dummy_name(p)) } + outer_without_invalid: + | inner_without_invalid '+' { CHECK_INVALID(0) } + | inner_without_invalid inside { CHECK_INVALID(0) } + | inside { CHECK_INVALID(0) } + inner_without_invalid (memo): 'value' NAME { CHECK_INVALID(0) } + inside: NAME { CHECK_INVALID(0) } + after: NAME { CHECK_INVALID(1) } + """ + test_source = """ + self.check_input_strings_for_grammar([ + "enable value name + tail", # Restore the enclosing disabled state. + "enable value name middle tail", # Restore it on a memoized return. + "enable fallback tail", # Restore it when the inner rule fails. + ]) + """ + self.run_test(grammar_source, test_source) + + def test_repetition_result_order(self) -> None: + grammar_source = """ + start[mod_ty]: a=statements NEWLINE ENDMARKER { + _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: + | 'repeat0' a=stmt* { (asdl_stmt_seq*)a } + | 'repeat1' a=stmt+ { (asdl_stmt_seq*)a } + | 'gather' a=','.stmt+ { (asdl_stmt_seq*)a } + stmt[stmt_ty]: a=NAME { _PyAST_Expr(a, EXTRA) } + """ + test_source = """ + for mode, separator in (("repeat0", " "), ("repeat1", " "), ("gather", ",")): + for count in (1, 2, 5, 17): + with self.subTest(mode=mode, count=count): + names = ["name" + str(index) for index in range(count)] + result = parse.parse_string(mode + " " + separator.join(names), mode=1) + self.assertEqual([stmt.value.id for stmt in result.body], names) + result = parse.parse_string("repeat0", mode=1) + self.assertEqual(result.body, []) + """ + self.run_test(grammar_source, test_source) + + def test_repetition_action_errors(self) -> None: + grammar_source = """ + start: ('repeat0' item* | 'repeat1' item+ | 'gather' ','.item+) NEWLINE ENDMARKER + item: NAME | 'fail' { PyTuple_New(-1) } + """ + test_source = """ + for mode, separator in (("repeat0", " "), ("repeat1", " "), ("gather", ",")): + for items in (("fail",), ("first", "second", "fail")): + with self.subTest(mode=mode, items=items): + with self.assertRaises(SystemError): + parse.parse_string(mode + " " + separator.join(items), mode=0) + parse.parse_string(mode + " first", mode=0) + """ + self.run_test(grammar_source, test_source) + def test_with_stmt_with_paren(self) -> None: grammar_source = """ start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index f39fcc2e0d8dafd..64c875367cef275 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -12,7 +12,7 @@ with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser from pegen.testutil import parse_string, generate_parser, make_parser - from pegen.grammar import GrammarVisitor, GrammarError, Grammar + from pegen.grammar import GrammarVisitor, GrammarError, Grammar, RuleKind from pegen.grammar_visualizer import ASTGrammarPrinter from pegen.parser import Parser from pegen.parser_generator import compute_nullables, compute_left_recursives @@ -537,6 +537,29 @@ def test_advanced_left_recursive(self) -> None: self.assertTrue(rules["start"].left_recursive) self.assertFalse(rules["sign"].left_recursive) + def test_helper_rule_kinds_do_not_depend_on_names(self) -> None: + grammar = parse_string(""" + start: NAME* NUMBER+ ','.NAME+ + """, GrammarParser) + generator = PythonParserGenerator(grammar, io.StringIO()) + generator.collect_rules() + helpers = [ + rule for rule in generator.all_rules.values() + if rule is not grammar.rules["start"] + ] + self.assertCountEqual( + [rule.kind for rule in helpers], + [RuleKind.LOOP0, RuleKind.LOOP1, RuleKind.LOOP0, RuleKind.GATHER], + ) + for rule in helpers: + is_loop, is_gather = rule.is_loop(), rule.is_gather() + rule.name = "renamed" + self.assertEqual(rule.is_loop(), is_loop) + self.assertEqual(rule.is_gather(), is_gather) + grammar.rules["start"].name = "_loop1_name_only" + self.assertFalse(grammar.rules["start"].is_loop()) + self.assertFalse(grammar.rules["start"].is_gather()) + def test_mutually_left_recursive(self) -> None: grammar_source = """ start: foo 'E' diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 2f5a5e273286590..b1fbfb5a0de34dd 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -17,7 +17,7 @@ "Test only runs when _remote_debugging is available" ) -from test.support import force_not_colorized_test_class +from test.support import force_colorized, force_not_colorized_test_class def print_sampled_stats(stats, sort=-1, limit=None, show_summary=True, sample_interval_usec=100): @@ -484,6 +484,15 @@ def test_print_sampled_stats_basic(self): self.assertIn("func1", result) self.assertIn("func2", result) self.assertIn("func3", result) + self.assertNotIn("\x1b[", result) + + @force_colorized + def test_print_sampled_stats_colorized(self): + with io.StringIO() as output, mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats) + result = output.getvalue() + + self.assertIn("\x1b[1;34mProfile Stats:", result) def test_print_sampled_stats_sorting(self): """Test different sorting options.""" @@ -751,6 +760,7 @@ def test_print_sampled_stats_sort_by_name(self): and not "calls" in line # Skip summary lines and not "total time" in line # Skip summary lines and not "cumulative time" in line + and not "filename:lineno(function)" in line # Skip header line ): # Skip summary lines data_lines.append(line) diff --git a/Lib/test/test_profiling/test_tracing_profiler.py b/Lib/test/test_profiling/test_tracing_profiler.py index 6a4d968f12ef156..50de9af86e86d02 100644 --- a/Lib/test/test_profiling/test_tracing_profiler.py +++ b/Lib/test/test_profiling/test_tracing_profiler.py @@ -85,6 +85,27 @@ def __call__(self): profiler_with_evil_timer.clear() self.assertEqual(cm.unraisable.exc_type, RuntimeError) + def test_enable_in_external_timer(self): + # gh-157639: Enabling the profiler from an external timer should not crash + import _lsprof + + # the timer re-arms monitoring from inside disable(), so the tool + # id stays claimed once the profiler is torn down + self.addCleanup(sys.monitoring.free_tool_id, sys.monitoring.PROFILER_ID) + + def timer(): + try: + profiler.enable() + except Exception: + pass + return 0 + + profiler = _lsprof.Profiler(timer=timer) + profiler.enable() + (lambda: None)() + profiler.disable() + profiler.clear() + def test_profile_enable_disable(self): prof = self.profilerclass() # Make sure we clean ourselves up if the test fails for some reason. diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index 2fc8398923cbf38..dbf7f91696b5591 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -1,6 +1,7 @@ import errno import itertools import os +import select import signal import sys import threading @@ -8,6 +9,7 @@ from functools import partial from _colorize import ANSIColors from test.support import force_color, os_helper, force_not_colorized_test_class +from test.support import is_android, is_apple_mobile, is_wasm32 from test.support import threading_helper from unittest import TestCase @@ -424,3 +426,93 @@ def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): # EIO error should be handled gracefully in restore() console.restore() + + +try: + import pty + import termios as _termios +except ImportError: + pty = None + + +@unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") +@unittest.skipUnless(pty, "requires pty") +@unittest.skipIf(is_android or is_apple_mobile or is_wasm32, + "pty is not available on this platform") +class TestUnixConsoleInputHook(TestCase): + # gh-152907: the console must restore cooked output (OPOST) around + # input-hook calls, then re-enter raw mode. + + def test_input_hook_output_is_cooked(self): + master_fd, slave_fd = pty.openpty() + self.addCleanup(os.close, master_fd) + + # tcsetattr(TCSADRAIN) blocks on some platforms (e.g. macOS) while the + # master still holds unread output, so empty it before each mode switch. + def drain(): + out = b"" + while select.select([master_fd], [], [], 0)[0]: + try: + data = os.read(master_fd, 4096) + except OSError: + break + if not data: + break + out += data + return out + + # Start from a cooked terminal so there are saved flags to restore. + attr = _termios.tcgetattr(slave_fd) + attr[1] |= _termios.OPOST | _termios.ONLCR + _termios.tcsetattr(slave_fd, _termios.TCSANOW, attr) + + console = UnixConsole(slave_fd, slave_fd, term="xterm") + console.prepare() + try: + drain() # discard prepare()'s own setup sequences + # pyrepl's own rendering runs with OPOST cleared. + self.assertFalse(_termios.tcgetattr(slave_fd)[1] & _termios.OPOST) + + observed = {} + + def fake_hook(): + observed["oflag"] = _termios.tcgetattr(slave_fd)[1] + os.write(slave_fd, b"line1\nline2\n") + observed["output"] = drain() + return 0 + + with patch("_pyrepl.unix_console.posix") as mock_posix: + mock_posix._is_inputhook_installed.return_value = True + mock_posix._inputhook.side_effect = fake_hook + hook = console.input_hook + self.assertIsNotNone(hook) + self.assertEqual(hook(), 0) + + # The hook ran with cooked output (OPOST on)... + self.assertTrue(observed["oflag"] & _termios.OPOST) + # ...and raw mode was restored afterwards. + self.assertFalse(_termios.tcgetattr(slave_fd)[1] & _termios.OPOST) + # The tty translated the hook's bare '\n' into '\r\n'. + self.assertEqual(observed["output"], b"line1\r\nline2\r\n") + finally: + # restore() writes and only then switches modes, so there is no + # point left to drain from here; keep the master empty elsewhere. + stop = threading.Event() + + def pump(): + while not stop.is_set(): + if select.select([master_fd], [], [], 0.05)[0]: + try: + if not os.read(master_fd, 4096): + break + except OSError: + break + + pump_thread = threading.Thread(target=pump) + pump_thread.start() + try: + console.restore() + finally: + stop.set() + pump_thread.join() + os.close(slave_fd) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index ed6eb706c40d223..21e603c561d73c0 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -185,9 +185,8 @@ def read_until(marker, start=0): @cpython_only def test_lexer_buffer_realloc_with_null_start(self): - # gh-144759: NULL pointer arithmetic in the lexer when start and - # multi_line_start are NULL (uninitialized in tok_mode_stack[0]) - # and the lexer buffer is reallocated while parsing long input. + # gh-144759: NULL pointer arithmetic when the lexer buffer grows + # while parsing long input. long_value = "a" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 862a20a058be75a..ec98e609c4e98f9 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -3,8 +3,8 @@ import unittest from test import support from test.support import script_helper -from test.support.os_helper import TESTFN, unlink, rmtree -from test.support.import_helper import unload +from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree +from test.support.import_helper import import_module, unload import importlib import os import sys @@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self): self.assertRaises(SyntaxError, compile, seq, '', 'exec') def test_invalid_utf8_offset_after_non_ascii(self): + for name in ('รฉ', 'รฉรฉ', '๐’œ'): + with self.subTest(name=name): + source = ('x = ' + name).encode() + b'\xff\n' + with self.assertRaises(SyntaxError) as caught: + compile(source, '', 'exec') + error = caught.exception + self.assertEqual( + (error.lineno, error.offset, error.end_lineno, error.end_offset), + (1, 5 + len(name), 1, 5 + len(name)), + ) + + @support.cpython_only + def test_invalid_utf8_file_offset_after_non_ascii(self): + _testcapi = import_module('_testcapi') + self.addCleanup(unlink, TESTFN_ASCII) + with open(TESTFN_ASCII, 'wb') as f: + f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n') with self.assertRaises(SyntaxError) as caught: - compile(b"x = \xc3\xa9\xff\n", "", "exec") + _testcapi.run_file( + os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {}) error = caught.exception self.assertEqual( (error.lineno, error.offset, error.end_lineno, error.end_offset), - (1, 6, 1, 6), + (2, 7, 2, 7), ) def test_long_bom_conflict_message_is_not_truncated(self): diff --git a/Lib/test/test_tools/test_inspection.py b/Lib/test/test_tools/test_inspection.py new file mode 100644 index 000000000000000..f4a0d163adbc562 --- /dev/null +++ b/Lib/test/test_tools/test_inspection.py @@ -0,0 +1,111 @@ +"""Tests for snippets in Tools/inspection.""" + +import unittest +from types import SimpleNamespace + +from test.test_tools import imports_under_tool, skip_if_missing + + +skip_if_missing("inspection") +with imports_under_tool("inspection"): + import snippets + + +def frame(funcname, *, filename="", lineno=None): + location = SimpleNamespace(lineno=lineno) if lineno is not None else None + return SimpleNamespace( + funcname=funcname, + filename=filename, + location=location, + ) + + +def frames(*names): + return [frame(name) for name in names] + + +class ClassifierTests(unittest.TestCase): + def test_classifiers(self): + flat_lines = snippets.FLAT_ALTERNATING_LINES + short_line = min(snippets.SHARED_LEAF_SHORT_LINES) + flat_a = [ + frame("leaf_a", lineno=flat_lines["leaf_a"]), + frame("hot_a", lineno=flat_lines["hot_a"]), + ] + flat_crossed = [ + frame("hot_a", lineno=flat_lines["hot_a"]), + frame("hot_b", lineno=flat_lines["hot_b"]), + ] + shared_a = [ + frame("shared_leaf", lineno=short_line), + frame("a_wrapper"), + ] + shared_crossed = [ + frame("shared_leaf", lineno=short_line), + frame("b_wrapper"), + ] + cases = [ + (snippets.classify_flat, flat_a, False), + (snippets.classify_flat, flat_crossed, True), + ( + snippets.classify_nested, + frames("burn_a", "a_leaf", "a_parent"), + False, + ), + ( + snippets.classify_nested, + frames("a_parent", "a_leaf", "burn_a"), + True, + ), + (snippets.classify_shared, shared_a, False), + (snippets.classify_shared, shared_crossed, True), + (snippets.classify_gen, frames("agen", "drv_a"), False), + (snippets.classify_gen, frames("agen", "drv_b"), True), + (snippets.classify_gen, frames("bgen", "drv_a"), True), + (snippets.classify_gen, frames("agen"), False), + ( + snippets.classify_gen, + frames("agen", "agen", "drv_a"), + False, + ), + + (snippets.classify_recursion, frames("a", "a"), False), + (snippets.classify_recursion, frames("a", "b"), True), + ( + snippets.classify_async_running_task, + (None, "hot", None, frames("leaf_hot")), + False, + ), + ( + snippets.classify_async_running_task, + (None, "hot", None, frames("leaf_rare")), + True, + ), + ( + snippets.classify_code_object_reuse, + [frame("func_a", filename="A_file.py")], + False, + ), + ( + snippets.classify_code_object_reuse, + [frame("func_a", filename="B_file.py")], + True, + ), + ( + snippets.classify_oversized_chunk, + [frame("big_a", filename="a.py")], + False, + ), + ( + snippets.classify_oversized_chunk, + [frame("big_b", filename="a.py")], + True, + ), + ] + for classifier, sample, expected in cases: + with self.subTest(classifier=classifier.__name__, sample=sample): + self.assertEqual(classifier(sample), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 20a5083f60d11ae..ea20688fdc97536 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -323,6 +323,8 @@ def test_nested_templates(self): def test_syntax_errors(self): for case, err in ( + ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"), + ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"), ("t'", "unterminated t-string literal"), ("t'''", "unterminated triple-quoted t-string literal"), ("t''''", "unterminated triple-quoted t-string literal"), diff --git a/Makefile.pre.in b/Makefile.pre.in index 71952252ac9f114..b29976ee041099c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,10 +395,10 @@ PEGEN_OBJS= \ TOKENIZER_OBJS= \ Parser/lexer/lexer.o \ + Parser/lexer/layout.o \ Parser/lexer/number.o \ Parser/lexer/state.o \ Parser/lexer/string.o \ - Parser/tokenizer/cursor.o \ Parser/tokenizer/decoder.o \ Parser/tokenizer/api.o \ Parser/tokenizer/reader.o \ @@ -411,10 +411,8 @@ PEGEN_HEADERS= \ $(srcdir)/Parser/string_parser.h TOKENIZER_HEADERS= \ - Parser/lexer/lexer.h \ Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ - Parser/tokenizer/cursor.h \ Parser/tokenizer/reader.h \ Parser/tokenizer/reader_internal.h \ Parser/tokenizer/source.h \ @@ -3471,7 +3469,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst b/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst index 3c19c0e15d2bc65..bb85114220bbe63 100644 --- a/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst +++ b/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst @@ -1,3 +1,3 @@ :c:func:`PyConfig_Set()` now also set global configuration variables. For example, ``PyConfig_Set("inspect", value)`` now also sets -:c:var:`Py_InspectFlag`. Patch by Victor Stinner. +:c:var:`!Py_InspectFlag`. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-09-23-19-37-53.gh-issue-158001.X_Xamm.rst b/Misc/NEWS.d/next/C_API/2026-09-23-19-37-53.gh-issue-158001.X_Xamm.rst new file mode 100644 index 000000000000000..1b81be97e283eac --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-23-19-37-53.gh-issue-158001.X_Xamm.rst @@ -0,0 +1,25 @@ +Remove 21 global configuration variables: + +* :c:var:`!Py_BytesWarningFlag`, +* :c:var:`!Py_DebugFlag`, +* :c:var:`!Py_DontWriteBytecodeFlag`, +* :c:var:`!Py_FileSystemDefaultEncodeErrors`, +* :c:var:`!Py_FileSystemDefaultEncoding`, +* :c:var:`!Py_FrozenFlag`, +* :c:var:`!Py_HasFileSystemDefaultEncoding`, +* :c:var:`!Py_HashRandomizationFlag`, +* :c:var:`!Py_IgnoreEnvironmentFlag`, +* :c:var:`!Py_InspectFlag`, +* :c:var:`!Py_InteractiveFlag`, +* :c:var:`!Py_IsolatedFlag`, +* :c:var:`!Py_LegacyWindowsFSEncodingFlag`, +* :c:var:`!Py_LegacyWindowsStdioFlag`, +* :c:var:`!Py_NoSiteFlag`, +* :c:var:`!Py_NoUserSiteDirectory`, +* :c:var:`!Py_OptimizeFlag`, +* :c:var:`!Py_QuietFlag`, +* :c:var:`!Py_UTF8Mode`, +* :c:var:`!Py_UnbufferedStdioFlag`, +* :c:var:`!Py_VerboseFlag`. + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-09-24-14-14-34.gh-issue-154757.-b-Fs7.rst b/Misc/NEWS.d/next/C_API/2026-09-24-14-14-34.gh-issue-154757.-b-Fs7.rst new file mode 100644 index 000000000000000..2e28e24ebff0a77 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-24-14-14-34.gh-issue-154757.-b-Fs7.rst @@ -0,0 +1,3 @@ +:c:func:`PyModule_GetFilename` is no longer deprecated, but using +:c:func:`PyModule_GetFilenameObject` instead is still recommended. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-14-38-49.gh-issue-145868.7iGRvU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-14-38-49.gh-issue-145868.7iGRvU.rst new file mode 100644 index 000000000000000..c3aa54979f3d16f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-14-38-49.gh-issue-145868.7iGRvU.rst @@ -0,0 +1,2 @@ +Suggest the closest valid feature name in the :exc:`SyntaxError` message +when an unknown name is imported from :mod:`__future__`. diff --git a/Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst b/Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst new file mode 100644 index 000000000000000..33247f0f3a3b968 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst @@ -0,0 +1,4 @@ +Restore cooked-mode terminal output flags around :c:data:`PyOS_InputHook` +callbacks in the new :term:`REPL` (:mod:`!_pyrepl`), so that output written +by an input hook (for example a GUI toolkit event loop) is no longer emitted +with ``OPOST`` disabled and keeps its carriage returns. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-12-23-48.gh-issue-154335.SRf8Gr.rst b/Misc/NEWS.d/next/Library/2026-07-21-12-23-48.gh-issue-154335.SRf8Gr.rst new file mode 100644 index 000000000000000..ffd923294551f2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-12-23-48.gh-issue-154335.SRf8Gr.rst @@ -0,0 +1,3 @@ +The :mod:`profiling.sampling` profiler now respects ``NO_COLOR``, +``FORCE_COLOR``, and ``PYTHON_COLORS`` and avoids colored output when +not writing to a terminal. diff --git a/Misc/NEWS.d/next/Library/2026-08-10-19-49-11.gh-issue-155496.B4rMR1.rst b/Misc/NEWS.d/next/Library/2026-08-10-19-49-11.gh-issue-155496.B4rMR1.rst new file mode 100644 index 000000000000000..75f0f987127b198 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-10-19-49-11.gh-issue-155496.B4rMR1.rst @@ -0,0 +1,3 @@ +:meth:`io.RawIOBase.readinto` and :meth:`io.RawIOBase.write` now raise +:exc:`TypeError` instead of :exc:`NotImplementedError` +if they are called without the required argument. diff --git a/Misc/NEWS.d/next/Library/2026-08-10-19-49-12.gh-issue-155496.tJtiAt.rst b/Misc/NEWS.d/next/Library/2026-08-10-19-49-12.gh-issue-155496.tJtiAt.rst new file mode 100644 index 000000000000000..1d3a248a645e701 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-10-19-49-12.gh-issue-155496.tJtiAt.rst @@ -0,0 +1,3 @@ +The ``ctypes.set_errno`` and ``ctypes.set_last_error`` auditing events are +now raised after the argument is converted to an integer, therefore they are +no longer raised if the argument is invalid. diff --git a/Misc/NEWS.d/next/Library/2026-09-17-03-20-24.gh-issue-157639.VP3kc4.rst b/Misc/NEWS.d/next/Library/2026-09-17-03-20-24.gh-issue-157639.VP3kc4.rst new file mode 100644 index 000000000000000..de1fd1286d7a574 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-17-03-20-24.gh-issue-157639.VP3kc4.rst @@ -0,0 +1,3 @@ +Fix a crash in :mod:`!cProfile` when an external timer re-enters the +profiler. Profiling events raised while the external timer runs are now +ignored. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-07-16-11-38-15.gh-issue-153804.KSTYg7.rst b/Misc/NEWS.d/next/Tools-Demos/2026-07-16-11-38-15.gh-issue-153804.KSTYg7.rst new file mode 100644 index 000000000000000..7d0f287a72445b5 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-07-16-11-38-15.gh-issue-153804.KSTYg7.rst @@ -0,0 +1,4 @@ +Add ``Tools/inspection/oracle_external_inspection.py``, a harness for +measuring the accuracy of Tachyon. It reports impossible-stack rates, speed, +error rates and the statistical distance from the blocking non-cached best +reference on selected snippets. Patch by Maurycy Pawล‚owski-Wieroล„ski. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 08fd7009dbc4b86..0425fba42cbe2c7 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1643,6 +1643,7 @@ added = '3.2' [data.Py_FileSystemDefaultEncoding] added = '3.2' + abi_only = true [function.Py_Finalize] added = '3.2' [function.Py_GetBuildInfo] @@ -1677,6 +1678,7 @@ added = '3.2' [data.Py_HasFileSystemDefaultEncoding] added = '3.2' + abi_only = true [function.Py_IncRef] added = '3.2' [function.Py_Initialize] @@ -2178,6 +2180,7 @@ added = '3.8' [data.Py_UTF8Mode] added = '3.8' + abi_only = true [function.PyExceptionClass_Name] added = '3.8' [function.PyIndex_Check] @@ -2264,6 +2267,7 @@ added = '3.10' [data.Py_FileSystemDefaultEncodeErrors] added = '3.10' + abi_only = true [function.PyCodec_Unregister] added = '3.10' [function.PyErr_SetInterruptEx] diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 1f2c49b5878cc70..84aa8c1443fdf58 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -95,6 +95,25 @@ module _ctypes #include "pycore_global_objects.h"// _Py_ID() #include "pycore_traceback.h" // _PyTraceback_Add() +#ifndef MS_WIN32 +# if HAVE_DECL_RTLD_LOCAL +# define DLOPEN_DEFAULT_MODE (RTLD_NOW | RTLD_LOCAL) +# else + /* cygwin doesn't define RTLD_LOCAL */ +# define DLOPEN_DEFAULT_MODE RTLD_NOW +# endif +#endif + +static int _parse_voidp(PyObject *obj, void *arg); +static int _parse_voidp_object(PyObject *obj, void *arg); + +/*[python input] +class voidp_converter(CConverter): + type = 'void *' + converter = '_parse_voidp' +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=c230831f53ec0946]*/ + #define clinic_state() (get_module_state(module)) #include "clinic/callproc.c.h" #undef clinic_state @@ -186,10 +205,10 @@ _ctypes_get_errobj(ctypes_state *st, int **pspace) } static PyObject * -get_error_internal(PyObject *self, PyObject *args, int index) +get_error_internal(PyObject *module, int index) { int *space; - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); PyObject *errobj = _ctypes_get_errobj(st, &space); PyObject *result; @@ -201,16 +220,13 @@ get_error_internal(PyObject *self, PyObject *args, int index) } static PyObject * -set_error_internal(PyObject *self, PyObject *args, int index) +set_error_internal(PyObject *module, int new_errno, int index) { - int new_errno, old_errno; + int old_errno; PyObject *errobj; int *space; - if (!PyArg_ParseTuple(args, "i", &new_errno)) { - return NULL; - } - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); errobj = _ctypes_get_errobj(st, &space); if (errobj == NULL) return NULL; @@ -220,42 +236,72 @@ set_error_internal(PyObject *self, PyObject *args, int index) return PyLong_FromLong(old_errno); } +/*[clinic input] +_ctypes.get_errno + +Return the current value of the ctypes-private copy of errno. +[clinic start generated code]*/ + static PyObject * -get_errno(PyObject *self, PyObject *args) +_ctypes_get_errno_impl(PyObject *module) +/*[clinic end generated code: output=82992e823984b48e input=bf578c7a7608b0f4]*/ { if (PySys_Audit("ctypes.get_errno", NULL) < 0) { return NULL; } - return get_error_internal(self, args, 0); + return get_error_internal(module, 0); } +/*[clinic input] +_ctypes.set_errno + + value: int + / +[clinic start generated code]*/ + static PyObject * -set_errno(PyObject *self, PyObject *args) +_ctypes_set_errno_impl(PyObject *module, int value) +/*[clinic end generated code: output=0c85e07ccf0749c2 input=b2550d9d2cdfab20]*/ { - if (PySys_Audit("ctypes.set_errno", "O", args) < 0) { + if (PySys_Audit("ctypes.set_errno", "i", value) < 0) { return NULL; } - return set_error_internal(self, args, 0); + return set_error_internal(module, value, 0); } #ifdef MS_WIN32 +/*[clinic input] +_ctypes.get_last_error + +Return the current value of the ctypes-private copy of the last error. +[clinic start generated code]*/ + static PyObject * -get_last_error(PyObject *self, PyObject *args) +_ctypes_get_last_error_impl(PyObject *module) +/*[clinic end generated code: output=e237c4d3b75eb1eb input=d25599a79c030bb4]*/ { if (PySys_Audit("ctypes.get_last_error", NULL) < 0) { return NULL; } - return get_error_internal(self, args, 1); + return get_error_internal(module, 1); } +/*[clinic input] +_ctypes.set_last_error + + value: int + / +[clinic start generated code]*/ + static PyObject * -set_last_error(PyObject *self, PyObject *args) +_ctypes_set_last_error_impl(PyObject *module, int value) +/*[clinic end generated code: output=dce5b4be6255bcb5 input=1a3f4ba970902db7]*/ { - if (PySys_Audit("ctypes.set_last_error", "O", args) < 0) { + if (PySys_Audit("ctypes.set_last_error", "i", value) < 0) { return NULL; } - return set_error_internal(self, args, 1); + return set_error_internal(module, value, 1); } static WCHAR *FormatError(DWORD code) @@ -446,12 +492,17 @@ static DWORD HandleException(EXCEPTION_POINTERS *ptrs, } #endif +/*[clinic input] +_ctypes._check_HRESULT + + hresult as hr: int + / +[clinic start generated code]*/ + static PyObject * -check_hresult(PyObject *self, PyObject *args) +_ctypes__check_HRESULT_impl(PyObject *module, int hr) +/*[clinic end generated code: output=ba872ad9e32b2d1f input=75e2334791d6cf29]*/ { - HRESULT hr; - if (!PyArg_ParseTuple(args, "i", &hr)) - return NULL; if (FAILED(hr)) return PyErr_SetFromWindowsErr(hr); return PyLong_FromLong(hr); @@ -1368,18 +1419,25 @@ _parse_voidp(PyObject *obj, void *arg) #ifdef MS_WIN32 -PyDoc_STRVAR(format_error_doc, -"FormatError([integer]) -> string\n\ -\n\ -Convert a win32 error code into a string. If the error code is not\n\ -given, the return value of a call to GetLastError() is used.\n"); -static PyObject *format_error(PyObject *self, PyObject *args) +/*[clinic input] +_ctypes.FormatError + + code: int = 0 + / + +Convert a win32 error code into a string. + +If the error code is not given, the return value of a call to +GetLastError() is used. +[clinic start generated code]*/ + +static PyObject * +_ctypes_FormatError_impl(PyObject *module, int code) +/*[clinic end generated code: output=ccc560dddfd0b3d7 input=e0c98f1c644e5153]*/ { PyObject *result; wchar_t *lpMsgBuf; - DWORD code = 0; - if (!PyArg_ParseTuple(args, "|i:FormatError", &code)) - return NULL; + if (code == 0) code = GetLastError(); lpMsgBuf = FormatError(code); @@ -1392,23 +1450,26 @@ static PyObject *format_error(PyObject *self, PyObject *args) return result; } -PyDoc_STRVAR(load_library_doc, -"LoadLibrary(name, load_flags) -> handle\n\ -\n\ -Load an executable (usually a DLL), and return a handle to it.\n\ -The handle may be used to locate exported functions in this\n\ -module. load_flags are as defined for LoadLibraryEx in the\n\ -Windows API.\n"); -static PyObject *load_library(PyObject *self, PyObject *args) +/*[clinic input] +_ctypes.LoadLibrary + + name as nameobj: unicode + load_flags: int = 0 + / + +Load an executable (usually a DLL), and return a handle to it. + +The handle may be used to locate exported functions in this module. +load_flags are as defined for LoadLibraryEx in the Windows API. +[clinic start generated code]*/ + +static PyObject * +_ctypes_LoadLibrary_impl(PyObject *module, PyObject *nameobj, int load_flags) +/*[clinic end generated code: output=4e33b2cda4b12af6 input=69e4b6512653825e]*/ { - PyObject *nameobj; - int load_flags = 0; HMODULE hMod; DWORD err; - if (!PyArg_ParseTuple(args, "U|i:LoadLibrary", &nameobj, &load_flags)) - return NULL; - if (PySys_Audit("ctypes.dlopen", "O", nameobj) < 0) { return NULL; } @@ -1443,17 +1504,21 @@ static PyObject *load_library(PyObject *self, PyObject *args) #endif } -PyDoc_STRVAR(free_library_doc, -"FreeLibrary(handle) -> void\n\ -\n\ -Free the handle of an executable previously loaded by LoadLibrary.\n"); -static PyObject *free_library(PyObject *self, PyObject *args) +/*[clinic input] +_ctypes.FreeLibrary + + handle as hMod: voidp + / + +Free the handle of an executable previously loaded by LoadLibrary. +[clinic start generated code]*/ + +static PyObject * +_ctypes_FreeLibrary_impl(PyObject *module, void *hMod) +/*[clinic end generated code: output=2424a8daf3fe93df input=e722e432560e849a]*/ { - void *hMod; BOOL result; DWORD err; - if (!PyArg_ParseTuple(args, "O&:FreeLibrary", &_parse_voidp, &hMod)) - return NULL; Py_BEGIN_ALLOW_THREADS result = FreeLibrary((HMODULE)hMod); @@ -1466,20 +1531,27 @@ static PyObject *free_library(PyObject *self, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(copy_com_pointer_doc, -"CopyComPointer(src, dst) -> HRESULT value\n"); +/*[clinic input] +_ctypes.CopyComPointer + + src as p1: object + dst as p2: object + / + +Copy a COM pointer and return the HRESULT value. +[clinic start generated code]*/ static PyObject * -copy_com_pointer(PyObject *self, PyObject *args) +_ctypes_CopyComPointer_impl(PyObject *module, PyObject *p1, PyObject *p2) +/*[clinic end generated code: output=bbe731fea856eba2 input=296c1f3e026bbf27]*/ { - PyObject *p1, *p2, *r = NULL; + PyObject *r = NULL; struct argument a, b; IUnknown *src, **pdst; - if (!PyArg_ParseTuple(args, "OO:CopyComPointer", &p1, &p2)) - return NULL; + a.keep = b.keep = NULL; - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); if (ConvParam(st, p1, 0, &a) < 0 || ConvParam(st, p2, 1, &b) < 0) { goto done; } @@ -1530,17 +1602,25 @@ __attribute__((destructor)) void unload_dyld_shared_cache_contains_path(void) { _dyld_shared_cache_contains_path != NULL #endif -static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) +/*[clinic input] +_ctypes._dyld_shared_cache_contains_path + + path as name: object + / + +Check whether a path is in the shared cache. +[clinic start generated code]*/ + +static PyObject * +_ctypes__dyld_shared_cache_contains_path(PyObject *module, PyObject *name) +/*[clinic end generated code: output=1d3fed04c0490294 input=bb910a42599d991f]*/ { - PyObject *name, *name2; + PyObject *name2; char *name_str; if (HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME) { int r; - if (!PyArg_ParseTuple(args, "O", &name)) - return NULL; - if (name == Py_None) Py_RETURN_FALSE; @@ -1565,19 +1645,24 @@ static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *ar } #endif -static PyObject *py_dl_open(PyObject *self, PyObject *args) +/*[clinic input] +_ctypes.dlopen + + name: object + mode: int(c_default="DLOPEN_DEFAULT_MODE") = RTLD_NOW | RTLD_LOCAL + / + +Open a shared library. +[clinic start generated code]*/ + +static PyObject * +_ctypes_dlopen_impl(PyObject *module, PyObject *name, int mode) +/*[clinic end generated code: output=d68f1775017199b8 input=df6aa38a12f91cfe]*/ { - PyObject *name, *name2; + PyObject *name2; const char *name_str; void * handle; -#if HAVE_DECL_RTLD_LOCAL - int mode = RTLD_NOW | RTLD_LOCAL; -#else - /* cygwin doesn't define RTLD_LOCAL */ - int mode = RTLD_NOW; -#endif - if (!PyArg_ParseTuple(args, "O|i:dlopen", &name, &mode)) - return NULL; + mode |= RTLD_NOW; if (name != Py_None) { if (PyUnicode_FSConverter(name, &name2) == 0) @@ -1604,12 +1689,19 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) return PyLong_FromVoidPtr(handle); } -static PyObject *py_dl_close(PyObject *self, PyObject *args) -{ - void *handle; +/*[clinic input] +_ctypes.dlclose - if (!PyArg_ParseTuple(args, "O&:dlclose", &_parse_voidp, &handle)) - return NULL; + handle: voidp + / + +Close a shared library. +[clinic start generated code]*/ + +static PyObject * +_ctypes_dlclose_impl(PyObject *module, void *handle) +/*[clinic end generated code: output=80a4d433a7c81348 input=b8a3d95439746deb]*/ +{ if (dlclose(handle)) { const char *errmsg = dlerror(); if (errmsg) { @@ -1622,16 +1714,24 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject *py_dl_sym(PyObject *self, PyObject *args) +/*[clinic input] +_ctypes.dlsym + + handle: voidp + name: str + / + +Find a symbol in a shared library. +[clinic start generated code]*/ + +static PyObject * +_ctypes_dlsym_impl(PyObject *module, void *handle, const char *name) +/*[clinic end generated code: output=fada3d0f0447614c input=ee37a4935a19550c]*/ { - char *name; - void *handle; void *ptr; - if (!PyArg_ParseTuple(args, "O&s:dlsym", - &_parse_voidp, &handle, &name)) - return NULL; - if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { + if (PySys_Audit("ctypes.dlsym/handle", "ns", (Py_ssize_t)handle, + name) < 0) { return NULL; } #undef USE_DLERROR @@ -1679,8 +1779,15 @@ _dllist_callback(struct dl_phdr_info *info, size_t size, void *data) return res; } +/*[clinic input] +_ctypes.dllist + +Return a list of loaded shared libraries. +[clinic start generated code]*/ + static PyObject * -dllist(PyObject *self, PyObject *Py_UNUSED(ignored)) +_ctypes_dllist_impl(PyObject *module) +/*[clinic end generated code: output=7b4eaaecc7abf21a input=5bfb4e345425f59f]*/ { // On NetBSD dl_iterate_phdr() only reports the link-map group of the // caller, so it cannot be called via a libffi trampoline. @@ -1704,20 +1811,21 @@ dllist(PyObject *self, PyObject *Py_UNUSED(ignored)) * * XXX Needs to accept more arguments: flags, argtypes, restype */ +/*[clinic input] +_ctypes.call_function + + func: voidp + arguments: object(subclass_of='&PyTuple_Type') + / +[clinic start generated code]*/ + static PyObject * -call_function(PyObject *self, PyObject *args) +_ctypes_call_function_impl(PyObject *module, void *func, PyObject *arguments) +/*[clinic end generated code: output=e90059bba8e0f0de input=2f2a7a5ea1b4031f]*/ { - void *func; - PyObject *arguments; PyObject *result; - if (!PyArg_ParseTuple(args, - "O&O!", - &_parse_voidp, &func, - &PyTuple_Type, &arguments)) - return NULL; - - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); result = _ctypes_callproc(st, (PPROC)func, arguments, @@ -1737,20 +1845,22 @@ call_function(PyObject *self, PyObject *args) * * XXX Needs to accept more arguments: flags, argtypes, restype */ +/*[clinic input] +_ctypes.call_cdeclfunction + + func: voidp + arguments: object(subclass_of='&PyTuple_Type') + / +[clinic start generated code]*/ + static PyObject * -call_cdeclfunction(PyObject *self, PyObject *args) +_ctypes_call_cdeclfunction_impl(PyObject *module, void *func, + PyObject *arguments) +/*[clinic end generated code: output=697e21ce2e298acd input=bc3260e856aa7416]*/ { - void *func; - PyObject *arguments; PyObject *result; - if (!PyArg_ParseTuple(args, - "O&O!", - &_parse_voidp, &func, - &PyTuple_Type, &arguments)) - return NULL; - - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); result = _ctypes_callproc(st, (PPROC)func, arguments, @@ -1803,15 +1913,22 @@ _ctypes_sizeof(PyObject *module, PyObject *obj) return NULL; } -PyDoc_STRVAR(alignment_doc, -"alignment(C type) -> integer\n" -"alignment(C instance) -> integer\n" -"Return the alignment requirements of a C instance"); +/*[clinic input] +_ctypes.alignment + + obj: object + / + +Return the alignment requirements of a C instance. + +The argument is a C type or a C instance. +[clinic start generated code]*/ static PyObject * -align_func(PyObject *self, PyObject *obj) +_ctypes_alignment(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=a7abe04d98641d93 input=3c78c5425bc83928]*/ { - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); StgInfo *info; if (PyStgInfo_FromAny(st, obj, &info) < 0) { return NULL; @@ -1873,36 +1990,60 @@ _ctypes_addressof_impl(PyObject *module, PyObject *obj) } static int -converter(PyObject *obj, void *arg) +_parse_voidp_object(PyObject *obj, void *arg) { - void **address = (void **)arg; + PyObject **address = (PyObject **)arg; *address = PyLong_AsVoidPtr(obj); return *address != NULL; } +/*[clinic input] +_ctypes.PyObj_FromPtr + + address as ob: object(converter="_parse_voidp_object") + / +[clinic start generated code]*/ + static PyObject * -My_PyObj_FromPtr(PyObject *self, PyObject *args) +_ctypes_PyObj_FromPtr_impl(PyObject *module, PyObject *ob) +/*[clinic end generated code: output=225d11c8e84c2926 input=1baea4849458fe7c]*/ { - PyObject *ob; - if (!PyArg_ParseTuple(args, "O&:PyObj_FromPtr", converter, &ob)) { - return NULL; - } if (PySys_Audit("ctypes.PyObj_FromPtr", "(O)", ob) < 0) { return NULL; } return Py_NewRef(ob); } +/*[clinic input] +_ctypes.Py_INCREF + + obj as arg: object + / + +Increment the reference count of the object and return it. +[clinic start generated code]*/ + static PyObject * -My_Py_INCREF(PyObject *self, PyObject *arg) +_ctypes_Py_INCREF(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=ebf05a7f9ba69657 input=04fcd9c30fee39ea]*/ { Py_INCREF(arg); /* that's what this function is for */ Py_INCREF(arg); /* that for returning it */ return arg; } +/*[clinic input] +_ctypes.Py_DECREF + + obj as arg: object + / + +Decrement the reference count of the object and return it. +[clinic start generated code]*/ + static PyObject * -My_Py_DECREF(PyObject *self, PyObject *arg) +_ctypes_Py_DECREF(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=d7933cd7be22ccde input=108543079a732003]*/ { Py_DECREF(arg); /* that's what this function is for */ Py_INCREF(arg); /* that's for returning it */ @@ -1969,13 +2110,20 @@ _ctypes_resize_impl(PyObject *module, CDataObject *obj, Py_ssize_t size) Py_RETURN_NONE; } +/*[clinic input] +_ctypes._unpickle + + cls as typ: object + state: object(subclass_of='&PyTuple_Type') + / +[clinic start generated code]*/ + static PyObject * -unpickle(PyObject *self, PyObject *args) +_ctypes__unpickle_impl(PyObject *module, PyObject *typ, PyObject *state) +/*[clinic end generated code: output=358df408ddcf5145 input=1cdb32990e0ef9b8]*/ { - PyObject *typ, *state, *meth, *obj, *result; + PyObject *meth, *obj, *result; - if (!PyArg_ParseTuple(args, "OO!", &typ, &PyTuple_Type, &state)) - return NULL; obj = PyObject_CallMethodOneArg(typ, &_Py_ID(__new__), typ); if (obj == NULL) return NULL; @@ -1999,13 +2147,23 @@ unpickle(PyObject *self, PyObject *args) return NULL; } +/*[clinic input] +_ctypes.buffer_info + + obj as arg: object + / + +Return buffer interface information. +[clinic start generated code]*/ + static PyObject * -buffer_info(PyObject *self, PyObject *arg) +_ctypes_buffer_info(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=31f942a6fce6d671 input=cb023e7f9c959f72]*/ { PyObject *shape; Py_ssize_t i; - ctypes_state *st = get_module_state(self); + ctypes_state *st = get_module_state(module); StgInfo *info; if (PyStgInfo_FromAny(st, arg, &info) < 0) { return NULL; @@ -2030,13 +2188,8 @@ buffer_info(PyObject *self, PyObject *arg) static PyObject * -_ctypes_getattr(PyObject *Py_UNUSED(self), PyObject *args) +_ctypes_getattr(PyObject *Py_UNUSED(module), PyObject *name) { - PyObject *name; - if (!PyArg_UnpackTuple(args, "__getattr__", 1, 1, &name)) { - return NULL; - } - if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "__version__")) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "'__version__' is deprecated and slated for " @@ -2054,41 +2207,39 @@ _ctypes_getattr(PyObject *Py_UNUSED(self), PyObject *args) PyMethodDef _ctypes_module_methods[] = { - {"__getattr__", _ctypes_getattr, METH_VARARGS}, - {"get_errno", get_errno, METH_NOARGS}, - {"set_errno", set_errno, METH_VARARGS}, - {"_unpickle", unpickle, METH_VARARGS }, - {"buffer_info", buffer_info, METH_O, "Return buffer interface information"}, + {"__getattr__", _ctypes_getattr, METH_O}, + _CTYPES_GET_ERRNO_METHODDEF + _CTYPES_SET_ERRNO_METHODDEF + _CTYPES__UNPICKLE_METHODDEF + _CTYPES_BUFFER_INFO_METHODDEF _CTYPES_RESIZE_METHODDEF #ifdef MS_WIN32 - {"get_last_error", get_last_error, METH_NOARGS}, - {"set_last_error", set_last_error, METH_VARARGS}, - {"CopyComPointer", copy_com_pointer, METH_VARARGS, copy_com_pointer_doc}, - {"FormatError", format_error, METH_VARARGS, format_error_doc}, - {"LoadLibrary", load_library, METH_VARARGS, load_library_doc}, - {"FreeLibrary", free_library, METH_VARARGS, free_library_doc}, - {"_check_HRESULT", check_hresult, METH_VARARGS}, + _CTYPES_GET_LAST_ERROR_METHODDEF + _CTYPES_SET_LAST_ERROR_METHODDEF + _CTYPES_COPYCOMPOINTER_METHODDEF + _CTYPES_FORMATERROR_METHODDEF + _CTYPES_LOADLIBRARY_METHODDEF + _CTYPES_FREELIBRARY_METHODDEF + _CTYPES__CHECK_HRESULT_METHODDEF #else - {"dlopen", py_dl_open, METH_VARARGS, - "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, - {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, - {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, + _CTYPES_DLOPEN_METHODDEF + _CTYPES_DLCLOSE_METHODDEF + _CTYPES_DLSYM_METHODDEF #if defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__) - {"dllist", dllist, METH_NOARGS, - "dllist() return a list of loaded shared libraries"}, + _CTYPES_DLLIST_METHODDEF #endif #endif #ifdef __APPLE__ - {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, + _CTYPES__DYLD_SHARED_CACHE_CONTAINS_PATH_METHODDEF #endif - {"alignment", align_func, METH_O, alignment_doc}, + _CTYPES_ALIGNMENT_METHODDEF _CTYPES_SIZEOF_METHODDEF _CTYPES_BYREF_METHODDEF _CTYPES_ADDRESSOF_METHODDEF - {"call_function", call_function, METH_VARARGS }, - {"call_cdeclfunction", call_cdeclfunction, METH_VARARGS }, - {"PyObj_FromPtr", My_PyObj_FromPtr, METH_VARARGS }, - {"Py_INCREF", My_Py_INCREF, METH_O }, - {"Py_DECREF", My_Py_DECREF, METH_O }, + _CTYPES_CALL_FUNCTION_METHODDEF + _CTYPES_CALL_CDECLFUNCTION_METHODDEF + _CTYPES_PYOBJ_FROMPTR_METHODDEF + _CTYPES_PY_INCREF_METHODDEF + _CTYPES_PY_DECREF_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Modules/_ctypes/clinic/callproc.c.h b/Modules/_ctypes/clinic/callproc.c.h index e0cfcc6f38def7e..ef68372110416a1 100644 --- a/Modules/_ctypes/clinic/callproc.c.h +++ b/Modules/_ctypes/clinic/callproc.c.h @@ -6,6 +6,516 @@ preserve #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(_ctypes_get_errno__doc__, +"get_errno($module, /)\n" +"--\n" +"\n" +"Return the current value of the ctypes-private copy of errno."); + +#define _CTYPES_GET_ERRNO_METHODDEF \ + {"get_errno", (PyCFunction)_ctypes_get_errno, METH_NOARGS, _ctypes_get_errno__doc__}, + +static PyObject * +_ctypes_get_errno_impl(PyObject *module); + +static PyObject * +_ctypes_get_errno(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _ctypes_get_errno_impl(module); +} + +PyDoc_STRVAR(_ctypes_set_errno__doc__, +"set_errno($module, value, /)\n" +"--\n" +"\n"); + +#define _CTYPES_SET_ERRNO_METHODDEF \ + {"set_errno", (PyCFunction)_ctypes_set_errno, METH_O, _ctypes_set_errno__doc__}, + +static PyObject * +_ctypes_set_errno_impl(PyObject *module, int value); + +static PyObject * +_ctypes_set_errno(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int value; + + value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _ctypes_set_errno_impl(module, value); + +exit: + return return_value; +} + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_get_last_error__doc__, +"get_last_error($module, /)\n" +"--\n" +"\n" +"Return the current value of the ctypes-private copy of the last error."); + +#define _CTYPES_GET_LAST_ERROR_METHODDEF \ + {"get_last_error", (PyCFunction)_ctypes_get_last_error, METH_NOARGS, _ctypes_get_last_error__doc__}, + +static PyObject * +_ctypes_get_last_error_impl(PyObject *module); + +static PyObject * +_ctypes_get_last_error(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _ctypes_get_last_error_impl(module); +} + +#endif /* defined(MS_WIN32) */ + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_set_last_error__doc__, +"set_last_error($module, value, /)\n" +"--\n" +"\n"); + +#define _CTYPES_SET_LAST_ERROR_METHODDEF \ + {"set_last_error", (PyCFunction)_ctypes_set_last_error, METH_O, _ctypes_set_last_error__doc__}, + +static PyObject * +_ctypes_set_last_error_impl(PyObject *module, int value); + +static PyObject * +_ctypes_set_last_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int value; + + value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _ctypes_set_last_error_impl(module, value); + +exit: + return return_value; +} + +#endif /* defined(MS_WIN32) */ + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes__check_HRESULT__doc__, +"_check_HRESULT($module, hresult, /)\n" +"--\n" +"\n"); + +#define _CTYPES__CHECK_HRESULT_METHODDEF \ + {"_check_HRESULT", (PyCFunction)_ctypes__check_HRESULT, METH_O, _ctypes__check_HRESULT__doc__}, + +static PyObject * +_ctypes__check_HRESULT_impl(PyObject *module, int hr); + +static PyObject * +_ctypes__check_HRESULT(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int hr; + + hr = PyLong_AsInt(arg); + if (hr == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _ctypes__check_HRESULT_impl(module, hr); + +exit: + return return_value; +} + +#endif /* defined(MS_WIN32) */ + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_FormatError__doc__, +"FormatError($module, code=0, /)\n" +"--\n" +"\n" +"Convert a win32 error code into a string.\n" +"\n" +"If the error code is not given, the return value of a call to\n" +"GetLastError() is used."); + +#define _CTYPES_FORMATERROR_METHODDEF \ + {"FormatError", _PyCFunction_CAST(_ctypes_FormatError), METH_FASTCALL, _ctypes_FormatError__doc__}, + +static PyObject * +_ctypes_FormatError_impl(PyObject *module, int code); + +static PyObject * +_ctypes_FormatError(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int code = 0; + + if (!_PyArg_CheckPositional("FormatError", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + code = PyLong_AsInt(args[0]); + if (code == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _ctypes_FormatError_impl(module, code); + +exit: + return return_value; +} + +#endif /* defined(MS_WIN32) */ + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_LoadLibrary__doc__, +"LoadLibrary($module, name, load_flags=0, /)\n" +"--\n" +"\n" +"Load an executable (usually a DLL), and return a handle to it.\n" +"\n" +"The handle may be used to locate exported functions in this module.\n" +"load_flags are as defined for LoadLibraryEx in the Windows API."); + +#define _CTYPES_LOADLIBRARY_METHODDEF \ + {"LoadLibrary", _PyCFunction_CAST(_ctypes_LoadLibrary), METH_FASTCALL, _ctypes_LoadLibrary__doc__}, + +static PyObject * +_ctypes_LoadLibrary_impl(PyObject *module, PyObject *nameobj, int load_flags); + +static PyObject * +_ctypes_LoadLibrary(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *nameobj; + int load_flags = 0; + + if (!_PyArg_CheckPositional("LoadLibrary", nargs, 1, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("LoadLibrary", "argument 1", "str", args[0]); + goto exit; + } + nameobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + load_flags = PyLong_AsInt(args[1]); + if (load_flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _ctypes_LoadLibrary_impl(module, nameobj, load_flags); + +exit: + return return_value; +} + +#endif /* defined(MS_WIN32) */ + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_FreeLibrary__doc__, +"FreeLibrary($module, handle, /)\n" +"--\n" +"\n" +"Free the handle of an executable previously loaded by LoadLibrary."); + +#define _CTYPES_FREELIBRARY_METHODDEF \ + {"FreeLibrary", (PyCFunction)_ctypes_FreeLibrary, METH_O, _ctypes_FreeLibrary__doc__}, + +static PyObject * +_ctypes_FreeLibrary_impl(PyObject *module, void *hMod); + +static PyObject * +_ctypes_FreeLibrary(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + void *hMod; + + if (!_parse_voidp(arg, &hMod)) { + goto exit; + } + return_value = _ctypes_FreeLibrary_impl(module, hMod); + +exit: + return return_value; +} + +#endif /* defined(MS_WIN32) */ + +#if defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_CopyComPointer__doc__, +"CopyComPointer($module, src, dst, /)\n" +"--\n" +"\n" +"Copy a COM pointer and return the HRESULT value."); + +#define _CTYPES_COPYCOMPOINTER_METHODDEF \ + {"CopyComPointer", _PyCFunction_CAST(_ctypes_CopyComPointer), METH_FASTCALL, _ctypes_CopyComPointer__doc__}, + +static PyObject * +_ctypes_CopyComPointer_impl(PyObject *module, PyObject *p1, PyObject *p2); + +static PyObject * +_ctypes_CopyComPointer(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *p1; + PyObject *p2; + + if (!_PyArg_CheckPositional("CopyComPointer", nargs, 2, 2)) { + goto exit; + } + p1 = args[0]; + p2 = args[1]; + return_value = _ctypes_CopyComPointer_impl(module, p1, p2); + +exit: + return return_value; +} + +#endif /* defined(MS_WIN32) */ + +#if !defined(MS_WIN32) && defined(__APPLE__) + +PyDoc_STRVAR(_ctypes__dyld_shared_cache_contains_path__doc__, +"_dyld_shared_cache_contains_path($module, path, /)\n" +"--\n" +"\n" +"Check whether a path is in the shared cache."); + +#define _CTYPES__DYLD_SHARED_CACHE_CONTAINS_PATH_METHODDEF \ + {"_dyld_shared_cache_contains_path", (PyCFunction)_ctypes__dyld_shared_cache_contains_path, METH_O, _ctypes__dyld_shared_cache_contains_path__doc__}, + +#endif /* !defined(MS_WIN32) && defined(__APPLE__) */ + +#if !defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_dlopen__doc__, +"dlopen($module, name, mode=RTLD_NOW | RTLD_LOCAL, /)\n" +"--\n" +"\n" +"Open a shared library."); + +#define _CTYPES_DLOPEN_METHODDEF \ + {"dlopen", _PyCFunction_CAST(_ctypes_dlopen), METH_FASTCALL, _ctypes_dlopen__doc__}, + +static PyObject * +_ctypes_dlopen_impl(PyObject *module, PyObject *name, int mode); + +static PyObject * +_ctypes_dlopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *name; + int mode = DLOPEN_DEFAULT_MODE; + + if (!_PyArg_CheckPositional("dlopen", nargs, 1, 2)) { + goto exit; + } + name = args[0]; + if (nargs < 2) { + goto skip_optional; + } + mode = PyLong_AsInt(args[1]); + if (mode == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _ctypes_dlopen_impl(module, name, mode); + +exit: + return return_value; +} + +#endif /* !defined(MS_WIN32) */ + +#if !defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_dlclose__doc__, +"dlclose($module, handle, /)\n" +"--\n" +"\n" +"Close a shared library."); + +#define _CTYPES_DLCLOSE_METHODDEF \ + {"dlclose", (PyCFunction)_ctypes_dlclose, METH_O, _ctypes_dlclose__doc__}, + +static PyObject * +_ctypes_dlclose_impl(PyObject *module, void *handle); + +static PyObject * +_ctypes_dlclose(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + void *handle; + + if (!_parse_voidp(arg, &handle)) { + goto exit; + } + return_value = _ctypes_dlclose_impl(module, handle); + +exit: + return return_value; +} + +#endif /* !defined(MS_WIN32) */ + +#if !defined(MS_WIN32) + +PyDoc_STRVAR(_ctypes_dlsym__doc__, +"dlsym($module, handle, name, /)\n" +"--\n" +"\n" +"Find a symbol in a shared library."); + +#define _CTYPES_DLSYM_METHODDEF \ + {"dlsym", _PyCFunction_CAST(_ctypes_dlsym), METH_FASTCALL, _ctypes_dlsym__doc__}, + +static PyObject * +_ctypes_dlsym_impl(PyObject *module, void *handle, const char *name); + +static PyObject * +_ctypes_dlsym(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + void *handle; + const char *name; + + if (!_PyArg_CheckPositional("dlsym", nargs, 2, 2)) { + goto exit; + } + if (!_parse_voidp(args[0], &handle)) { + goto exit; + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("dlsym", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = _ctypes_dlsym_impl(module, handle, name); + +exit: + return return_value; +} + +#endif /* !defined(MS_WIN32) */ + +#if !defined(MS_WIN32) && (defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__)) + +PyDoc_STRVAR(_ctypes_dllist__doc__, +"dllist($module, /)\n" +"--\n" +"\n" +"Return a list of loaded shared libraries."); + +#define _CTYPES_DLLIST_METHODDEF \ + {"dllist", (PyCFunction)_ctypes_dllist, METH_NOARGS, _ctypes_dllist__doc__}, + +static PyObject * +_ctypes_dllist_impl(PyObject *module); + +static PyObject * +_ctypes_dllist(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _ctypes_dllist_impl(module); +} + +#endif /* !defined(MS_WIN32) && (defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__)) */ + +PyDoc_STRVAR(_ctypes_call_function__doc__, +"call_function($module, func, arguments, /)\n" +"--\n" +"\n"); + +#define _CTYPES_CALL_FUNCTION_METHODDEF \ + {"call_function", _PyCFunction_CAST(_ctypes_call_function), METH_FASTCALL, _ctypes_call_function__doc__}, + +static PyObject * +_ctypes_call_function_impl(PyObject *module, void *func, PyObject *arguments); + +static PyObject * +_ctypes_call_function(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + void *func; + PyObject *arguments; + + if (!_PyArg_CheckPositional("call_function", nargs, 2, 2)) { + goto exit; + } + if (!_parse_voidp(args[0], &func)) { + goto exit; + } + if (!PyTuple_Check(args[1])) { + _PyArg_BadArgument("call_function", "argument 2", "tuple", args[1]); + goto exit; + } + arguments = args[1]; + return_value = _ctypes_call_function_impl(module, func, arguments); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ctypes_call_cdeclfunction__doc__, +"call_cdeclfunction($module, func, arguments, /)\n" +"--\n" +"\n"); + +#define _CTYPES_CALL_CDECLFUNCTION_METHODDEF \ + {"call_cdeclfunction", _PyCFunction_CAST(_ctypes_call_cdeclfunction), METH_FASTCALL, _ctypes_call_cdeclfunction__doc__}, + +static PyObject * +_ctypes_call_cdeclfunction_impl(PyObject *module, void *func, + PyObject *arguments); + +static PyObject * +_ctypes_call_cdeclfunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + void *func; + PyObject *arguments; + + if (!_PyArg_CheckPositional("call_cdeclfunction", nargs, 2, 2)) { + goto exit; + } + if (!_parse_voidp(args[0], &func)) { + goto exit; + } + if (!PyTuple_Check(args[1])) { + _PyArg_BadArgument("call_cdeclfunction", "argument 2", "tuple", args[1]); + goto exit; + } + arguments = args[1]; + return_value = _ctypes_call_cdeclfunction_impl(module, func, arguments); + +exit: + return return_value; +} + PyDoc_STRVAR(_ctypes_sizeof__doc__, "sizeof($module, obj, /)\n" "--\n" @@ -15,6 +525,17 @@ PyDoc_STRVAR(_ctypes_sizeof__doc__, #define _CTYPES_SIZEOF_METHODDEF \ {"sizeof", (PyCFunction)_ctypes_sizeof, METH_O, _ctypes_sizeof__doc__}, +PyDoc_STRVAR(_ctypes_alignment__doc__, +"alignment($module, obj, /)\n" +"--\n" +"\n" +"Return the alignment requirements of a C instance.\n" +"\n" +"The argument is a C type or a C instance."); + +#define _CTYPES_ALIGNMENT_METHODDEF \ + {"alignment", (PyCFunction)_ctypes_alignment, METH_O, _ctypes_alignment__doc__}, + PyDoc_STRVAR(_ctypes_byref__doc__, "byref($module, obj, offset=0, /)\n" "--\n" @@ -97,6 +618,50 @@ _ctypes_addressof(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_ctypes_PyObj_FromPtr__doc__, +"PyObj_FromPtr($module, address, /)\n" +"--\n" +"\n"); + +#define _CTYPES_PYOBJ_FROMPTR_METHODDEF \ + {"PyObj_FromPtr", (PyCFunction)_ctypes_PyObj_FromPtr, METH_O, _ctypes_PyObj_FromPtr__doc__}, + +static PyObject * +_ctypes_PyObj_FromPtr_impl(PyObject *module, PyObject *ob); + +static PyObject * +_ctypes_PyObj_FromPtr(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *ob; + + if (!_parse_voidp_object(arg, &ob)) { + goto exit; + } + return_value = _ctypes_PyObj_FromPtr_impl(module, ob); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ctypes_Py_INCREF__doc__, +"Py_INCREF($module, obj, /)\n" +"--\n" +"\n" +"Increment the reference count of the object and return it."); + +#define _CTYPES_PY_INCREF_METHODDEF \ + {"Py_INCREF", (PyCFunction)_ctypes_Py_INCREF, METH_O, _ctypes_Py_INCREF__doc__}, + +PyDoc_STRVAR(_ctypes_Py_DECREF__doc__, +"Py_DECREF($module, obj, /)\n" +"--\n" +"\n" +"Decrement the reference count of the object and return it."); + +#define _CTYPES_PY_DECREF_METHODDEF \ + {"Py_DECREF", (PyCFunction)_ctypes_Py_DECREF, METH_O, _ctypes_Py_DECREF__doc__}, + PyDoc_STRVAR(_ctypes_resize__doc__, "resize($module, obj, size, /)\n" "--\n" @@ -142,4 +707,94 @@ _ctypes_resize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=23c74aced603977d input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_ctypes__unpickle__doc__, +"_unpickle($module, cls, state, /)\n" +"--\n" +"\n"); + +#define _CTYPES__UNPICKLE_METHODDEF \ + {"_unpickle", _PyCFunction_CAST(_ctypes__unpickle), METH_FASTCALL, _ctypes__unpickle__doc__}, + +static PyObject * +_ctypes__unpickle_impl(PyObject *module, PyObject *typ, PyObject *state); + +static PyObject * +_ctypes__unpickle(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *typ; + PyObject *state; + + if (!_PyArg_CheckPositional("_unpickle", nargs, 2, 2)) { + goto exit; + } + typ = args[0]; + if (!PyTuple_Check(args[1])) { + _PyArg_BadArgument("_unpickle", "argument 2", "tuple", args[1]); + goto exit; + } + state = args[1]; + return_value = _ctypes__unpickle_impl(module, typ, state); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ctypes_buffer_info__doc__, +"buffer_info($module, obj, /)\n" +"--\n" +"\n" +"Return buffer interface information."); + +#define _CTYPES_BUFFER_INFO_METHODDEF \ + {"buffer_info", (PyCFunction)_ctypes_buffer_info, METH_O, _ctypes_buffer_info__doc__}, + +#ifndef _CTYPES_GET_LAST_ERROR_METHODDEF + #define _CTYPES_GET_LAST_ERROR_METHODDEF +#endif /* !defined(_CTYPES_GET_LAST_ERROR_METHODDEF) */ + +#ifndef _CTYPES_SET_LAST_ERROR_METHODDEF + #define _CTYPES_SET_LAST_ERROR_METHODDEF +#endif /* !defined(_CTYPES_SET_LAST_ERROR_METHODDEF) */ + +#ifndef _CTYPES__CHECK_HRESULT_METHODDEF + #define _CTYPES__CHECK_HRESULT_METHODDEF +#endif /* !defined(_CTYPES__CHECK_HRESULT_METHODDEF) */ + +#ifndef _CTYPES_FORMATERROR_METHODDEF + #define _CTYPES_FORMATERROR_METHODDEF +#endif /* !defined(_CTYPES_FORMATERROR_METHODDEF) */ + +#ifndef _CTYPES_LOADLIBRARY_METHODDEF + #define _CTYPES_LOADLIBRARY_METHODDEF +#endif /* !defined(_CTYPES_LOADLIBRARY_METHODDEF) */ + +#ifndef _CTYPES_FREELIBRARY_METHODDEF + #define _CTYPES_FREELIBRARY_METHODDEF +#endif /* !defined(_CTYPES_FREELIBRARY_METHODDEF) */ + +#ifndef _CTYPES_COPYCOMPOINTER_METHODDEF + #define _CTYPES_COPYCOMPOINTER_METHODDEF +#endif /* !defined(_CTYPES_COPYCOMPOINTER_METHODDEF) */ + +#ifndef _CTYPES__DYLD_SHARED_CACHE_CONTAINS_PATH_METHODDEF + #define _CTYPES__DYLD_SHARED_CACHE_CONTAINS_PATH_METHODDEF +#endif /* !defined(_CTYPES__DYLD_SHARED_CACHE_CONTAINS_PATH_METHODDEF) */ + +#ifndef _CTYPES_DLOPEN_METHODDEF + #define _CTYPES_DLOPEN_METHODDEF +#endif /* !defined(_CTYPES_DLOPEN_METHODDEF) */ + +#ifndef _CTYPES_DLCLOSE_METHODDEF + #define _CTYPES_DLCLOSE_METHODDEF +#endif /* !defined(_CTYPES_DLCLOSE_METHODDEF) */ + +#ifndef _CTYPES_DLSYM_METHODDEF + #define _CTYPES_DLSYM_METHODDEF +#endif /* !defined(_CTYPES_DLSYM_METHODDEF) */ + +#ifndef _CTYPES_DLLIST_METHODDEF + #define _CTYPES_DLLIST_METHODDEF +#endif /* !defined(_CTYPES_DLLIST_METHODDEF) */ +/*[clinic end generated code: output=71a41a6d90e69821 input=a9049054013a1b77]*/ diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 5537947f6a51c11..2cc3774e75c9dad 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2333,94 +2333,143 @@ bufferedrwpair_dealloc(PyObject *op) Py_DECREF(tp); } +/* Call the method of the underlying reader or writer. The argument is + only passed if it is not NULL, so that the default of that method is + used otherwise. */ static PyObject * -_forward_call(buffered *self, PyObject *name, PyObject *args) +_forward_call(buffered *self, PyObject *name, PyObject *arg) { - PyObject *func, *ret; if (self == NULL) { PyErr_SetString(PyExc_ValueError, "I/O operation on uninitialized object"); return NULL; } - func = PyObject_GetAttr((PyObject *)self, name); - if (func == NULL) { - PyErr_SetObject(PyExc_AttributeError, name); - return NULL; + if (arg == NULL) { + return PyObject_CallMethodNoArgs((PyObject *)self, name); } - - ret = PyObject_CallObject(func, args); - Py_DECREF(func); - return ret; + return PyObject_CallMethodOneArg((PyObject *)self, name, arg); } +/*[clinic input] +_io.BufferedRWPair.read + size: object(c_default="NULL") = -1 + / +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_read(PyObject *op, PyObject *args) +_io_BufferedRWPair_read_impl(rwpair *self, PyObject *size) +/*[clinic end generated code: output=0668e3c5dbd3e93d input=eddb5e52aba9ebe5]*/ { - rwpair *self = rwpair_CAST(op); - return _forward_call(self->reader, &_Py_ID(read), args); + return _forward_call(self->reader, &_Py_ID(read), size); } +/*[clinic input] +_io.BufferedRWPair.peek + size: object(c_default="NULL") = 0 + / +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_peek(PyObject *op, PyObject *args) +_io_BufferedRWPair_peek_impl(rwpair *self, PyObject *size) +/*[clinic end generated code: output=190a267bd694efa0 input=36af95964bebe355]*/ { - rwpair *self = rwpair_CAST(op); - return _forward_call(self->reader, &_Py_ID(peek), args); + return _forward_call(self->reader, &_Py_ID(peek), size); } +/*[clinic input] +_io.BufferedRWPair.read1 + size: object(c_default="NULL") = -1 + / +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_read1(PyObject *op, PyObject *args) +_io_BufferedRWPair_read1_impl(rwpair *self, PyObject *size) +/*[clinic end generated code: output=17ec19608f2bb825 input=9e94db423e490b58]*/ { - rwpair *self = rwpair_CAST(op); - return _forward_call(self->reader, &_Py_ID(read1), args); + return _forward_call(self->reader, &_Py_ID(read1), size); } +/*[clinic input] +_io.BufferedRWPair.readinto + buffer: object + / +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_readinto(PyObject *op, PyObject *args) +_io_BufferedRWPair_readinto_impl(rwpair *self, PyObject *buffer) +/*[clinic end generated code: output=16c86b071015f7a4 input=ccd86ce2666261f7]*/ { - rwpair *self = rwpair_CAST(op); - return _forward_call(self->reader, &_Py_ID(readinto), args); + return _forward_call(self->reader, &_Py_ID(readinto), buffer); } +/*[clinic input] +_io.BufferedRWPair.readinto1 + buffer: object + / +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_readinto1(PyObject *op, PyObject *args) +_io_BufferedRWPair_readinto1_impl(rwpair *self, PyObject *buffer) +/*[clinic end generated code: output=f1577b6f54c2b02a input=613d9bf127f88a4a]*/ { - rwpair *self = rwpair_CAST(op); - return _forward_call(self->reader, &_Py_ID(readinto1), args); + return _forward_call(self->reader, &_Py_ID(readinto1), buffer); } +/*[clinic input] +_io.BufferedRWPair.write + buffer: object + / +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_write(PyObject *op, PyObject *args) +_io_BufferedRWPair_write_impl(rwpair *self, PyObject *buffer) +/*[clinic end generated code: output=6f7509a747410c68 input=66c602422e3ec36f]*/ { - rwpair *self = rwpair_CAST(op); - return _forward_call(self->writer, &_Py_ID(write), args); + return _forward_call(self->writer, &_Py_ID(write), buffer); } +/*[clinic input] +_io.BufferedRWPair.flush +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_flush(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_BufferedRWPair_flush_impl(rwpair *self) +/*[clinic end generated code: output=0b2dcbe828718d6b input=e853da796ee61df1]*/ { - rwpair *self = rwpair_CAST(op); return _forward_call(self->writer, &_Py_ID(flush), NULL); } +/*[clinic input] +_io.BufferedRWPair.readable +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_readable(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_BufferedRWPair_readable_impl(rwpair *self) +/*[clinic end generated code: output=615967d4aa58f122 input=0475ed73d0a3167f]*/ { - rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(readable), NULL); } +/*[clinic input] +_io.BufferedRWPair.writable +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_writable(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_BufferedRWPair_writable_impl(rwpair *self) +/*[clinic end generated code: output=c5a43c84e0195c11 input=3cfd44fb4757082f]*/ { - rwpair *self = rwpair_CAST(op); return _forward_call(self->writer, &_Py_ID(writable), NULL); } +/*[clinic input] +_io.BufferedRWPair.close +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_close(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_BufferedRWPair_close_impl(rwpair *self) +/*[clinic end generated code: output=5924ba5ecc78752a input=4087d69f2d8fc368]*/ { - rwpair *self = rwpair_CAST(op); PyObject *exc = NULL; PyObject *ret = _forward_call(self->writer, &_Py_ID(close), NULL); if (ret == NULL) { @@ -2437,10 +2486,14 @@ bufferedrwpair_close(PyObject *op, PyObject *Py_UNUSED(dummy)) return ret; } +/*[clinic input] +_io.BufferedRWPair.isatty +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_isatty(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_BufferedRWPair_isatty_impl(rwpair *self) +/*[clinic end generated code: output=d017c621ed879cb7 input=92833e3d60586e14]*/ { - rwpair *self = rwpair_CAST(op); PyObject *ret = _forward_call(self->writer, &_Py_ID(isatty), NULL); if (ret != Py_False) { @@ -2452,10 +2505,15 @@ bufferedrwpair_isatty(PyObject *op, PyObject *Py_UNUSED(dummy)) return _forward_call(self->reader, &_Py_ID(isatty), NULL); } +/*[clinic input] +@getter +_io.BufferedRWPair.closed +[clinic start generated code]*/ + static PyObject * -bufferedrwpair_closed_get(PyObject *op, void *Py_UNUSED(dummy)) +_io_BufferedRWPair_closed_get_impl(rwpair *self) +/*[clinic end generated code: output=4117400c74766f21 input=8248430ac54e5b25]*/ { - rwpair *self = rwpair_CAST(op); if (self->writer == NULL) { PyErr_SetString(PyExc_RuntimeError, "the BufferedRWPair object is being garbage-collected"); @@ -2670,20 +2728,20 @@ PyType_Spec _Py_bufferedwriter_spec = { }; static PyMethodDef bufferedrwpair_methods[] = { - {"read", bufferedrwpair_read, METH_VARARGS}, - {"peek", bufferedrwpair_peek, METH_VARARGS}, - {"read1", bufferedrwpair_read1, METH_VARARGS}, - {"readinto", bufferedrwpair_readinto, METH_VARARGS}, - {"readinto1", bufferedrwpair_readinto1, METH_VARARGS}, + _IO_BUFFEREDRWPAIR_READ_METHODDEF + _IO_BUFFEREDRWPAIR_PEEK_METHODDEF + _IO_BUFFEREDRWPAIR_READ1_METHODDEF + _IO_BUFFEREDRWPAIR_READINTO_METHODDEF + _IO_BUFFEREDRWPAIR_READINTO1_METHODDEF - {"write", bufferedrwpair_write, METH_VARARGS}, - {"flush", bufferedrwpair_flush, METH_NOARGS}, + _IO_BUFFEREDRWPAIR_WRITE_METHODDEF + _IO_BUFFEREDRWPAIR_FLUSH_METHODDEF - {"readable", bufferedrwpair_readable, METH_NOARGS}, - {"writable", bufferedrwpair_writable, METH_NOARGS}, + _IO_BUFFEREDRWPAIR_READABLE_METHODDEF + _IO_BUFFEREDRWPAIR_WRITABLE_METHODDEF - {"close", bufferedrwpair_close, METH_NOARGS}, - {"isatty", bufferedrwpair_isatty, METH_NOARGS}, + _IO_BUFFEREDRWPAIR_CLOSE_METHODDEF + _IO_BUFFEREDRWPAIR_ISATTY_METHODDEF {NULL, NULL} }; @@ -2695,7 +2753,7 @@ static PyMemberDef bufferedrwpair_members[] = { }; static PyGetSetDef bufferedrwpair_getset[] = { - {"closed", bufferedrwpair_closed_get, NULL, NULL}, + _IO_BUFFEREDRWPAIR_CLOSED_GETSETDEF {NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index b7c1555c2637dd5..ff9cc0f2166bd74 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -297,20 +297,19 @@ write_bytes_lock_held(bytesio *self, PyObject *b) return len; } -static PyObject * -bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) +/*[clinic input] +@critical_section +@getter +_io.BytesIO.closed -> bool + +True if the file is closed. +[clinic start generated code]*/ + +static int +_io_BytesIO_closed_get_impl(bytesio *self) +/*[clinic end generated code: output=7cdc647cc7a71683 input=0687f923344225ee]*/ { - PyObject *ret; - bytesio *self = bytesio_CAST(op); - Py_BEGIN_CRITICAL_SECTION(self); - if (self->buf == NULL) { - ret = Py_True; - } - else { - ret = Py_False; - } - Py_END_CRITICAL_SECTION(); - return ret; + return self->buf == NULL; } /*[clinic input] @@ -950,54 +949,55 @@ _io_BytesIO_close_impl(bytesio *self) function to use the efficient instance representation of PEP 307. */ - static PyObject * - bytesio_getstate_lock_held(PyObject *op) - { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - - bytesio *self = bytesio_CAST(op); - PyObject *initvalue = _io_BytesIO_getvalue_impl(self); - PyObject *dict; - PyObject *state; - - if (initvalue == NULL) - return NULL; - if (self->dict == NULL) { - dict = Py_NewRef(Py_None); - } - else { - dict = PyDict_Copy(self->dict); - if (dict == NULL) { - Py_DECREF(initvalue); - return NULL; - } - } - - state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); - Py_DECREF(initvalue); - return state; -} + +/*[clinic input] +@critical_section +_io.BytesIO.__getstate__ +[clinic start generated code]*/ static PyObject * -bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_BytesIO___getstate___impl(bytesio *self) +/*[clinic end generated code: output=4a776270c8443b85 input=f41e5bc9731475c4]*/ { - PyObject *ret; - Py_BEGIN_CRITICAL_SECTION(op); - ret = bytesio_getstate_lock_held(op); - Py_END_CRITICAL_SECTION(); - return ret; + PyObject *initvalue = _io_BytesIO_getvalue_impl(self); + PyObject *dict; + PyObject *state; + + if (initvalue == NULL) + return NULL; + if (self->dict == NULL) { + dict = Py_NewRef(Py_None); + } + else { + dict = PyDict_Copy(self->dict); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + } + + state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); + Py_DECREF(initvalue); + return state; } + +/*[clinic input] +@critical_section +_io.BytesIO.__setstate__ + + state: object + / +[clinic start generated code]*/ + static PyObject * -bytesio_setstate_lock_held(PyObject *op, PyObject *state) +_io_BytesIO___setstate___impl(bytesio *self, PyObject *state) +/*[clinic end generated code: output=3605abdec171bb98 input=82a189599ba75083]*/ { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - PyObject *result; PyObject *position_obj; PyObject *dict; Py_ssize_t pos; - bytesio *self = bytesio_CAST(op); assert(state != NULL); @@ -1068,16 +1068,6 @@ bytesio_setstate_lock_held(PyObject *op, PyObject *state) Py_RETURN_NONE; } -static PyObject * -bytesio_setstate(PyObject *op, PyObject *state) -{ - PyObject *ret; - Py_BEGIN_CRITICAL_SECTION(op); - ret = bytesio_setstate_lock_held(op, state); - Py_END_CRITICAL_SECTION(); - return ret; -} - static void bytesio_dealloc(PyObject *op) { @@ -1159,12 +1149,18 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) return 0; } + +/*[clinic input] +@critical_section +_io.BytesIO.__sizeof__ + +Size of object in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -bytesio_sizeof_lock_held(PyObject *op) +_io_BytesIO___sizeof___impl(bytesio *self) +/*[clinic end generated code: output=f61b601bd055c4de input=6f01c36e6ff64c17]*/ { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - - bytesio *self = bytesio_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { size_t s = _PySys_GetSizeOf(self->buf); @@ -1176,16 +1172,6 @@ bytesio_sizeof_lock_held(PyObject *op) return PyLong_FromSize_t(res); } -static PyObject * -bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) -{ - PyObject *ret; - Py_BEGIN_CRITICAL_SECTION(op); - ret = bytesio_sizeof_lock_held(op); - Py_END_CRITICAL_SECTION(); - return ret; -} - static int bytesio_traverse(PyObject *op, visitproc visit, void *arg) { @@ -1213,8 +1199,7 @@ bytesio_clear(PyObject *op) #undef clinic_state static PyGetSetDef bytesio_getsetlist[] = { - {"closed", bytesio_get_closed, NULL, - "True if the file is closed."}, + _IO_BYTESIO_CLOSED_GETSETDEF {NULL}, /* sentinel */ }; @@ -1238,9 +1223,9 @@ static struct PyMethodDef bytesio_methods[] = { _IO_BYTESIO_GETVALUE_METHODDEF _IO_BYTESIO_SEEK_METHODDEF _IO_BYTESIO_TRUNCATE_METHODDEF - {"__getstate__", bytesio_getstate, METH_NOARGS, NULL}, - {"__setstate__", bytesio_setstate, METH_O, NULL}, - {"__sizeof__", bytesio_sizeof, METH_NOARGS, NULL}, + _IO_BYTESIO___GETSTATE___METHODDEF + _IO_BYTESIO___SETSTATE___METHODDEF + _IO_BYTESIO___SIZEOF___METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 3ca28c5a390736b..7575be251273f0d 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -1156,6 +1156,256 @@ _io_BufferedRWPair___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(_io_BufferedRWPair_read__doc__, +"read($self, size=-1, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_READ_METHODDEF \ + {"read", _PyCFunction_CAST(_io_BufferedRWPair_read), METH_FASTCALL, _io_BufferedRWPair_read__doc__}, + +static PyObject * +_io_BufferedRWPair_read_impl(rwpair *self, PyObject *size); + +static PyObject * +_io_BufferedRWPair_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *size = NULL; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + size = args[0]; +skip_optional: + return_value = _io_BufferedRWPair_read_impl((rwpair *)self, size); + +exit: + return return_value; +} + +PyDoc_STRVAR(_io_BufferedRWPair_peek__doc__, +"peek($self, size=0, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_PEEK_METHODDEF \ + {"peek", _PyCFunction_CAST(_io_BufferedRWPair_peek), METH_FASTCALL, _io_BufferedRWPair_peek__doc__}, + +static PyObject * +_io_BufferedRWPair_peek_impl(rwpair *self, PyObject *size); + +static PyObject * +_io_BufferedRWPair_peek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *size = NULL; + + if (!_PyArg_CheckPositional("peek", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + size = args[0]; +skip_optional: + return_value = _io_BufferedRWPair_peek_impl((rwpair *)self, size); + +exit: + return return_value; +} + +PyDoc_STRVAR(_io_BufferedRWPair_read1__doc__, +"read1($self, size=-1, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_READ1_METHODDEF \ + {"read1", _PyCFunction_CAST(_io_BufferedRWPair_read1), METH_FASTCALL, _io_BufferedRWPair_read1__doc__}, + +static PyObject * +_io_BufferedRWPair_read1_impl(rwpair *self, PyObject *size); + +static PyObject * +_io_BufferedRWPair_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *size = NULL; + + if (!_PyArg_CheckPositional("read1", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + size = args[0]; +skip_optional: + return_value = _io_BufferedRWPair_read1_impl((rwpair *)self, size); + +exit: + return return_value; +} + +PyDoc_STRVAR(_io_BufferedRWPair_readinto__doc__, +"readinto($self, buffer, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_READINTO_METHODDEF \ + {"readinto", (PyCFunction)_io_BufferedRWPair_readinto, METH_O, _io_BufferedRWPair_readinto__doc__}, + +static PyObject * +_io_BufferedRWPair_readinto_impl(rwpair *self, PyObject *buffer); + +static PyObject * +_io_BufferedRWPair_readinto(PyObject *self, PyObject *buffer) +{ + PyObject *return_value = NULL; + + return_value = _io_BufferedRWPair_readinto_impl((rwpair *)self, buffer); + + return return_value; +} + +PyDoc_STRVAR(_io_BufferedRWPair_readinto1__doc__, +"readinto1($self, buffer, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_READINTO1_METHODDEF \ + {"readinto1", (PyCFunction)_io_BufferedRWPair_readinto1, METH_O, _io_BufferedRWPair_readinto1__doc__}, + +static PyObject * +_io_BufferedRWPair_readinto1_impl(rwpair *self, PyObject *buffer); + +static PyObject * +_io_BufferedRWPair_readinto1(PyObject *self, PyObject *buffer) +{ + PyObject *return_value = NULL; + + return_value = _io_BufferedRWPair_readinto1_impl((rwpair *)self, buffer); + + return return_value; +} + +PyDoc_STRVAR(_io_BufferedRWPair_write__doc__, +"write($self, buffer, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_WRITE_METHODDEF \ + {"write", (PyCFunction)_io_BufferedRWPair_write, METH_O, _io_BufferedRWPair_write__doc__}, + +static PyObject * +_io_BufferedRWPair_write_impl(rwpair *self, PyObject *buffer); + +static PyObject * +_io_BufferedRWPair_write(PyObject *self, PyObject *buffer) +{ + PyObject *return_value = NULL; + + return_value = _io_BufferedRWPair_write_impl((rwpair *)self, buffer); + + return return_value; +} + +PyDoc_STRVAR(_io_BufferedRWPair_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_io_BufferedRWPair_flush, METH_NOARGS, _io_BufferedRWPair_flush__doc__}, + +static PyObject * +_io_BufferedRWPair_flush_impl(rwpair *self); + +static PyObject * +_io_BufferedRWPair_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_BufferedRWPair_flush_impl((rwpair *)self); +} + +PyDoc_STRVAR(_io_BufferedRWPair_readable__doc__, +"readable($self, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_READABLE_METHODDEF \ + {"readable", (PyCFunction)_io_BufferedRWPair_readable, METH_NOARGS, _io_BufferedRWPair_readable__doc__}, + +static PyObject * +_io_BufferedRWPair_readable_impl(rwpair *self); + +static PyObject * +_io_BufferedRWPair_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_BufferedRWPair_readable_impl((rwpair *)self); +} + +PyDoc_STRVAR(_io_BufferedRWPair_writable__doc__, +"writable($self, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_WRITABLE_METHODDEF \ + {"writable", (PyCFunction)_io_BufferedRWPair_writable, METH_NOARGS, _io_BufferedRWPair_writable__doc__}, + +static PyObject * +_io_BufferedRWPair_writable_impl(rwpair *self); + +static PyObject * +_io_BufferedRWPair_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_BufferedRWPair_writable_impl((rwpair *)self); +} + +PyDoc_STRVAR(_io_BufferedRWPair_close__doc__, +"close($self, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_CLOSE_METHODDEF \ + {"close", (PyCFunction)_io_BufferedRWPair_close, METH_NOARGS, _io_BufferedRWPair_close__doc__}, + +static PyObject * +_io_BufferedRWPair_close_impl(rwpair *self); + +static PyObject * +_io_BufferedRWPair_close(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_BufferedRWPair_close_impl((rwpair *)self); +} + +PyDoc_STRVAR(_io_BufferedRWPair_isatty__doc__, +"isatty($self, /)\n" +"--\n" +"\n"); + +#define _IO_BUFFEREDRWPAIR_ISATTY_METHODDEF \ + {"isatty", (PyCFunction)_io_BufferedRWPair_isatty, METH_NOARGS, _io_BufferedRWPair_isatty__doc__}, + +static PyObject * +_io_BufferedRWPair_isatty_impl(rwpair *self); + +static PyObject * +_io_BufferedRWPair_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_BufferedRWPair_isatty_impl((rwpair *)self); +} + +static PyObject * +_io_BufferedRWPair_closed_get_impl(rwpair *self); + +static PyObject * +_io_BufferedRWPair_closed_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io_BufferedRWPair_closed_get_impl((rwpair *)self); +} + PyDoc_STRVAR(_io_BufferedRandom___init____doc__, "BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE)\n" "--\n" @@ -1241,4 +1491,6 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) #define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, (setter)NULL, NULL}, -/*[clinic end generated code: output=cac02514680dfc60 input=a9049054013a1b77]*/ +#define _IO_BUFFEREDRWPAIR_CLOSED_GETSETDEF {"closed", (getter)_io_BufferedRWPair_closed_get, (setter)NULL, NULL}, + +/*[clinic end generated code: output=6fc10169822af939 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index a0159c063b58591..afdb60e506a2ce2 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -10,6 +10,30 @@ preserve #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(_io_BytesIO_closed__doc__, +"True if the file is closed."); + +static int +_io_BytesIO_closed_get_impl(bytesio *self); + +static PyObject * +_io_BytesIO_closed_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + int _return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + _return_value = _io_BytesIO_closed_get_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_io_BytesIO_readable__doc__, "readable($self, /)\n" "--\n" @@ -621,6 +645,52 @@ _io_BytesIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_io_BytesIO___getstate____doc__, +"__getstate__($self, /)\n" +"--\n" +"\n"); + +#define _IO_BYTESIO___GETSTATE___METHODDEF \ + {"__getstate__", (PyCFunction)_io_BytesIO___getstate__, METH_NOARGS, _io_BytesIO___getstate____doc__}, + +static PyObject * +_io_BytesIO___getstate___impl(bytesio *self); + +static PyObject * +_io_BytesIO___getstate__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO___getstate___impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_io_BytesIO___setstate____doc__, +"__setstate__($self, state, /)\n" +"--\n" +"\n"); + +#define _IO_BYTESIO___SETSTATE___METHODDEF \ + {"__setstate__", (PyCFunction)_io_BytesIO___setstate__, METH_O, _io_BytesIO___setstate____doc__}, + +static PyObject * +_io_BytesIO___setstate___impl(bytesio *self, PyObject *state); + +static PyObject * +_io_BytesIO___setstate__(PyObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO___setstate___impl((bytesio *)self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_io_BytesIO___init____doc__, "BytesIO(initial_bytes=b\'\')\n" "--\n" @@ -684,4 +754,30 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b5e625e31b2a82f0 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_io_BytesIO___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Size of object in memory, in bytes."); + +#define _IO_BYTESIO___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)_io_BytesIO___sizeof__, METH_NOARGS, _io_BytesIO___sizeof____doc__}, + +static PyObject * +_io_BytesIO___sizeof___impl(bytesio *self); + +static PyObject * +_io_BytesIO___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO___sizeof___impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +#define _IO_BYTESIO_CLOSED_GETSETDEF {"closed", (getter)_io_BytesIO_closed_get, (setter)NULL, _io_BytesIO_closed__doc__}, + +/*[clinic end generated code: output=a3dc7a416dae6ae9 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 890b6bc3fac9d55..207ad64fe62f958 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -9,6 +9,27 @@ preserve #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(_io_FileIO__dealloc_warn__doc__, +"_dealloc_warn($self, source, /)\n" +"--\n" +"\n"); + +#define _IO_FILEIO__DEALLOC_WARN_METHODDEF \ + {"_dealloc_warn", (PyCFunction)_io_FileIO__dealloc_warn, METH_O, _io_FileIO__dealloc_warn__doc__}, + +static PyObject * +_io_FileIO__dealloc_warn_impl(fileio *self, PyObject *source); + +static PyObject * +_io_FileIO__dealloc_warn(PyObject *self, PyObject *source) +{ + PyObject *return_value = NULL; + + return_value = _io_FileIO__dealloc_warn_impl((fileio *)self, source); + + return return_value; +} + PyDoc_STRVAR(_io_FileIO_close__doc__, "close($self, /)\n" "--\n" @@ -550,7 +571,110 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) return _io_FileIO_isatty_impl((fileio *)self); } +PyDoc_STRVAR(_io_FileIO__isatty_open_only__doc__, +"_isatty_open_only($self, /)\n" +"--\n" +"\n"); + +#define _IO_FILEIO__ISATTY_OPEN_ONLY_METHODDEF \ + {"_isatty_open_only", (PyCFunction)_io_FileIO__isatty_open_only, METH_NOARGS, _io_FileIO__isatty_open_only__doc__}, + +static PyObject * +_io_FileIO__isatty_open_only_impl(fileio *self); + +static PyObject * +_io_FileIO__isatty_open_only(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_FileIO__isatty_open_only_impl((fileio *)self); +} + +PyDoc_STRVAR(_io_FileIO_closed__doc__, +"True if the file is closed."); + +static int +_io_FileIO_closed_get_impl(fileio *self); + +static PyObject * +_io_FileIO_closed_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = _io_FileIO_closed_get_impl((fileio *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_io_FileIO_closefd__doc__, +"True if the file descriptor will be closed by close()."); + +static int +_io_FileIO_closefd_get_impl(fileio *self); + +static PyObject * +_io_FileIO_closefd_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = _io_FileIO_closefd_get_impl((fileio *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_io_FileIO_mode__doc__, +"String giving the file mode."); + +static PyObject * +_io_FileIO_mode_get_impl(fileio *self); + +static PyObject * +_io_FileIO_mode_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io_FileIO_mode_get_impl((fileio *)self); +} + +PyDoc_STRVAR(_io_FileIO__blksize__doc__, +"Stat st_blksize if available."); + +static long +_io_FileIO__blksize_get_impl(fileio *self); + +static PyObject * +_io_FileIO__blksize_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + long _return_value; + + _return_value = _io_FileIO__blksize_get_impl((fileio *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} + #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=453d584e2e72f986 input=a9049054013a1b77]*/ +#define _IO_FILEIO_CLOSED_GETSETDEF {"closed", (getter)_io_FileIO_closed_get, (setter)NULL, _io_FileIO_closed__doc__}, + +#define _IO_FILEIO_CLOSEFD_GETSETDEF {"closefd", (getter)_io_FileIO_closefd_get, (setter)NULL, _io_FileIO_closefd__doc__}, + +#define _IO_FILEIO_MODE_GETSETDEF {"mode", (getter)_io_FileIO_mode_get, (setter)NULL, _io_FileIO_mode__doc__}, + +#define _IO_FILEIO__BLKSIZE_GETSETDEF {"_blksize", (getter)_io_FileIO__blksize_get, (setter)NULL, _io_FileIO__blksize__doc__}, + +/*[clinic end generated code: output=d0a63950bc345c85 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index e4438c26431aa8e..3020cf03972ee9f 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -171,6 +171,96 @@ _io__IOBase_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) return _io__IOBase_flush_impl(self); } +PyDoc_STRVAR(_io__IOBase_closed__doc__, +"True if the file is closed."); + +static int +_io__IOBase_closed_get_impl(PyObject *self); + +static PyObject * +_io__IOBase_closed_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = _io__IOBase_closed_get_impl(self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_io__IOBase__checkClosed__doc__, +"_checkClosed($self, /)\n" +"--\n" +"\n"); + +#define _IO__IOBASE__CHECKCLOSED_METHODDEF \ + {"_checkClosed", (PyCFunction)_io__IOBase__checkClosed, METH_NOARGS, _io__IOBase__checkClosed__doc__}, + +static PyObject * +_io__IOBase__checkClosed_impl(PyObject *self); + +static PyObject * +_io__IOBase__checkClosed(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__IOBase__checkClosed_impl(self); +} + +PyDoc_STRVAR(_io__IOBase__checkSeekable__doc__, +"_checkSeekable($self, /)\n" +"--\n" +"\n"); + +#define _IO__IOBASE__CHECKSEEKABLE_METHODDEF \ + {"_checkSeekable", (PyCFunction)_io__IOBase__checkSeekable, METH_NOARGS, _io__IOBase__checkSeekable__doc__}, + +static PyObject * +_io__IOBase__checkSeekable_impl(PyObject *self); + +static PyObject * +_io__IOBase__checkSeekable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__IOBase__checkSeekable_impl(self); +} + +PyDoc_STRVAR(_io__IOBase__checkReadable__doc__, +"_checkReadable($self, /)\n" +"--\n" +"\n"); + +#define _IO__IOBASE__CHECKREADABLE_METHODDEF \ + {"_checkReadable", (PyCFunction)_io__IOBase__checkReadable, METH_NOARGS, _io__IOBase__checkReadable__doc__}, + +static PyObject * +_io__IOBase__checkReadable_impl(PyObject *self); + +static PyObject * +_io__IOBase__checkReadable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__IOBase__checkReadable_impl(self); +} + +PyDoc_STRVAR(_io__IOBase__checkWritable__doc__, +"_checkWritable($self, /)\n" +"--\n" +"\n"); + +#define _IO__IOBASE__CHECKWRITABLE_METHODDEF \ + {"_checkWritable", (PyCFunction)_io__IOBase__checkWritable, METH_NOARGS, _io__IOBase__checkWritable__doc__}, + +static PyObject * +_io__IOBase__checkWritable_impl(PyObject *self); + +static PyObject * +_io__IOBase__checkWritable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__IOBase__checkWritable_impl(self); +} + PyDoc_STRVAR(_io__IOBase_close__doc__, "close($self, /)\n" "--\n" @@ -252,6 +342,55 @@ _io__IOBase_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) return _io__IOBase_writable_impl(self); } +PyDoc_STRVAR(_io__IOBase___enter____doc__, +"__enter__($self, /)\n" +"--\n" +"\n" +"Context management protocol. Returns the stream itself."); + +#define _IO__IOBASE___ENTER___METHODDEF \ + {"__enter__", (PyCFunction)_io__IOBase___enter__, METH_NOARGS, _io__IOBase___enter____doc__}, + +static PyObject * +_io__IOBase___enter___impl(PyObject *self); + +static PyObject * +_io__IOBase___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _io__IOBase___enter___impl(self); +} + +PyDoc_STRVAR(_io__IOBase___exit____doc__, +"__exit__($self, /, *args)\n" +"--\n" +"\n" +"Context management protocol. Calls close()."); + +#define _IO__IOBASE___EXIT___METHODDEF \ + {"__exit__", _PyCFunction_CAST(_io__IOBase___exit__), METH_FASTCALL, _io__IOBase___exit____doc__}, + +static PyObject * +_io__IOBase___exit___impl(PyObject *self, PyObject *args); + +static PyObject * +_io__IOBase___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + __clinic_args = PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = _io__IOBase___exit___impl(self, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + PyDoc_STRVAR(_io__IOBase_fileno__doc__, "fileno($self, /)\n" "--\n" @@ -445,4 +584,22 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=28c06bb6db32c096 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_io__RawIOBase_readinto__doc__, +"readinto($self, buffer, /)\n" +"--\n" +"\n"); + +#define _IO__RAWIOBASE_READINTO_METHODDEF \ + {"readinto", (PyCFunction)_io__RawIOBase_readinto, METH_O, _io__RawIOBase_readinto__doc__}, + +PyDoc_STRVAR(_io__RawIOBase_write__doc__, +"write($self, buffer, /)\n" +"--\n" +"\n"); + +#define _IO__RAWIOBASE_WRITE_METHODDEF \ + {"write", (PyCFunction)_io__RawIOBase_write, METH_O, _io__RawIOBase_write__doc__}, +#define _IO__IOBASE_CLOSED_GETSETDEF {"closed", (getter)_io__IOBase_closed_get, (setter)NULL, _io__IOBase_closed__doc__}, + +/*[clinic end generated code: output=fe3b46799e3cf0a9 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index e9c6c39dc64b049..66b7a57cdf93337 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -477,33 +477,45 @@ _io_StringIO___setstate__(PyObject *self, PyObject *state) return return_value; } -static PyObject * +static int _io_StringIO_closed_get_impl(stringio *self); static PyObject * _io_StringIO_closed_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; + int _return_value; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_closed_get_impl((stringio *)self); + _return_value = _io_StringIO_closed_get_impl((stringio *)self); Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); +exit: return return_value; } -static PyObject * +static int _io_StringIO_line_buffering_get_impl(stringio *self); static PyObject * _io_StringIO_line_buffering_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; + int _return_value; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_line_buffering_get_impl((stringio *)self); + _return_value = _io_StringIO_line_buffering_get_impl((stringio *)self); Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); +exit: return return_value; } @@ -527,4 +539,4 @@ _io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) #define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, (setter)NULL, NULL}, -/*[clinic end generated code: output=6fa0c0dd69543304 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4100afa5e4f295d7 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index f782ec197146636..a0cefdf3bc9a9e7 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -467,6 +467,15 @@ _io_IncrementalNewlineDecoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored) return return_value; } +static PyObject * +_io_IncrementalNewlineDecoder_newlines_get_impl(nldecoder_object *self); + +static PyObject * +_io_IncrementalNewlineDecoder_newlines_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io_IncrementalNewlineDecoder_newlines_get_impl((nldecoder_object *)self); +} + PyDoc_STRVAR(_io_TextIOWrapper___init____doc__, "TextIOWrapper(buffer, encoding=None, errors=None, newline=None,\n" " line_buffering=False, write_through=False)\n" @@ -1200,29 +1209,35 @@ _io_TextIOWrapper_errors_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -static PyObject * +static Py_ssize_t _io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self); static PyObject * _io_TextIOWrapper__CHUNK_SIZE_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; + Py_ssize_t _return_value; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl((textio *)self); + _return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl((textio *)self); Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); +exit: return return_value; } static int -_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value); +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, Py_ssize_t value); static int _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *arg, void *Py_UNUSED(context)) { int return_value = -1; - PyObject *value; + Py_ssize_t value; if (arg == NULL) { PyErr_Format(PyExc_AttributeError, @@ -1230,11 +1245,23 @@ _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *arg, void *Py_UNUSED Py_TYPE(self)->tp_name); return -1; } - value = arg; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + value = ival; + } Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper__CHUNK_SIZE_set_impl((textio *)self, value); Py_END_CRITICAL_SECTION(); +exit: return return_value; } @@ -1258,6 +1285,8 @@ _io_TextIOWrapper_buffer_get(PyObject *self, void *Py_UNUSED(context)) #define _IO__TEXTIOBASE_ERRORS_GETSETDEF {"errors", (getter)_io__TextIOBase_errors_get, (setter)NULL, _io__TextIOBase_errors__doc__}, +#define _IO_INCREMENTALNEWLINEDECODER_NEWLINES_GETSETDEF {"newlines", (getter)_io_IncrementalNewlineDecoder_newlines_get, (setter)NULL, NULL}, + #define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, (setter)NULL, NULL}, #define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, (setter)NULL, NULL}, @@ -1270,4 +1299,4 @@ _io_TextIOWrapper_buffer_get(PyObject *self, void *Py_UNUSED(context)) #define _IO_TEXTIOWRAPPER_BUFFER_GETSETDEF {"buffer", (getter)_io_TextIOWrapper_buffer_get, (setter)NULL, NULL}, -/*[clinic end generated code: output=72ad4f1b23cc606f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=102ae91df6216de2 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index bd8073cd0af3f64..a0ebce1772ae903 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -428,6 +428,92 @@ _io__WindowsConsoleIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ +#if defined(HAVE_WINDOWS_CONSOLE_IO) + +PyDoc_STRVAR(_io__WindowsConsoleIO_closed__doc__, +"True if the file is closed."); +#if defined(_IO__WINDOWSCONSOLEIO_CLOSED_DOCSTR) +# undef _IO__WINDOWSCONSOLEIO_CLOSED_DOCSTR +#endif +#define _IO__WINDOWSCONSOLEIO_CLOSED_DOCSTR _io__WindowsConsoleIO_closed__doc__ + +#define _IO__WINDOWSCONSOLEIO_CLOSED_GETTER _io__WindowsConsoleIO_closed_get + +static int +_io__WindowsConsoleIO_closed_get_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_closed_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = _io__WindowsConsoleIO_closed_get_impl((winconsoleio *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ + +#if defined(HAVE_WINDOWS_CONSOLE_IO) + +PyDoc_STRVAR(_io__WindowsConsoleIO_closefd__doc__, +"True if the file descriptor will be closed by close()."); +#if defined(_IO__WINDOWSCONSOLEIO_CLOSEFD_DOCSTR) +# undef _IO__WINDOWSCONSOLEIO_CLOSEFD_DOCSTR +#endif +#define _IO__WINDOWSCONSOLEIO_CLOSEFD_DOCSTR _io__WindowsConsoleIO_closefd__doc__ + +#define _IO__WINDOWSCONSOLEIO_CLOSEFD_GETTER _io__WindowsConsoleIO_closefd_get + +static int +_io__WindowsConsoleIO_closefd_get_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_closefd_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = _io__WindowsConsoleIO_closefd_get_impl((winconsoleio *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ + +#if defined(HAVE_WINDOWS_CONSOLE_IO) + +PyDoc_STRVAR(_io__WindowsConsoleIO_mode__doc__, +"String giving the file mode."); +#if defined(_IO__WINDOWSCONSOLEIO_MODE_DOCSTR) +# undef _IO__WINDOWSCONSOLEIO_MODE_DOCSTR +#endif +#define _IO__WINDOWSCONSOLEIO_MODE_DOCSTR _io__WindowsConsoleIO_mode__doc__ + +#define _IO__WINDOWSCONSOLEIO_MODE_GETTER _io__WindowsConsoleIO_mode_get + +static PyObject * +_io__WindowsConsoleIO_mode_get_impl(winconsoleio *self); + +static PyObject * +_io__WindowsConsoleIO_mode_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__WindowsConsoleIO_mode_get_impl((winconsoleio *)self); +} + +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ + #ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF) */ @@ -463,4 +549,49 @@ _io__WindowsConsoleIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=dfe49dd71f4f4b1d input=a9049054013a1b77]*/ +#if defined(_IO__WINDOWSCONSOLEIO_CLOSED_GETTER) || defined(_IO__WINDOWSCONSOLEIO_CLOSED_SETTER) +# if !defined(_IO__WINDOWSCONSOLEIO_CLOSED_GETTER) +# define _IO__WINDOWSCONSOLEIO_CLOSED_GETTER NULL +# endif +# if !defined(_IO__WINDOWSCONSOLEIO_CLOSED_SETTER) +# define _IO__WINDOWSCONSOLEIO_CLOSED_SETTER NULL +# endif +# if !defined(_IO__WINDOWSCONSOLEIO_CLOSED_DOCSTR) +# define _IO__WINDOWSCONSOLEIO_CLOSED_DOCSTR NULL +# endif +# define _IO__WINDOWSCONSOLEIO_CLOSED_GETSETDEF {"closed", (getter)_IO__WINDOWSCONSOLEIO_CLOSED_GETTER, (setter)_IO__WINDOWSCONSOLEIO_CLOSED_SETTER, _IO__WINDOWSCONSOLEIO_CLOSED_DOCSTR}, +#else +# define _IO__WINDOWSCONSOLEIO_CLOSED_GETSETDEF +#endif + +#if defined(_IO__WINDOWSCONSOLEIO_CLOSEFD_GETTER) || defined(_IO__WINDOWSCONSOLEIO_CLOSEFD_SETTER) +# if !defined(_IO__WINDOWSCONSOLEIO_CLOSEFD_GETTER) +# define _IO__WINDOWSCONSOLEIO_CLOSEFD_GETTER NULL +# endif +# if !defined(_IO__WINDOWSCONSOLEIO_CLOSEFD_SETTER) +# define _IO__WINDOWSCONSOLEIO_CLOSEFD_SETTER NULL +# endif +# if !defined(_IO__WINDOWSCONSOLEIO_CLOSEFD_DOCSTR) +# define _IO__WINDOWSCONSOLEIO_CLOSEFD_DOCSTR NULL +# endif +# define _IO__WINDOWSCONSOLEIO_CLOSEFD_GETSETDEF {"closefd", (getter)_IO__WINDOWSCONSOLEIO_CLOSEFD_GETTER, (setter)_IO__WINDOWSCONSOLEIO_CLOSEFD_SETTER, _IO__WINDOWSCONSOLEIO_CLOSEFD_DOCSTR}, +#else +# define _IO__WINDOWSCONSOLEIO_CLOSEFD_GETSETDEF +#endif + +#if defined(_IO__WINDOWSCONSOLEIO_MODE_GETTER) || defined(_IO__WINDOWSCONSOLEIO_MODE_SETTER) +# if !defined(_IO__WINDOWSCONSOLEIO_MODE_GETTER) +# define _IO__WINDOWSCONSOLEIO_MODE_GETTER NULL +# endif +# if !defined(_IO__WINDOWSCONSOLEIO_MODE_SETTER) +# define _IO__WINDOWSCONSOLEIO_MODE_SETTER NULL +# endif +# if !defined(_IO__WINDOWSCONSOLEIO_MODE_DOCSTR) +# define _IO__WINDOWSCONSOLEIO_MODE_DOCSTR NULL +# endif +# define _IO__WINDOWSCONSOLEIO_MODE_GETSETDEF {"mode", (getter)_IO__WINDOWSCONSOLEIO_MODE_GETTER, (setter)_IO__WINDOWSCONSOLEIO_MODE_SETTER, _IO__WINDOWSCONSOLEIO_MODE_DOCSTR}, +#else +# define _IO__WINDOWSCONSOLEIO_MODE_GETSETDEF +#endif + +/*[clinic end generated code: output=24fe425795203cee input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index e8e9c132dd3465e..e5cd7aca5de2f17 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -99,10 +99,17 @@ _PyFileIO_closed(PyObject *self) /* Because this can call arbitrary code, it shouldn't be called when the refcount is 0 (that is, not directly from tp_dealloc unless the refcount has been temporarily re-incremented). */ +/*[clinic input] +_io.FileIO._dealloc_warn + + source: object + / +[clinic start generated code]*/ + static PyObject * -fileio_dealloc_warn(PyObject *op, PyObject *source) +_io_FileIO__dealloc_warn_impl(fileio *self, PyObject *source) +/*[clinic end generated code: output=c7b7d122decd6575 input=3ea7e1cc0685edc2]*/ { - fileio *self = PyFileIO_CAST(op); if (self->fd >= 0 && self->closefd) { PyObject *exc = PyErr_GetRaisedException(); if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { @@ -176,7 +183,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) exc = PyErr_GetRaisedException(); } if (self->finalizing) { - PyObject *r = fileio_dealloc_warn((PyObject*)self, (PyObject *) self); + PyObject *r = _io_FileIO__dealloc_warn_impl(self, (PyObject *)self); if (r) { Py_DECREF(r); } @@ -1250,10 +1257,14 @@ _io_FileIO_isatty_impl(fileio *self) information. Use the stat result to skip a system call. Outside of that context TOCTOU issues (the fd could be arbitrarily modified by surrounding code). */ +/*[clinic input] +_io.FileIO._isatty_open_only +[clinic start generated code]*/ + static PyObject * -_io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(dummy)) +_io_FileIO__isatty_open_only_impl(fileio *self) +/*[clinic end generated code: output=2b4689154d4b8b84 input=228767ff567cdfb6]*/ { - fileio *self = PyFileIO_CAST(op); if (self->stat_atopen != NULL && !S_ISCHR(self->stat_atopen->st_mode)) { Py_RETURN_FALSE; } @@ -1276,53 +1287,80 @@ static PyMethodDef fileio_methods[] = { _IO_FILEIO_WRITABLE_METHODDEF _IO_FILEIO_FILENO_METHODDEF _IO_FILEIO_ISATTY_METHODDEF - {"_isatty_open_only", _io_FileIO_isatty_open_only, METH_NOARGS}, - {"_dealloc_warn", fileio_dealloc_warn, METH_O, NULL}, + _IO_FILEIO__ISATTY_OPEN_ONLY_METHODDEF + _IO_FILEIO__DEALLOC_WARN_METHODDEF {"__getstate__", _PyIOBase_cannot_pickle, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ -static PyObject * -fileio_get_closed(PyObject *op, void *closure) +/*[clinic input] +@getter +_io.FileIO.closed -> bool + +True if the file is closed. +[clinic start generated code]*/ + +static int +_io_FileIO_closed_get_impl(fileio *self) +/*[clinic end generated code: output=5052fd0688b475f2 input=6024cca6d484d9a2]*/ { - fileio *self = PyFileIO_CAST(op); - return PyBool_FromLong((long)(self->fd < 0)); + return self->fd < 0; } -static PyObject * -fileio_get_closefd(PyObject *op, void *closure) +/*[clinic input] +@getter +_io.FileIO.closefd -> bool + +True if the file descriptor will be closed by close(). +[clinic start generated code]*/ + +static int +_io_FileIO_closefd_get_impl(fileio *self) +/*[clinic end generated code: output=11963fca763399cc input=ed3d108a6f07ad5c]*/ { - fileio *self = PyFileIO_CAST(op); - return PyBool_FromLong((long)(self->closefd)); + return self->closefd; } +/*[clinic input] +@getter +_io.FileIO.mode + +String giving the file mode. +[clinic start generated code]*/ + static PyObject * -fileio_get_mode(PyObject *op, void *closure) +_io_FileIO_mode_get_impl(fileio *self) +/*[clinic end generated code: output=d5178215493e2e7e input=300a448fc9b2fea4]*/ { - fileio *self = PyFileIO_CAST(op); return PyUnicode_FromString(mode_string(self)); } -static PyObject * -fileio_get_blksize(PyObject *op, void *closure) +/*[clinic input] +@getter +_io.FileIO._blksize -> long + +Stat st_blksize if available. +[clinic start generated code]*/ + +static long +_io_FileIO__blksize_get_impl(fileio *self) +/*[clinic end generated code: output=e47eaf8cd6c0079f input=699d87d750798ccb]*/ { #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE - fileio *self = PyFileIO_CAST(op); if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) { - return PyLong_FromLong(self->stat_atopen->st_blksize); + return self->stat_atopen->st_blksize; } #endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ - return PyLong_FromLong(DEFAULT_BUFFER_SIZE); + return DEFAULT_BUFFER_SIZE; } static PyGetSetDef fileio_getsetlist[] = { - {"closed", fileio_get_closed, NULL, "True if the file is closed"}, - {"closefd", fileio_get_closefd, NULL, - "True if the file descriptor will be closed by close()."}, - {"mode", fileio_get_mode, NULL, "String giving the file mode"}, - {"_blksize", fileio_get_blksize, NULL, "Stat st_blksize if available"}, + _IO_FILEIO_CLOSED_GETSETDEF + _IO_FILEIO_CLOSEFD_GETSETDEF + _IO_FILEIO_MODE_GETSETDEF + _IO_FILEIO__BLKSIZE_GETSETDEF {NULL}, }; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index b6d10c8568feb68..5b9b9c856431b07 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -184,14 +184,18 @@ _io__IOBase_flush_impl(PyObject *self) return NULL; } -static PyObject * -iobase_closed_get(PyObject *self, void *context) +/*[clinic input] +@getter +_io._IOBase.closed -> bool + +True if the file is closed. +[clinic start generated code]*/ + +static int +_io__IOBase_closed_get_impl(PyObject *self) +/*[clinic end generated code: output=561af4ea236d16d9 input=e730fe4ccc4c096c]*/ { - int closed = iobase_is_closed(self); - if (closed < 0) { - return NULL; - } - return PyBool_FromLong(closed); + return iobase_is_closed(self); } static int @@ -225,25 +229,51 @@ _PyIOBase_check_closed(PyObject *self, PyObject *args) Py_RETURN_NONE; } +/*[clinic input] +_io._IOBase._checkClosed +[clinic start generated code]*/ + +static PyObject * +_io__IOBase__checkClosed_impl(PyObject *self) +/*[clinic end generated code: output=8fb8412185623f1a input=8d18d2e67b2270bb]*/ +{ + return _PyIOBase_check_closed(self, NULL); +} + +/*[clinic input] +_io._IOBase._checkSeekable +[clinic start generated code]*/ + static PyObject * -iobase_check_seekable(PyObject *self, PyObject *args) +_io__IOBase__checkSeekable_impl(PyObject *self) +/*[clinic end generated code: output=10f09b515a9f4c4f input=479853eded776b0e]*/ { _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); - return _PyIOBase_check_seekable(state, self, args); + return _PyIOBase_check_seekable(state, self, NULL); } +/*[clinic input] +_io._IOBase._checkReadable +[clinic start generated code]*/ + static PyObject * -iobase_check_readable(PyObject *self, PyObject *args) +_io__IOBase__checkReadable_impl(PyObject *self) +/*[clinic end generated code: output=33ccab8a7c4550fb input=63fbf50b36772323]*/ { _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); - return _PyIOBase_check_readable(state, self, args); + return _PyIOBase_check_readable(state, self, NULL); } +/*[clinic input] +_io._IOBase._checkWritable +[clinic start generated code]*/ + static PyObject * -iobase_check_writable(PyObject *self, PyObject *args) +_io__IOBase__checkWritable_impl(PyObject *self) +/*[clinic end generated code: output=8cfc6b4b2469d2a3 input=2758c7291cf5911e]*/ { _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); - return _PyIOBase_check_writable(state, self, args); + return _PyIOBase_check_writable(state, self, NULL); } PyObject * @@ -495,8 +525,15 @@ _PyIOBase_check_writable(_PyIO_State *state, PyObject *self, PyObject *args) /* Context manager */ +/*[clinic input] +_io._IOBase.__enter__ + +Context management protocol. Returns the stream itself. +[clinic start generated code]*/ + static PyObject * -iobase_enter(PyObject *self, PyObject *args) +_io__IOBase___enter___impl(PyObject *self) +/*[clinic end generated code: output=d1e8e5b58bde3680 input=92e2d5e34ee714e5]*/ { if (iobase_check_closed(self)) return NULL; @@ -504,8 +541,16 @@ iobase_enter(PyObject *self, PyObject *args) return Py_NewRef(self); } +/*[clinic input] +_io._IOBase.__exit__ + *args: tuple + +Context management protocol. Calls close(). +[clinic start generated code]*/ + static PyObject * -iobase_exit(PyObject *self, PyObject *args) +_io__IOBase___exit___impl(PyObject *self, PyObject *args) +/*[clinic end generated code: output=452f3e33a34e2c5c input=fd3f46a32773a170]*/ { return PyObject_CallMethodNoArgs(self, &_Py_ID(close)); } @@ -837,16 +882,16 @@ static PyMethodDef iobase_methods[] = { _IO__IOBASE_READABLE_METHODDEF _IO__IOBASE_WRITABLE_METHODDEF - {"_checkClosed", _PyIOBase_check_closed, METH_NOARGS}, - {"_checkSeekable", iobase_check_seekable, METH_NOARGS}, - {"_checkReadable", iobase_check_readable, METH_NOARGS}, - {"_checkWritable", iobase_check_writable, METH_NOARGS}, + _IO__IOBASE__CHECKCLOSED_METHODDEF + _IO__IOBASE__CHECKSEEKABLE_METHODDEF + _IO__IOBASE__CHECKREADABLE_METHODDEF + _IO__IOBASE__CHECKWRITABLE_METHODDEF _IO__IOBASE_FILENO_METHODDEF _IO__IOBASE_ISATTY_METHODDEF - {"__enter__", iobase_enter, METH_NOARGS}, - {"__exit__", iobase_exit, METH_VARARGS}, + _IO__IOBASE___ENTER___METHODDEF + _IO__IOBASE___EXIT___METHODDEF _IO__IOBASE_READLINE_METHODDEF _IO__IOBASE_READLINES_METHODDEF @@ -857,7 +902,7 @@ static PyMethodDef iobase_methods[] = { static PyGetSetDef iobase_getset[] = { {"__dict__", PyObject_GenericGetDict, NULL, NULL}, - {"closed", iobase_closed_get, NULL, NULL}, + _IO__IOBASE_CLOSED_GETSETDEF {NULL} }; @@ -1014,15 +1059,29 @@ _io__RawIOBase_readall_impl(PyObject *self) return PyBytesWriter_Finish(writer); } +/*[clinic input] +_io._RawIOBase.readinto + buffer: object + / +[clinic start generated code]*/ + static PyObject * -rawiobase_readinto(PyObject *self, PyObject *args) +_io__RawIOBase_readinto(PyObject *self, PyObject *buffer) +/*[clinic end generated code: output=081b8cdfaf3a40ff input=99d4bea2acd659c8]*/ { PyErr_SetNone(PyExc_NotImplementedError); return NULL; } +/*[clinic input] +_io._RawIOBase.write + buffer: object + / +[clinic start generated code]*/ + static PyObject * -rawiobase_write(PyObject *self, PyObject *args) +_io__RawIOBase_write(PyObject *self, PyObject *buffer) +/*[clinic end generated code: output=7add3f2c8715c3a8 input=e6a1534adb876fe2]*/ { PyErr_SetNone(PyExc_NotImplementedError); return NULL; @@ -1031,8 +1090,8 @@ rawiobase_write(PyObject *self, PyObject *args) static PyMethodDef rawiobase_methods[] = { _IO__RAWIOBASE_READ_METHODDEF _IO__RAWIOBASE_READALL_METHODDEF - {"readinto", rawiobase_readinto, METH_VARARGS}, - {"write", rawiobase_write, METH_VARARGS}, + _IO__RAWIOBASE_READINTO_METHODDEF + _IO__RAWIOBASE_WRITE_METHODDEF {NULL, NULL} }; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 27605b4c44239e9..93a82bcc0c6e3e9 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -68,6 +68,20 @@ static int _io_StringIO___init__(PyObject *self, PyObject *args, PyObject *kwarg return NULL; \ } +#define CHECK_INITIALIZED_INT(self) \ + if (self->ok <= 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on uninitialized object"); \ + return -1; \ + } + +#define CHECK_CLOSED_INT(self) \ + if (self->closed) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on closed file"); \ + return -1; \ + } + #define ENSURE_REALIZED(self) \ if (realize(self) < 0) { \ return NULL; \ @@ -1013,30 +1027,30 @@ _io_StringIO___setstate___impl(stringio *self, PyObject *state) /*[clinic input] @critical_section @getter -_io.StringIO.closed +_io.StringIO.closed -> bool [clinic start generated code]*/ -static PyObject * +static int _io_StringIO_closed_get_impl(stringio *self) -/*[clinic end generated code: output=531ddca7954331d6 input=178d2ef24395fd49]*/ +/*[clinic end generated code: output=754068c44422cafa input=ea05e89b945e721c]*/ { - CHECK_INITIALIZED(self); - return PyBool_FromLong(self->closed); + CHECK_INITIALIZED_INT(self); + return self->closed; } /*[clinic input] @critical_section @getter -_io.StringIO.line_buffering +_io.StringIO.line_buffering -> bool [clinic start generated code]*/ -static PyObject * +static int _io_StringIO_line_buffering_get_impl(stringio *self) -/*[clinic end generated code: output=360710e0112966ae input=6a7634e7f890745e]*/ +/*[clinic end generated code: output=56c0edde9001fb37 input=326f8b3bd1feb699]*/ { - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - Py_RETURN_FALSE; + CHECK_INITIALIZED_INT(self); + CHECK_CLOSED_INT(self); + return 0; } /*[clinic input] diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index a744a885932cdf5..8f98f7d80095b0b 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -635,10 +635,15 @@ _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) Py_RETURN_NONE; } +/*[clinic input] +@getter +_io.IncrementalNewlineDecoder.newlines +[clinic start generated code]*/ + static PyObject * -incrementalnewlinedecoder_newlines_get(PyObject *op, void *Py_UNUSED(context)) +_io_IncrementalNewlineDecoder_newlines_get_impl(nldecoder_object *self) +/*[clinic end generated code: output=4370cf5202a83d08 input=040f9a26aef317a8]*/ { - nldecoder_object *self = nldecoder_object_CAST(op); CHECK_INITIALIZED_DECODER(self); switch (self->seennl) { @@ -3394,38 +3399,35 @@ _io_TextIOWrapper_errors_get_impl(textio *self) /*[clinic input] @critical_section @getter -_io.TextIOWrapper._CHUNK_SIZE +_io.TextIOWrapper._CHUNK_SIZE -> Py_ssize_t [clinic start generated code]*/ -static PyObject * +static Py_ssize_t _io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self) -/*[clinic end generated code: output=039925cd2df375bc input=e9715b0e06ff0fa6]*/ +/*[clinic end generated code: output=3fe34b873b6b2c8c input=cae767b74b6b46bd]*/ { - CHECK_ATTACHED(self); - return PyLong_FromSsize_t(self->chunk_size); + CHECK_ATTACHED_INT(self); + return self->chunk_size; } /*[clinic input] @critical_section @setter _io.TextIOWrapper._CHUNK_SIZE + value: Py_ssize_t [clinic start generated code]*/ static int -_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value) -/*[clinic end generated code: output=edb86d2db660a5ab input=32fc99861db02a0a]*/ +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, Py_ssize_t value) +/*[clinic end generated code: output=5170a93dd4d74917 input=ede7bae6145f941b]*/ { - Py_ssize_t n; CHECK_ATTACHED_INT(self); - n = PyNumber_AsSsize_t(value, PyExc_ValueError); - if (n == -1 && PyErr_Occurred()) - return -1; - if (n <= 0) { + if (value <= 0) { PyErr_SetString(PyExc_ValueError, "a strictly positive integer is required"); return -1; } - self->chunk_size = n; + self->chunk_size = value; return 0; } @@ -3451,7 +3453,7 @@ static PyMethodDef incrementalnewlinedecoder_methods[] = { }; static PyGetSetDef incrementalnewlinedecoder_getset[] = { - {"newlines", incrementalnewlinedecoder_newlines_get, NULL, NULL}, + _IO_INCREMENTALNEWLINEDECODER_NEWLINES_GETSETDEF {NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 11e29227d9276e9..e3e57d8df455530 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1202,32 +1202,52 @@ static PyMethodDef winconsoleio_methods[] = { /* 'closed' and 'mode' are attributes for compatibility with FileIO. */ -static PyObject * -get_closed(PyObject *op, void *Py_UNUSED(closure)) +/*[clinic input] +@getter +_io._WindowsConsoleIO.closed -> bool + +True if the file is closed. +[clinic start generated code]*/ + +static int +_io__WindowsConsoleIO_closed_get_impl(winconsoleio *self) +/*[clinic end generated code: output=a3f8b93c6ddcf660 input=65b6d1bed0aadbb8]*/ { - winconsoleio *self = winconsoleio_CAST(op); - return PyBool_FromLong((long)(self->fd == -1)); + return self->fd == -1; } -static PyObject * -get_closefd(PyObject *op, void *Py_UNUSED(closure)) +/*[clinic input] +@getter +_io._WindowsConsoleIO.closefd -> bool + +True if the file descriptor will be closed by close(). +[clinic start generated code]*/ + +static int +_io__WindowsConsoleIO_closefd_get_impl(winconsoleio *self) +/*[clinic end generated code: output=ec221c2f949530a3 input=5839d3a815edecee]*/ { - winconsoleio *self = winconsoleio_CAST(op); - return PyBool_FromLong((long)(self->closefd)); + return self->closefd; } +/*[clinic input] +@getter +_io._WindowsConsoleIO.mode + +String giving the file mode. +[clinic start generated code]*/ + static PyObject * -get_mode(PyObject *op, void *Py_UNUSED(closure)) +_io__WindowsConsoleIO_mode_get_impl(winconsoleio *self) +/*[clinic end generated code: output=2d7c4cabf96e5281 input=e1fa9cd881117c0f]*/ { - winconsoleio *self = winconsoleio_CAST(op); return PyUnicode_FromString(self->readable ? "rb" : "wb"); } static PyGetSetDef winconsoleio_getsetlist[] = { - {"closed", get_closed, NULL, "True if the file is closed"}, - {"closefd", get_closefd, NULL, - "True if the file descriptor will be closed by close()."}, - {"mode", get_mode, NULL, "String giving the file mode"}, + _IO__WINDOWSCONSOLEIO_CLOSED_GETSETDEF + _IO__WINDOWSCONSOLEIO_CLOSEFD_GETSETDEF + _IO__WINDOWSCONSOLEIO_MODE_GETSETDEF {NULL}, }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 4e50ca64f59af22..65857c495f2339f 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -363,6 +363,12 @@ ptrace_enter_call(PyObject *self, void *key, PyObject *userObj) ProfilerEntry *profEntry; ProfilerContext *pContext; + /* Events raised by the external timer must be ignored: it can run + arbitrary code while a context is still being unwound. */ + if (pObj->flags & POF_EXT_TIMER) { + return; + } + /* In the case of entering a generator expression frame via a * throw (gen_send_ex(.., 1)), we may already have an * Exception set here. We must not mess around with this @@ -405,6 +411,10 @@ ptrace_leave_call(PyObject *self, void *key) ProfilerEntry *profEntry; ProfilerContext *pContext; + if (pObj->flags & POF_EXT_TIMER) { + return; + } + pContext = pObj->currentProfilerContext; if (pContext == NULL) return; @@ -980,10 +990,13 @@ profiler_dealloc(PyObject *op) } } + /* Drop the external timer before flushing: it is Python code, and the + profiler can be deallocated by the garbage collector. */ + Py_CLEAR(self->externalTimer); + flush_unmatched(self); clearEntries(self); Py_XDECREF(self->missing); - Py_XDECREF(self->externalTimer); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index df481cb832a4363..1f89c12f223c7d9 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -1,6 +1,6 @@ #include "parts.h" -#include "../../Parser/tokenizer/cursor.h" +#include "../../Parser/tokenizer/source.h" static int check(int condition, const char *message) @@ -24,13 +24,14 @@ check_system_error(int failed, const char *message) } static int -same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right) +check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno, + const char *expected) { - return left->source == right->source && - left->pos == right->pos && - left->line_start == right->line_start && - left->line_end == right->line_end && - left->lineno == right->lineno; + Py_ssize_t len; + const char *line = _PyTok_SourceLineView(source, lineno, &len); + return check(len == (Py_ssize_t)strlen(expected) && + memcmp(line, expected, len) == 0, + "wrong source line view"); } static PyObject * @@ -40,300 +41,54 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), _PyTok_SourceText source; _PyTok_SourceInit(&source); - _PyTok_Loc loc; - _PyTok_Line line; - if (check(_PyTok_SourceLocation( - &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate empty source") < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong empty source location") < 0 || - check(_PyTok_SourceLine(&source, 1, &line) == 0, - "cannot find empty source line") < 0 || - check(line.start == 0 && line.end == 0, - "wrong empty source line") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, - "accepted empty source line") < 0 || + if (check_line_view(&source, 1, "") < 0) { + goto error; + } + + if (check_system_error( + _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + "accepted empty source line") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, "accepted multiple source lines") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0) { - goto error; - } - - if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0, - "wrong first source offset") < 0 || + "accepted missing implicit newline") < 0 || + check(_PyTok_SourceAppendLine( + &source, "alpha\n", 6, 0) == 0, + "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( &source, "\xce\xb2\n", 3, 1) == 6, "wrong second source offset") < 0 || - check(_PyTok_SourceAppendLine( - &source, "nul\0x\n", 6, 0) == 9, - "wrong third source offset") < 0) { - goto error; - } - - int marker_line = 257; - int final_line = 300; - _PyTok_Off marker_start = -1; - for (int lineno = 4; lineno <= final_line; lineno++) { - const char *text = lineno == marker_line ? "marker\n" : "x\n"; - Py_ssize_t len = (Py_ssize_t)strlen(text); - _PyTok_Off start = _PyTok_SourceAppendLine( - &source, text, len, lineno == final_line); - if (start < 0) { - goto error; - } - if (lineno == marker_line) { - marker_start = start; - } - } - - if (check(source.nlines == final_line, "wrong source line count") < 0 || - check(_PyTok_SourceLine(&source, marker_line, &line) == 0, - "cannot find late source line") < 0 || - check(line.start == marker_start && - line.end == marker_start + 7, - "wrong late source line") < 0 || - check(!line.implicit_newline && !line.contains_nul, - "wrong late source flags") < 0 || - check(_PyTok_SourceLine(&source, 2, &line) == 0, - "cannot find second source line") < 0 || - check(line.start == 6 && line.end == 9 && - line.implicit_newline && !line.contains_nul, - "wrong second source line") < 0 || check(!_PyTok_SourceLineIsImplicit(&source, 1) && _PyTok_SourceLineIsImplicit(&source, 2), - "wrong early implicit newline flags") < 0 || - check(_PyTok_SourceLine(&source, 3, &line) == 0, - "cannot find third source line") < 0 || - check(line.contains_nul, "missing null byte flag") < 0 || - check(_PyTok_SourceLine(&source, final_line, &line) == 0, - "cannot find final source line") < 0 || - check(line.implicit_newline && - _PyTok_SourceLineIsImplicit(&source, final_line), - "missing late implicit newline flag") < 0) { + "wrong implicit newline flags") < 0) { goto error; } - Py_ssize_t view_len; - const char *view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(6, 8), &view_len); - if (check(view != NULL && view_len == 2 && - memcmp(view, "\xce\xb2", 2) == 0, - "wrong source span view") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left line boundary") < 0 || - check(loc.lineno == marker_line - 1 && loc.byte_col == 2, - "wrong left boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right line boundary") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 0, - "wrong right boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start + 1, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate late source byte") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 1, - "wrong late source location") < 0) { + if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 || + check_line_view(&source, 1, "alpha") < 0 || + check_line_view(&source, 2, "\xce\xb2") < 0 || + check_line_view(&source, 3, "") < 0 || + check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) { goto error; } - if (check(_PyTok_SourceLocation( - &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left EOF") < 0 || - check(loc.lineno == final_line && loc.byte_col == 2, - "wrong left EOF location") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right EOF") < 0 || - check(loc.lineno == final_line + 1 && loc.byte_col == 0, - "wrong right EOF location") < 0 || - check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0, - "cannot find virtual EOF line") < 0 || - check(line.start == source.len && line.end == source.len, - "wrong virtual EOF line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 0) && - !_PyTok_SourceLineIsImplicit( - &source, final_line + 1), - "virtual or invalid line is implicit") < 0) { - goto error; - } - - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len); - if (check_system_error(view == NULL, "accepted invalid source span") < 0 || - check_system_error( - _PyTok_SourceLocation( - &source, source.len + 1, - _PYTOK_AFFINITY_RIGHT, &loc) < 0, - "accepted invalid source offset") < 0 || - check_system_error( - _PyTok_SourceLine(&source, final_line + 2, &line) < 0, - "accepted invalid source line") < 0) { + if (check(source.len == 9 && + memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - _PyTok_SourceInit(&source); if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, - "appended after unterminated source line") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate unterminated EOF") < 0 || - check(loc.lineno == 1 && loc.byte_col == 4, - "wrong unterminated EOF location") < 0) { - goto error; - } - - _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, - "wrong retained source offset") < 0 || - _PyTok_SourceLine(&source, 1, &line) < 0 || - check(line.start == 4 && line.end == 6, - "wrong retained source line") < 0 || - _PyTok_SourceLocation( - &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong retained source location") < 0) { - goto error; - } - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(4, 5), &view_len); - if (check(view != NULL && view_len == 1 && view[0] == 'a', - "wrong retained source span") < 0 || - check_system_error(_PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL, - "accepted discarded source span") < 0) { - goto error; - } - - _PyTok_SourceClear(&source); - Py_RETURN_NONE; - -error: - _PyTok_SourceClear(&source); - return NULL; -} - -static PyObject * -test_tokenizer_cursor(PyObject *Py_UNUSED(module), - PyObject *Py_UNUSED(args)) -{ - _PyTok_SourceText source; - _PyTok_SourceInit(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) { - goto error; - } - - _PyTok_Cursor cursor; - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 || - check(cursor.lineno == 3 && cursor.pos == source.len, - "wrong cursor at virtual EOF") < 0 || - _PyTok_CursorSetLine(&cursor, 1) < 0) { - goto error; - } - - char large[BUFSIZ + 1]; - memset(large, 'z', sizeof(large)); - large[sizeof(large) - 1] = '\n'; - if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) { - goto error; - } - - if (check(_PyTok_CursorPeek(&cursor, 0) == 'a', - "wrong cursor peek after relocation") < 0 || - check(_PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong distant cursor peek") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'a', - "wrong first cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'b', - "wrong second cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong final cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == EOF, - "cursor advanced past line") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 2) == 0, - "cannot seek cursor offset") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong cursor byte after seek") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 3) == 0, - "cannot seek line boundary") < 0 || - check(cursor.lineno == 2 && cursor.line_start == 3 && - _PyTok_CursorAdvance(&cursor) == 'c', - "wrong cursor at line boundary") < 0 || - check(_PyTok_CursorSetLine(&cursor, 3) == 0, - "cannot advance cursor to final line") < 0 || - check(cursor.line_start == 6 && - _PyTok_CursorAdvance(&cursor) == 'z', - "wrong cursor byte on final line") < 0) { - goto error; - } - - _PyTok_Cursor saved = cursor; - if (check_system_error( - _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0, - "accepted invalid cursor offset") < 0 || - check(same_cursor(&cursor, &saved), - "invalid offset changed cursor") < 0 || - check_system_error( - _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0, - "accepted invalid cursor line") < 0 || - check(same_cursor(&cursor, &saved), - "invalid line changed cursor") < 0 || - check(_PyTok_CursorSetOffset(&cursor, source.len) == 0, - "cannot set cursor to EOF") < 0 || - check(cursor.lineno == 4 && cursor.pos == source.len, - "wrong cursor at EOF") < 0) { - goto error; - } - -#if SIZEOF_VOID_P > 4 - char byte = 0; - _PyTok_SourceText huge_source = { - .bytes = &byte, - .len = (_PyTok_Off)INT_MAX + 1, - }; - _PyTok_Cursor huge_cursor = { - .source = &huge_source, - .pos = INT_MAX, - .line_end = (_PyTok_Off)INT_MAX + 1, - .lineno = 1, - }; - if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF && - huge_cursor.pos == INT_MAX, - "cursor advanced past maximum column") < 0) { - goto error; - } -#endif - - _PyTok_Off base = source.len; - _PyTok_SourceDiscard(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) { + "appended after unterminated source line") < 0) { goto error; } - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetLine(&cursor, 1) < 0 || - check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong retained cursor line") < 0 || - _PyTok_CursorSetLine(&cursor, 2) < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'c', - "wrong retained cursor byte") < 0 || - _PyTok_CursorSetOffset(&cursor, base + 5) < 0 || - check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF, - "wrong retained cursor EOF") < 0) { + if (check_line_view(&source, 1, "tail") < 0 || + check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) { goto error; } @@ -410,7 +165,6 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), static PyMethodDef test_methods[] = { {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, - {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS}, {NULL}, }; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 69833f132b5e4d4..36a1d768486a3f7 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -183,6 +183,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 207552113c3dd23..b358e09950a4439 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -463,6 +463,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e4766fd7e5f764a..c2d8f64a72b1191 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -422,9 +422,7 @@ - - @@ -595,9 +593,9 @@ + - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 993c7ed24d56f48..2536fc971700deb 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -321,18 +321,12 @@ Objects - - Parser - Parser Parser - - Parser - Parser @@ -1355,6 +1349,9 @@ Parser + + Parser + Parser @@ -1364,9 +1361,6 @@ Parser - - Parser - Parser diff --git a/Parser/lexer/layout.c b/Parser/lexer/layout.c new file mode 100644 index 000000000000000..6091f9adeac958c --- /dev/null +++ b/Parser/lexer/layout.c @@ -0,0 +1,195 @@ +#include "Python.h" +#include "errcode.h" +#include "pycore_token.h" + +#include "lexer_internal.h" +#include "../tokenizer/helpers.h" +#include "../tokenizer/reader.h" + +#define TABSIZE 8 +#define ALTTABSIZE 1 + +int +_PyLexer_ContinueLine(struct tok_state *tok) +{ + int c = tok_nextc(tok); + if (c == '\r') { + c = tok_nextc(tok); + } + if (c != '\n') { + tok->done = E_LINECONT; + return -1; + } + c = tok_nextc(tok); + if (c == EOF) { + tok->done = E_EOF; + tok->cur = tok->inp; + return -1; + } else { + tok_backup(tok, c); + } + return c; +} + + +static int +update_indentation(struct tok_state *tok, int col, int altcol) +{ + lexer_layout_state *layout = &tok->layout; + if (col == layout->stack[layout->depth].column) { + if (altcol != layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + } + else if (col > layout->stack[layout->depth].column) { + if (layout->depth + 1 >= MAXINDENT) { + tok->done = E_TOODEEP; + tok->cur = tok->inp; + return -1; + } + if (altcol <= layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + layout->pending++; + layout->stack[++layout->depth] = (indentation_level){col, altcol}; + } + else { + while (layout->depth > 0 && + col < layout->stack[layout->depth].column) { + layout->pending--; + layout->depth--; + } + if (col != layout->stack[layout->depth].column) { + tok->done = E_DEDENT; + tok->cur = tok->inp; + return -1; + } + if (altcol != layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + } + return 0; +} + +int +_PyLexer_BeginLine(struct tok_state *tok) +{ + assert(tok->layout.at_bol); + int c; + int blankline = 0; + int col = 0; + int altcol = 0; + tok->layout.at_bol = 0; + int cont_line_col = 0; + for (;;) { + c = tok_nextc(tok); + if (c == ' ') { + col++, altcol++; + } + else if (c == '\t') { + col = (col / TABSIZE + 1) * TABSIZE; + altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; + } + else if (c == '\014') {/* Control-L (formfeed) */ + col = altcol = 0; /* For Emacs users */ + } + else if (c == '\\') { + // Indentation cannot be split over multiple physical lines + // using backslashes. This means that if we found a backslash + // preceded by whitespace, **the first one we find** determines + // the level of indentation of whatever comes next. + cont_line_col = cont_line_col ? cont_line_col : col; + if ((c = _PyLexer_ContinueLine(tok)) == -1) { + return -1; + } + } + else if (c == EOF && PyErr_Occurred()) { + return -1; + } + else { + break; + } + } + tok_backup(tok, c); + if (c == '#' || c == '\n' || c == '\r') { + int interactive = _PyTok_ReaderIsInteractive(tok); + /* Lines with only whitespace and/or comments + shouldn't affect the indentation and are + not passed to the parser as NEWLINE tokens, + except *totally* empty lines in interactive + mode, which signal the end of a command group. */ + if (col == 0 && c == '\n' && interactive) { + blankline = 0; /* Let it through */ + } + else if (interactive && tok->lineno == 1) { + /* In interactive mode, if the first line contains + only spaces and/or a comment, let it through. */ + blankline = 0; + col = altcol = 0; + } + else { + blankline = 1; /* Ignore completely */ + } + } + if (!blankline && tok->level == 0) { + col = cont_line_col ? cont_line_col : col; + altcol = cont_line_col ? cont_line_col : altcol; + if (update_indentation(tok, col, altcol) < 0) { + return -1; + } + } + return blankline; +} + +int +_PyLexer_IndentationToken(struct tok_state *tok, struct token *token) +{ + assert(tok->layout.pending != 0); + _PyTok_Off p_start = -1; + _PyTok_Off p_end = -1; + if (tok->layout.pending < 0) { + if (tok->tok_extra_tokens) { + p_start = tok->cur; + p_end = tok->cur; + } + tok->layout.pending++; + return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end); + } + else { + if (tok->tok_extra_tokens) { + p_start = tok->buf_offset; + p_end = tok->cur; + } + tok->layout.pending--; + return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end); + } +} + +int +_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline) +{ + tok->layout.at_bol = 1; + if (blankline || tok->level > 0) { + if (!tok->tok_extra_tokens) { + return 0; + } + } + else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) { + return _PyLexer_token_setup(tok, token, NEWLINE, + tok->start, tok->cur - 1); + } + tok->layout.comment_newline = 0; + return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur); +} + +void +_PyLexer_ImplyDedents(struct tok_state *tok) +{ + if (tok->layout.depth != 0) { + tok->layout.pending = -tok->layout.depth; + tok->layout.depth = 0; + } +} diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 110a225750f0550..f27fee8d61b9085 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -7,10 +7,6 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -#define TABSIZE 8 -#define ALTTABSIZE 1 - - #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) /* Spaces in this constant are treated as "zero or more spaces or tabs" when @@ -93,6 +89,7 @@ verify_identifier(struct tok_state *tok) assert(PyUnicode_GET_LENGTH(s) > 0); if (invalid < PyUnicode_GET_LENGTH(s)) { Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid); + _PyTok_Off error_cursor = tok->cur; if (invalid + 1 < PyUnicode_GET_LENGTH(s)) { /* Determine the offset in UTF-8 encoded input */ Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1)); @@ -103,14 +100,20 @@ verify_identifier(struct tok_state *tok) tok->done = E_ERROR; return 0; } - tok->cur = tok->start + PyBytes_GET_SIZE(s); + error_cursor = tok->start + PyBytes_GET_SIZE(s); } Py_DECREF(s); if (Py_UNICODE_ISPRINTABLE(ch)) { - _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); + _PyTokenizer_syntaxerror_at( + tok, _PyLexer_BufferPointer(tok, tok->line_start), + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid character '%c' (U+%04X)", ch, ch); } else { - _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch); + _PyTokenizer_syntaxerror_at( + tok, _PyLexer_BufferPointer(tok, tok->line_start), + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid non-printable character U+%04X", ch); } return 0; } @@ -118,31 +121,6 @@ verify_identifier(struct tok_state *tok) return 1; } - - -static inline int -tok_continuation_line(struct tok_state *tok) { - int c = tok_nextc(tok); - if (c == '\r') { - c = tok_nextc(tok); - } - if (c != '\n') { - tok->done = E_LINECONT; - return -1; - } - c = tok_nextc(tok); - if (c == EOF) { - tok->done = E_EOF; - tok->cur = tok->inp; - return -1; - } else { - tok_backup(tok, c); - } - return c; -} - - - int _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token) { @@ -160,102 +138,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token blankline = 0; - /* Get indentation level */ - if (tok->atbol) { - int col = 0; - int altcol = 0; - tok->atbol = 0; - int cont_line_col = 0; - for (;;) { - c = tok_nextc(tok); - if (c == ' ') { - col++, altcol++; - } - else if (c == '\t') { - col = (col / TABSIZE + 1) * TABSIZE; - altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; - } - else if (c == '\014') {/* Control-L (formfeed) */ - col = altcol = 0; /* For Emacs users */ - } - else if (c == '\\') { - // Indentation cannot be split over multiple physical lines - // using backslashes. This means that if we found a backslash - // preceded by whitespace, **the first one we find** determines - // the level of indentation of whatever comes next. - cont_line_col = cont_line_col ? cont_line_col : col; - if ((c = tok_continuation_line(tok)) == -1) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else if (c == EOF && PyErr_Occurred()) { - return MAKE_TOKEN(ERRORTOKEN); - } - else { - break; - } - } - tok_backup(tok, c); - if (c == '#' || c == '\n' || c == '\r') { - /* Lines with only whitespace and/or comments - shouldn't affect the indentation and are - not passed to the parser as NEWLINE tokens, - except *totally* empty lines in interactive - mode, which signal the end of a command group. */ - if (col == 0 && c == '\n' && tok->prompt != NULL) { - blankline = 0; /* Let it through */ - } - else if (tok->prompt != NULL && tok->lineno == 1) { - /* In interactive mode, if the first line contains - only spaces and/or a comment, let it through. */ - blankline = 0; - col = altcol = 0; - } - else { - blankline = 1; /* Ignore completely */ - } - /* We can't jump back right here since we still - may need to skip to the end of a comment */ - } - if (!blankline && tok->level == 0) { - col = cont_line_col ? cont_line_col : col; - altcol = cont_line_col ? cont_line_col : altcol; - if (col == tok->indstack[tok->indent]) { - /* No change */ - if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - } - else if (col > tok->indstack[tok->indent]) { - /* Indent -- always one */ - if (tok->indent+1 >= MAXINDENT) { - tok->done = E_TOODEEP; - tok->cur = tok->inp; - return MAKE_TOKEN(ERRORTOKEN); - } - if (altcol <= tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - tok->pendin++; - tok->indstack[++tok->indent] = col; - tok->altindstack[tok->indent] = altcol; - } - else /* col < tok->indstack[tok->indent] */ { - /* Dedent -- any number, must be consistent */ - while (tok->indent > 0 && - col < tok->indstack[tok->indent]) { - tok->pendin--; - tok->indent--; - } - if (col != tok->indstack[tok->indent]) { - tok->done = E_DEDENT; - tok->cur = tok->inp; - return MAKE_TOKEN(ERRORTOKEN); - } - if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - } + if (tok->layout.at_bol) { + blankline = _PyLexer_BeginLine(tok); + if (blankline < 0) { + return MAKE_TOKEN(ERRORTOKEN); } } @@ -263,24 +149,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token tok->start_loc = (_PyTok_Loc){ tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1}; - /* Return pending indents/dedents */ - if (tok->pendin != 0) { - if (tok->pendin < 0) { - if (tok->tok_extra_tokens) { - p_start = tok->cur; - p_end = tok->cur; - } - tok->pendin++; - return MAKE_TOKEN(DEDENT); - } - else { - if (tok->tok_extra_tokens) { - p_start = tok->buf_offset; - p_end = tok->cur; - } - tok->pendin--; - return MAKE_TOKEN(INDENT); - } + if (tok->layout.pending != 0) { + return _PyLexer_IndentationToken(tok, token); } /* Peek ahead at the next character */ @@ -373,7 +243,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token /* If this type ignore is the only thing on the line, consume the newline also. */ if (blankline) { tok_nextc(tok); - tok->atbol = 1; + tok->layout.at_bol = 1; } } else { p_start = _PyLexer_BufferOffset(tok, type_start); @@ -389,7 +259,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token tok_backup(tok, c); /* don't eat the newline or EOF */ p_start = _PyLexer_BufferOffset(tok, p); p_end = tok->cur; - tok->comment_newline = blankline; + tok->layout.comment_newline = blankline; return MAKE_TOKEN(COMMENT); } } @@ -470,29 +340,12 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token c = tok_nextc(tok); } - /* Newline */ if (c == '\n') { - tok->atbol = 1; - if (blankline || tok->level > 0) { - if (tok->tok_extra_tokens) { - if (tok->comment_newline) { - tok->comment_newline = 0; - } - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); - } + int type = _PyLexer_Newline(tok, token, blankline); + if (type == 0) { goto nextline; } - if (tok->comment_newline && tok->tok_extra_tokens) { - tok->comment_newline = 0; - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); - } - p_start = tok->start; - p_end = tok->cur - 1; /* Leave '\n' out of the string */ - return MAKE_TOKEN(NEWLINE); + return type; } /* Period or number starting with period? */ @@ -533,7 +386,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token /* Line continuation */ if (c == '\\') { - if ((c = tok_continuation_line(tok)) == -1) { + if ((c = _PyLexer_ContinueLine(tok)) == -1) { return MAKE_TOKEN(ERRORTOKEN); } goto again; /* Read next line */ diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h deleted file mode 100644 index 7302198cc91010b..000000000000000 --- a/Parser/lexer/lexer.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _PY_LEXER_LEXER_H_ -#define _PY_LEXER_LEXER_H_ - -#include "state.h" - -#endif diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index 210d182d42dd9f5..f0d9576d95f8ff3 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -2,7 +2,7 @@ #define _PY_LEXER_INTERNAL_H_ #include "errcode.h" -#include "lexer.h" +#include "state.h" #define is_potential_identifier_start(c) (\ (c >= 'a' && c <= 'z')\ @@ -50,6 +50,12 @@ tok_nextc(struct tok_state *tok) tok->source.bytes[tok->cur++ - tok->source.base_offset]); } +/* Return -1 on error, otherwise whether the line is blank. */ +int _PyLexer_BeginLine(struct tok_state *); +int _PyLexer_ContinueLine(struct tok_state *); +int _PyLexer_IndentationToken(struct tok_state *, struct token *); +/* Return zero when the newline is suppressed, otherwise its token type. */ +int _PyLexer_Newline(struct tok_state *, struct token *, int); void _PyLexer_backup(struct tok_state *, int); int _PyLexer_record_ftstring_comment( struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index a6617c33480c855..75ff26f16d47ba7 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_pystate.h" #include "pycore_token.h" #include "errcode.h" @@ -7,55 +6,6 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -/* Create and initialize a new tok_state structure */ -struct tok_state * -_PyTokenizer_tok_new(void) -{ - struct tok_state *tok = (struct tok_state *)PyMem_Calloc( - 1, - sizeof(struct tok_state)); - if (tok == NULL) { - PyErr_NoMemory(); - return NULL; - } - - tok->cur = tok->inp = 0; - tok->line_start = -1; - tok->fp_interactive = 0; - tok->interactive_src_start = NULL; - tok->interactive_src_end = NULL; - tok->start = -1; - tok->done = E_OK; - tok->fp = NULL; - tok->indent = 0; - tok->indstack[0] = 0; - tok->atbol = 1; - tok->pendin = 0; - tok->prompt = NULL; - tok->lineno = 0; - tok->start_loc = (_PyTok_Loc){-1, -1}; - tok->level = 0; - tok->altindstack[0] = 0; - tok->encoding = NULL; - tok->filename = NULL; - tok->module = NULL; - tok->type_comments = 0; - tok->interactive_underflow = IUNDERFLOW_NORMAL; - tok->str = NULL; - tok->report_warnings = 1; - tok->tok_extra_tokens = 0; - tok->comment_newline = 0; - tok->implicit_newline = 0; - _PyTok_SourceInit(&tok->source); - tok->reader = NULL; - tok->ftstring_stack = tok->ftstring_stack_inline; - tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY; -#ifdef Py_DEBUG - tok->debug = _Py_GetConfig()->parser_debug; -#endif - return tok; -} - ftstring_state * _PyLexer_PushFTString(struct tok_state *tok) { diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 0f9ddb6d45e9611..dff67c2ba83dae4 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -10,15 +10,6 @@ #define MAXFTSTRINGLEVEL 150 #define FTSTRING_STACK_INLINE_CAPACITY 1 -enum interactive_underflow_t { - /* Normal mode of operation: return a new token when asked in interactive mode */ - IUNDERFLOW_NORMAL, - /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This - * can be used to prevent the tokenizer to prompt the user for new tokens */ - IUNDERFLOW_STOP, -}; - - typedef enum { FTSTRING_MODE_MIDDLE, FTSTRING_MODE_EXPRESSION, @@ -66,6 +57,19 @@ _PyLexer_IsRawString(ftstring_kind kind) return kind == RAW_FSTRING || kind == RAW_TSTRING; } +typedef struct { + int column; + int alternate_column; +} indentation_level; + +typedef struct { + int depth; + int pending; + int at_bol; + int comment_newline; + indentation_level stack[MAXINDENT]; +} lexer_layout_state; + /* Tokenizer state */ struct tok_state { _PyTok_Off buf_offset; @@ -74,19 +78,13 @@ struct tok_state { _PyTok_Off start; _PyTok_Off line_start; _PyTok_SourceText source; - int fp_interactive; /* If the file descriptor is interactive */ - char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ - char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ - int indent; /* Current indentation index */ - int indstack[MAXINDENT]; /* Stack of indents */ - int atbol; /* Nonzero if at begin of new line */ - int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ - const char *prompt; /* For interactive prompting */ + lexer_layout_state layout; int lineno; /* Current line number */ _PyTok_Loc start_loc; + _PyTokenizer_Diagnostic diagnostic; int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; @@ -94,25 +92,18 @@ struct tok_state { int parencolstack[MAXLEVEL]; PyObject *filename; PyObject *module; - /* Stuff for checking on different tab sizes */ - int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ char *encoding; /* Source encoding. */ - char* str; /* Source string being tokenized (if tokenizing from a string)*/ struct _PyTok_Reader *reader; int type_comments; /* Whether to look for type comments */ - /* How to proceed when asked for a new token in interactive mode */ - enum interactive_underflow_t interactive_underflow; - int report_warnings; ftstring_state *ftstring_stack; ftstring_state ftstring_stack_inline[FTSTRING_STACK_INLINE_CAPACITY]; int ftstring_depth; int ftstring_capacity; int tok_extra_tokens; - int comment_newline; int implicit_newline; #ifdef Py_DEBUG int debug; @@ -182,7 +173,8 @@ _PyLexer_ByteColumn(const struct tok_state *tok) int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end); -struct tok_state *_PyTokenizer_tok_new(void); +void _PyLexer_ImplyDedents(struct tok_state *); + void _PyTokenizer_Free(struct tok_state *); ftstring_state *_PyLexer_PushFTString(struct tok_state *); void _PyLexer_PopFTString(struct tok_state *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 1fb684e337de972..a48cedf270362ab 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,13 +7,18 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -static void -rewind_to_string_start(struct tok_state *tok, _PyTok_Off start, - _PyTok_Loc location) +static int +string_error_token(struct tok_state *tok, struct token *token, + _PyTok_Off start, _PyTok_Loc location) { - tok->cur = start + 1; - tok->line_start = start - location.byte_col; - tok->lineno = location.lineno; + tok->diagnostic = (_PyTokenizer_Diagnostic){ + .location = {location.lineno, location.byte_col + 1}, + .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + }; + int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); + token->start_loc = location; + token->end_loc = (_PyTok_Loc){location.lineno, -1}; + return type; } int @@ -351,7 +356,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) } if (c == EOF || (quote_size == 1 && c == '\n')) { int end_lineno = tok->lineno; - rewind_to_string_start(tok, tok->start, tok->start_loc); + _PyTok_Loc location = tok->start_loc; + const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; const ftstring_state *state = _PyLexer_CurrentFTString(tok); if (state != NULL) { @@ -364,41 +371,49 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) assert(tok->parenstack[level] == '{'); int lineno = tok->parenlinenostack[level]; if (lineno != tok->lineno) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}' to close '{' on line %d", - _PyLexer_StringPrefix(state->kind), lineno)); + _PyLexer_StringPrefix(state->kind), lineno); + } + else { + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "%c-string: expecting '}'", + _PyLexer_StringPrefix(state->kind)); } - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expecting '}'", - _PyLexer_StringPrefix(state->kind))); + return string_error_token(tok, token, tok->start, location); } } if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", end_lineno); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted string literal" + " (detected at line %d)", end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } else { if (has_escaped_quote) { - _PyTokenizer_syntaxerror( - tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "unterminated string literal (detected at line %d); " "perhaps you escaped the end quote?", end_lineno ); } else { - _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", end_lineno + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated string literal (detected at line %d)", end_lineno ); } if (c != '\n') { tok->done = E_EOLS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } } if (c == quote) { @@ -462,25 +477,29 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok } int end_lineno = tok->lineno; - rewind_to_string_start(tok, - current->start, - current->start_loc); + _PyTok_Loc location = current->start_loc; + const char *line = _PyLexer_BufferPointer(tok, current->start) - location.byte_col; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, - "unterminated triple-quoted %c-string literal" - " (detected at line %d)", - _PyLexer_StringPrefix(current->kind), end_lineno); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted %c-string literal" + " (detected at line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, + current->start, location); } else { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "unterminated %c-string literal (detected at" - " line %d)", - _PyLexer_StringPrefix(current->kind), end_lineno)); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated %c-string literal (detected at line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno); + return string_error_token(tok, token, + current->start, location); } } diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 7841f01b612915c..74ab56c912ae1af 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -206,7 +206,9 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err Py_ssize_t end_col_offset = -1; if (t->col_offset == -1) { _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); - if (info.cursor == info.input_span.start) { + if (info.diagnostic.location.lineno != 0) { + col_offset = info.diagnostic.location.byte_col; + } else if (info.cursor == info.input_span.start) { col_offset = 0; } else { col_offset = Py_SAFE_DOWNCAST( @@ -256,8 +258,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, PyObject *tmp = NULL; p->error_indicator = 1; _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); - _PyTok_Loc location = info.location; - _PyTok_Span text_span = info.line_span; + _PyTok_Loc location = info.diagnostic.location.lineno != 0 + ? info.diagnostic.location : info.location; + _PyTok_Span text_span = info.diagnostic.location.lineno != 0 + ? info.diagnostic.text_span : info.line_span; if (end_lineno == CURRENT_POS) { end_lineno = location.lineno; diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c index 67e129d52441c76..3f5efa8dc8f26e6 100644 --- a/Parser/tokenizer/api.c +++ b/Parser/tokenizer/api.c @@ -4,7 +4,6 @@ #include "tokenizer.h" #include "reader.h" -#include "reader_internal.h" #include "../lexer/state.h" _PyTokenizer_Info @@ -12,6 +11,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok) { _PyTokenizer_Info info = { .status = tok->done, + .diagnostic = tok->diagnostic, .location = {tok->lineno, tok->line_start < 0 ? -1 : (int)(tok->cur - tok->line_start)}, .cursor = tok->cur, @@ -20,7 +20,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok) .level = tok->level, .delimiter_loc = {-1, -1}, .in_formatted_string = tok->ftstring_depth != 0, - .is_interactive = tok->reader->kind == _PYTOK_READER_INTERACTIVE, + .is_interactive = _PyTok_ReaderIsInteractive(tok), .is_file = tok->fp != NULL && tok->fp != stdin, .filename = tok->filename, .module = tok->module, @@ -81,32 +81,7 @@ const char * _PyTokenizer_LineView(const struct tok_state *tok, Py_ssize_t lineno, Py_ssize_t *length) { - const char *line = _PyTokenizer_RetainedSource(tok); - if (line == NULL) { - line = _PyLexer_BufferPointer(tok, tok->buf_offset); - } - for (Py_ssize_t i = 1; i < lineno; i++) { - const char *next = strchr(line, '\n'); - if (next == NULL) { - break; - } - line = next + 1; - } - const char *end = strchr(line, '\n'); - *length = end != NULL ? end - line : (Py_ssize_t)strlen(line); - return line; -} - -const char * -_PyTokenizer_RetainedSource(const struct tok_state *tok) -{ - if (tok->reader->kind == _PYTOK_READER_PREPARED) { - return _PyTok_SourceData(&tok->source); - } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) { - return tok->source.bytes; - } - return NULL; + return _PyTok_SourceLineView(&tok->source, lineno, length); } void @@ -130,10 +105,7 @@ _PyTokenizer_SetOptions(struct tok_state *tok, int extra_tokens, void _PyTokenizer_ImplyDedents(struct tok_state *tok) { - if (tok->indent != 0) { - tok->pendin = -tok->indent; - tok->indent = 0; - } + _PyLexer_ImplyDedents(tok); } int @@ -160,11 +132,11 @@ _PyTokenizer_HasTrailingStatement(const struct tok_state *tok) int _PyTokenizer_IsInteractive(const struct tok_state *tok) { - return tok->prompt != NULL; + return _PyTok_ReaderIsInteractive(tok); } void _PyTokenizer_StopInteractive(struct tok_state *tok) { - tok->interactive_underflow = IUNDERFLOW_STOP; + _PyTok_ReaderStopInteractive(tok); } diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c deleted file mode 100644 index 523b99dedc6160a..000000000000000 --- a/Parser/tokenizer/cursor.c +++ /dev/null @@ -1,82 +0,0 @@ -#include "Python.h" - -#include "cursor.h" - -static void -set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start, - _PyTok_Off end) -{ - cursor->pos = start; - cursor->line_start = start; - cursor->line_end = end; - cursor->lineno = lineno; -} - -int -_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - if (lineno > 0 && cursor->lineno == lineno - 1 && - lineno <= source->nlines) { - _PyTok_Off start = cursor->line_end; - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, lineno, start, end); - return 0; - } - - _PyTok_Line line; - if (_PyTok_SourceLine(source, lineno, &line) < 0) { - return -1; - } - set_line(cursor, lineno, line.start, line.end); - return 0; -} - -int -_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - int stays_on_line = cursor->lineno > 0 && - offset >= cursor->line_start && offset < cursor->line_end; - if (!stays_on_line && cursor->lineno > 0 && - offset == cursor->line_end && - offset - source->base_offset == source->len && - (source->len == 0 || source->bytes[source->len - 1] != '\n')) { - stays_on_line = 1; - } - if (stays_on_line) { - cursor->pos = offset; - return 0; - } - - _PyTok_Loc loc; - if (_PyTok_SourceLocation( - source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) { - return -1; - } - _PyTok_Off start = offset - loc.byte_col; - _PyTok_Off end = source->base_offset + source->len; - if (loc.lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, loc.lineno, start, end); - cursor->pos = offset; - return 0; -} diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h deleted file mode 100644 index 18e404251316f0c..000000000000000 --- a/Parser/tokenizer/cursor.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef Py_TOKENIZER_CURSOR_H -#define Py_TOKENIZER_CURSOR_H - -#include "source.h" - -typedef struct { - /* The source must remain initialized at this address while in use. */ - const _PyTok_SourceText *source; - _PyTok_Off pos; - _PyTok_Off line_start; - _PyTok_Off line_end; - int lineno; -} _PyTok_Cursor; - -/* Move to the start of a 1-based line. Both setters preserve the cursor on - error. */ -PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int); -/* Move to an offset. A line boundary selects the following line. */ -PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off); - -static inline void -_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source) -{ - _PyTok_Off base = source != NULL ? source->base_offset : 0; - *cursor = (_PyTok_Cursor){ - .source = source, - .pos = base, - .line_start = base, - .line_end = base, - }; -} - -/* Read one byte from the current line, including its terminating newline. - EOF marks the line boundary, not necessarily the end of the source. It is - also returned if advancing would make the byte column unrepresentable. */ -static inline int -_PyTok_CursorAdvance(_PyTok_Cursor *cursor) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_start >= cursor->source->base_offset); - assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); - if (cursor->pos >= cursor->line_end) { - return EOF; - } - if (cursor->pos - cursor->line_start >= INT_MAX) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[ - cursor->pos++ - cursor->source->base_offset]); -} - -/* Return the byte at a nonnegative distance within the current line, or EOF - if the distance reaches or crosses the line boundary. */ -static inline int -_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_start >= cursor->source->base_offset); - assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); - assert(distance >= 0); - if (distance < 0 || - distance >= cursor->line_end - cursor->pos) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[ - cursor->pos - cursor->source->base_offset + distance]); -} - -#endif diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index be7442af55c4e31..69c3bb371add960 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -103,7 +103,9 @@ normalize_newlines_into(char *result, const char *data, Py_ssize_t len, } result[write] = '\0'; *out_len = write; - *implicit_newline = implicit; + if (implicit_newline != NULL) { + *implicit_newline = implicit; + } } char * @@ -262,7 +264,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, end_col--; } _PyTokenizer_syntaxerror_at( - tok, line_data, 0, cookie_line, 0, end_col, "encoding problem: %s with BOM", cookie); + tok, line_data, 0, cookie_line, 0, end_col, + "encoding problem: %s with BOM", cookie); PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } @@ -420,10 +423,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only, if (stored < 0) { return -1; } - tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)""; if (!utf8_only && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && - !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) { + !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) { return -1; } return 0; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index d0ada5ac1131a71..0d3ea85109ec49e 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -137,10 +137,6 @@ _PyTokenizer_indenterror(struct tok_state *tok) int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { - if (!tok->report_warnings) { - return 0; - } - PyObject *msg = PyUnicode_FromFormat( "\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. " @@ -226,10 +222,6 @@ _PyTokenizer_raise_init_error(PyObject *filename) int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { - if (!tok->report_warnings) { - return 0; - } - PyObject *errmsg; va_list vargs; va_start(vargs, format); @@ -341,12 +333,9 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) } } if (badchar) { - tok->lineno = lineno; - tok->line_start = _PyLexer_BufferOffset(tok, line_start); - tok->cur = _PyLexer_BufferOffset(tok, badchar); - _PyTokenizer_syntaxerror_known_range(tok, - (int)(badchar - line_start) + 1, - (int)(badchar - line_start) + 1, + _PyTokenizer_syntaxerror_at( + tok, line_start, badchar - line_start + 1, lineno, + -1, -1, "Non-UTF-8 code starting with '\\x%.2x'" "%s%V on line %i, " "but no encoding declared; " @@ -389,12 +378,4 @@ _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size) } putc('"', f); } - -void -_PyTokenizer_tok_dump(int type, char *start, char *end) -{ - fprintf(stderr, "%s", _PyParser_TokenNames[type]); - if (type == NAME || type == NUMBER || type == STRING || type == OP) - fprintf(stderr, "(%.*s)", (int)(end - start), start); -} #endif diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 24f2d0cff1effb6..f33d8f55ed1d17c 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -5,20 +5,24 @@ #include "../lexer/state.h" -int _PyTokenizer_syntaxerror_at(struct tok_state *, const char *, - Py_ssize_t, int, int, int, const char *, ...); int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); +/* Positive range columns are 1-based byte columns. A start column of -1 + derives the character column from the reporting cursor; an end column of + -1 uses the start column. */ int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); +int _PyTokenizer_syntaxerror_at( + struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset, + int lineno, int col_offset, int end_col_offset, const char *format, ...); int _PyTokenizer_indenterror(struct tok_state *tok); int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); + void _PyTokenizer_raise_init_error(PyObject *filename); int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno); #ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); -void _PyTokenizer_tok_dump(int type, char *start, char *end); #endif diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 77bfecc0cf12f70..c003deba18dd8f2 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,11 +1,11 @@ #include "Python.h" #include "pycore_fileutils.h" +#include "pycore_pystate.h" #include "errcode.h" #include "helpers.h" #include "reader.h" #include "reader_internal.h" -#include "../lexer/lexer.h" #include "../lexer/state.h" #ifdef HAVE_UNISTD_H @@ -139,10 +139,10 @@ chunk_is_line(const _PyTok_Chunk *chunk) static _PyTok_ReadResult next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) { - int lineno = tok->lineno + 1; - if (lineno > tok->source.nlines) { + if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } + int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -208,7 +208,6 @@ initialize_file(struct tok_state *tok) if (result != _PYTOK_READ_LINE) { return -1; } - reader->prefetched_count = 1; Py_ssize_t bom_len; _PyTok_EncodingResult detection = _PyTok_DetectEncoding( tok, &reader->prefetched_lines[0], NULL, 0, &bom_len); @@ -226,16 +225,13 @@ initialize_file(struct tok_state *tok) reader->prefetched_lines[0].data = first; reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM; result = read_file_line(tok, &reader->prefetched_lines[1]); - if (result == _PYTOK_READ_LINE) { - reader->prefetched_count = 2; - } - else if (result == _PYTOK_READ_EOF) { + if (result == _PYTOK_READ_EOF) { reader->file_eof = 1; } - else { + else if (result != _PYTOK_READ_LINE) { return -1; } - _PyTok_Chunk *second = reader->prefetched_count == 2 + _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL ? &reader->prefetched_lines[1] : NULL; detection = _PyTok_DetectEncoding( tok, &reader->prefetched_lines[0], second, 1, &bom_len); @@ -305,10 +301,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } _PyTok_Chunk input = {0}; - if (reader->prefetched_index < reader->prefetched_count) { - input = reader->prefetched_lines[reader->prefetched_index]; - reader->prefetched_lines[reader->prefetched_index++] = - (_PyTok_Chunk){0}; + if (reader->prefetched_lines[0].data != NULL) { + input = reader->prefetched_lines[0]; + reader->prefetched_lines[0] = (_PyTok_Chunk){0}; + } + else if (reader->prefetched_lines[1].data != NULL) { + input = reader->prefetched_lines[1]; + reader->prefetched_lines[1] = (_PyTok_Chunk){0}; } else if (!reader->file_eof) { _PyTok_ReadResult result = read_file_line(tok, &input); @@ -477,13 +476,13 @@ static _PyTok_ReadResult next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) { _PyTok_Reader *reader = tok->reader; - if (tok->interactive_underflow == IUNDERFLOW_STOP) { + if (reader->stop_interactive) { return _PYTOK_READ_STOPPED; } char *input = PyOS_Readline( - tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt); + tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt); if (reader->nextprompt != NULL) { - tok->prompt = reader->nextprompt; + reader->prompt = reader->nextprompt; } if (input == NULL) { return _PYTOK_READ_INTERRUPT; @@ -506,7 +505,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) } chunk->data = _PyTok_NormalizeNewlines( decoded.data, decoded.len, 0, 0, - &chunk->len, &chunk->implicit_newline); + &chunk->len, NULL); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { PyErr_NoMemory(); @@ -517,6 +516,32 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } +int +_PyTok_ReaderIsInteractive(const struct tok_state *tok) +{ + return tok->reader->kind == _PYTOK_READER_INTERACTIVE; +} + +const char * +_PyTokenizer_RetainedSource(const struct tok_state *tok) +{ + if (tok->reader->kind == _PYTOK_READER_PREPARED) { + return _PyTok_SourceData(&tok->source); + } + if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) { + return tok->source.bytes; + } + return NULL; +} + +void +_PyTok_ReaderStopInteractive(struct tok_state *tok) +{ + if (_PyTok_ReaderIsInteractive(tok)) { + tok->reader->stop_interactive = 1; + } +} + static _PyTok_ReadResult reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) { @@ -581,12 +606,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) } return 0; } - - Py_ssize_t scan_len = chunk.len; - if (kind == _PYTOK_READER_INTERACTIVE && - chunk.implicit_newline) { - scan_len--; + if (tok->lineno == INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines"); + tok->done = E_ERROR; + _PyTok_ChunkClear(&chunk); + return 0; } + if (!prepared) { if (streaming && reset_buffer) { reset_streaming_buffer(tok); @@ -606,11 +632,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->line_start = tok->buf_offset; tok->start = -1; } - tok->inp = source_start + scan_len; - } - if (tok->fp_interactive) { - tok->interactive_src_start = tok->source.bytes; - tok->interactive_src_end = tok->source.bytes + tok->source.len; + tok->inp = source_start + chunk.len; } if (prepared) { if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) { @@ -635,10 +657,21 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) static struct tok_state * tokenizer_new_with_reader(_PyTok_ReaderKind kind) { - struct tok_state *tok = _PyTokenizer_tok_new(); + struct tok_state *tok = PyMem_Calloc(1, sizeof(*tok)); if (tok == NULL) { + PyErr_NoMemory(); return NULL; } + tok->start = tok->line_start = -1; + _PyTok_SourceInit(&tok->source); + tok->done = E_OK; + tok->layout.at_bol = 1; + tok->start_loc = (_PyTok_Loc){-1, -1}; + tok->ftstring_stack = tok->ftstring_stack_inline; + tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY; +#ifdef Py_DEBUG + tok->debug = _Py_GetConfig()->parser_debug; +#endif tok->reader = PyMem_Calloc(1, sizeof(*tok->reader)); if (tok->reader == NULL) { PyErr_NoMemory(); @@ -716,7 +749,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding, return NULL; } tok->fp = fp; - tok->prompt = ps1; + tok->reader->prompt = ps1; tok->reader->nextprompt = ps2; return tok; } @@ -770,13 +803,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) _PyTokenizer_Free(tok); return NULL; } - /* Reporting a warning here could recursively ask for the encoding. */ - tok->report_warnings = 0; - while (tok->lineno < 2 && tok->done == E_OK) { - struct token token; - _PyToken_Init(&token); - _PyTokenizer_Get(tok, &token); - _PyToken_Free(&token); + if (initialize_file(tok) < 0) { + fclose(fp); + _PyTokenizer_Free(tok); + return NULL; } fclose(fp); char *encoding = tok->encoding == NULL diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h index c27bc2aa3fb8197..2913e52b9d563b4 100644 --- a/Parser/tokenizer/reader.h +++ b/Parser/tokenizer/reader.h @@ -5,5 +5,7 @@ struct tok_state; void _PyTok_ReaderFree(struct tok_state *); int _PyTok_ReaderUnderflow(struct tok_state *); +int _PyTok_ReaderIsInteractive(const struct tok_state *); +void _PyTok_ReaderStopInteractive(struct tok_state *); #endif diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 5071df35c142d31..390b6ed9d8581a5 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -32,33 +32,32 @@ typedef enum { typedef struct { char *data; - Py_ssize_t len; - int implicit_newline; PyObject *owner; + Py_ssize_t len; _PyTok_ChunkOwnership ownership; + unsigned char implicit_newline; } _PyTok_Chunk; typedef struct _PyTok_Reader { - _PyTok_ReaderKind kind; PyObject *readline; PyObject *decoder; + const char *prompt; const char *nextprompt; char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; - int prefetched_index; - int prefetched_count; char *decoded; Py_ssize_t decoded_pos; Py_ssize_t decoded_len; Py_ssize_t decoded_cap; - int decoded_tail_is_implicit; - - int file_initialized; - int file_eof; - int decoder_finalized; + _PyTok_ReaderKind kind; + unsigned char decoded_tail_is_implicit; + unsigned char file_initialized; + unsigned char file_eof; + unsigned char decoder_finalized; + unsigned char stop_interactive; } _PyTok_Reader; struct tok_state; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 2f2aaf2589246d9..d69eab93923e81c 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -2,8 +2,6 @@ #include "source.h" -#define LINE_CHECKPOINT_INTERVAL 256 - void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -14,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->line_checkpoints); PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -74,34 +71,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_checkpoints(_PyTok_SourceText *source, int needed) -{ - if (needed <= source->checkpoints_cap) { - return 0; - } - int cap; - if (source->checkpoints_cap == 0) { - cap = 16; - } - else if (source->checkpoints_cap <= INT_MAX / 2) { - cap = source->checkpoints_cap * 2; - } - else { - PyErr_NoMemory(); - return -1; - } - _PyTok_Off *checkpoints = source->line_checkpoints; - PyMem_Resize(checkpoints, _PyTok_Off, cap); - if (checkpoints == NULL) { - PyErr_NoMemory(); - return -1; - } - source->line_checkpoints = checkpoints; - source->checkpoints_cap = cap; - return 0; -} - static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { @@ -165,11 +134,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, return -1; } int nlines = source->nlines + 1; - int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0; - int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - if ((checkpoint && - reserve_checkpoints(source, checkpoint_count) < 0) || - (implicit_newline && reserve_implicit_lines(source, nlines) < 0) || + if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -178,10 +143,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = - source->base_offset + start; - } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); @@ -191,16 +152,23 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, } const char * -_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, +_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len) { - if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset || - span.end - source->base_offset > source->len || len == NULL) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span"); - return NULL; + assert(len != NULL); + const char *line = _PyTok_SourceData(source); + const char *end = line + source->len; + while (lineno > 1) { + const char *newline = memchr(line, '\n', end - line); + if (newline == NULL) { + break; + } + line = newline + 1; + lineno--; } - *len = span.end - span.start; - return _PyTok_SourceData(source) + (span.start - source->base_offset); + const char *newline = memchr(line, '\n', end - line); + *len = (newline != NULL ? newline : end) - line; + return line; } int @@ -213,124 +181,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) return (source->implicit_lines[(lineno - 1) / 8] >> ((lineno - 1) & 7)) & 1; } - -static int -source_ends_in_newline(const _PyTok_SourceText *source) -{ - return source->len > 0 && source->bytes[source->len - 1] == '\n'; -} - -static int -eof_lineno(const _PyTok_SourceText *source) -{ - if (source->nlines == 0) { - return 1; - } - return source->nlines + source_ends_in_newline(source); -} - -int -_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, - _PyTok_Line *line) -{ - if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); - return -1; - } - if (lineno > source->nlines) { - *line = (_PyTok_Line){ - .start = source->base_offset + source->len, - .end = source->base_offset + source->len, - }; - return 0; - } - - int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL; - int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (current < lineno) { - start = _PyTok_SourceFindLineEnd(source, start); - if (start < 0) { - return -1; - } - current++; - } - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - *line = (_PyTok_Line){ - .start = start, - .end = end, - .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), - .contains_nul = memchr( - source->bytes + (start - source->base_offset), - 0, end - start) != NULL, - }; - return 0; -} - -int -_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, - _PyTok_Affinity affinity, _PyTok_Loc *loc) -{ - if (offset < source->base_offset || - offset - source->base_offset > source->len || loc == NULL || - (affinity != _PYTOK_AFFINITY_LEFT && - affinity != _PYTOK_AFFINITY_RIGHT)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); - return -1; - } - if (source->nlines == 0 || - (offset - source->base_offset == source->len && - source_ends_in_newline(source) && - affinity == _PYTOK_AFFINITY_RIGHT)) { - *loc = (_PyTok_Loc){eof_lineno(source), 0}; - return 0; - } - - _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) { - key--; - } - int low = 0; - int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - while (low < high) { - int middle = low + (high - low) / 2; - if (source->line_checkpoints[middle] <= key) { - low = middle + 1; - } - else { - high = middle; - } - } - int checkpoint = low - 1; - if (checkpoint < 0) { - PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); - return -1; - } - int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (lineno < source->nlines) { - _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - if (offset < end || - (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) { - break; - } - start = end; - lineno++; - } - _PyTok_Off byte_col = offset - start; - if (byte_col > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large"); - return -1; - } - *loc = (_PyTok_Loc){lineno, (int)byte_col}; - return 0; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 2f74ed8b1f4fab3..9576419aade1d46 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -5,28 +5,13 @@ #include "types.h" -typedef enum { - _PYTOK_AFFINITY_LEFT, - _PYTOK_AFFINITY_RIGHT, -} _PyTok_Affinity; - -/* The half-open range includes the terminating newline when present. */ -typedef struct { - _PyTok_Off start; - _PyTok_Off end; - unsigned implicit_newline : 1; - unsigned contains_nul : 1; -} _PyTok_Line; - typedef struct { char *bytes; _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - _PyTok_Off *line_checkpoints; unsigned char *implicit_lines; int nlines; - int checkpoints_cap; Py_ssize_t implicit_cap; } _PyTok_SourceText; @@ -37,9 +22,9 @@ _PyTok_SourceData(const _PyTok_SourceText *source) } PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); -/* Clear invalidates all cursors, spans, and views for the source. */ +/* Clear invalidates all spans and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); -/* Discard the retained window and invalidate its cursors, spans, and views. +/* Discard the retained window and invalidate its spans and views. Keep its allocation and advance the logical base to the end of the window. */ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may @@ -49,39 +34,14 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, int implicit_newline); -/* The returned view is invalidated by SourceAppendLine and SourceClear. */ -PyAPI_FUNC(const char *) _PyTok_SourceSpanView( - const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line in the retained window. Empty and newline-terminated - sources have an empty virtual line at EOF. */ -PyAPI_FUNC(int) _PyTok_SourceLine( - const _PyTok_SourceText *, int, _PyTok_Line *); +/* Return borrowed bytes excluding '\n', writing the byte length to *len. + Line numbers are 1-based and clamp to the first or final line; a trailing + '\n' adds an empty final line. The view need not be NUL-terminated. + This does not set an exception. Append, discard, and clear invalidate the view. */ +PyAPI_FUNC(const char *) _PyTok_SourceLineView( + const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); /* Return false for invalid line numbers and the virtual EOF line. */ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( const _PyTok_SourceText *, int); -/* At a line boundary, left affinity selects the preceding line at its end; - right affinity selects the following line at byte column zero. */ -PyAPI_FUNC(int) _PyTok_SourceLocation( - const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *); - -static inline _PyTok_Off -_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) -{ - if (source->bytes == NULL || start < source->base_offset || - start - source->base_offset >= source->len) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - _PyTok_Off relative_start = start - source->base_offset; - const char *newline = memchr( - source->bytes + relative_start, '\n', source->len - relative_start); - if (newline == NULL) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - return source->base_offset + (newline - source->bytes) + 1; -} #endif diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index e9229d120871624..82a84830d7cbfe7 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -28,8 +28,17 @@ typedef struct { int at_eof; } _PyToken_View; +/* Supplemental source context for a terminal error. location is the reporting + cursor, independent of the scanner cursor; lineno == 0 means absent. + The text span may cover multiple physical lines. */ +typedef struct { + _PyTok_Loc location; + _PyTok_Span text_span; +} _PyTokenizer_Diagnostic; + typedef struct { int status; + _PyTokenizer_Diagnostic diagnostic; _PyTok_Loc location; _PyTok_Off cursor; _PyTok_Span input_span; @@ -98,6 +107,4 @@ struct tok_state *_PyTokenizer_FromFile( An exception is set on error. */ char *_PyTokenizer_FindEncodingFilename(int, PyObject *); -#define tok_dump _Py_tok_dump - #endif /* !Py_TOKENIZER_H */ diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7e82a6365d9808b..79e817829c15947 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -31,6 +31,9 @@ extern void Py_SetPythonHome(const wchar_t *); // for the stable ABI. We want to test them in this program. extern void PySys_ResetWarnOptions(void); +// Variable removed from Python 3.16 limited C API, but kept in the stable ABI +PyAPI_DATA(int) Py_UTF8Mode; + int main_argc; char **main_argv; diff --git a/Python/future.c b/Python/future.c index 79b6c0c503bace1..41c4fa07916189a 100644 --- a/Python/future.c +++ b/Python/future.c @@ -1,9 +1,10 @@ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_pyerrors.h" // _Py_CalculateSuggestions() #include "pycore_symtable.h" // _PyFutureFeatures #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() -#define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" +#define UNDEFINED_FUTURE_FEATURE "future feature '%.100s' is not defined" static int future_check_features(_PyFutureFeatures *ff, stmt_ty s, PyObject *filename) @@ -48,8 +49,37 @@ future_check_features(_PyFutureFeatures *ff, stmt_ty s, PyObject *filename) name->end_col_offset + 1); return 0; } else { - PyErr_Format(PyExc_SyntaxError, - UNDEFINED_FUTURE_FEATURE, feature); + // Keep this list in sync with the feature checks above. + PyObject *future_features = Py_BuildValue("[ssssssssss]", + FUTURE_NESTED_SCOPES, + FUTURE_GENERATORS, + FUTURE_DIVISION, + FUTURE_ABSOLUTE_IMPORT, + FUTURE_WITH_STATEMENT, + FUTURE_PRINT_FUNCTION, + FUTURE_UNICODE_LITERALS, + FUTURE_BARRY_AS_BDFL, + FUTURE_GENERATOR_STOP, + FUTURE_ANNOTATIONS); + PyObject *suggestion = NULL; + if (future_features != NULL) { + suggestion = _Py_CalculateSuggestions(future_features, + name->name); + } + if (suggestion != NULL) { + PyErr_Format(PyExc_SyntaxError, + UNDEFINED_FUTURE_FEATURE ". Did you mean: %R?", + feature, suggestion); + Py_DECREF(suggestion); + } + else { + // Do not fail on missing suggestion, + // just show the default message. + PyErr_Format(PyExc_SyntaxError, + UNDEFINED_FUTURE_FEATURE, + feature); + } + Py_XDECREF(future_features); PyErr_RangedSyntaxLocationObject(filename, name->lineno, name->col_offset + 1, diff --git a/Python/initconfig.c b/Python/initconfig.c index 464c76f9e3df2fc..d683fdd6abc6e1b 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -95,128 +95,115 @@ typedef struct { PyConfigMemberType type; PyConfigMemberVisibility visibility; PyConfigSysSpec sys; - PyConfigGlobalVar global_var; } PyConfigSpec; -#define SPEC(MEMBER, TYPE, VISIBILITY, sys, global_var) \ +#define SPEC(MEMBER, TYPE, VISIBILITY, sys) \ {#MEMBER, offsetof(PyConfig, MEMBER), \ - PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys, global_var} + PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys} #define SYS_ATTR(name) {name, -1, NULL} #define SYS_FLAG_SETTER(index, setter) {NULL, index, setter} #define SYS_FLAG(index) SYS_FLAG_SETTER(index, NULL) #define NO_SYS SYS_ATTR(NULL) -#define GLOBAL(ptr, not) {ptr, not} -#define NO_GLOBAL GLOBAL(NULL, 0) - -// Ignore deprecations on global variables such as Py_IsolatedFlag -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - // Update _test_embed_set_config when adding new members static const PyConfigSpec PYCONFIG_SPEC[] = { // --- Public options ----------- - SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv"), NO_GLOBAL), - SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix"), NO_GLOBAL), - SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable"), NO_GLOBAL), - SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix"), NO_GLOBAL), - SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9), GLOBAL(&Py_BytesWarningFlag, 0)), - SPEC(cpu_count, INT, PUBLIC, NO_SYS, NO_GLOBAL), - SPEC(lazy_imports, INT, PUBLIC, NO_SYS, NO_GLOBAL), - SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix"), NO_GLOBAL), - SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable"), NO_GLOBAL), - SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1), GLOBAL(&Py_InspectFlag, 0)), - SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS, NO_GLOBAL), - SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2), GLOBAL(&Py_InteractiveFlag, 0)), - SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path"), NO_GLOBAL), - SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3), GLOBAL(&Py_OptimizeFlag, 0)), - SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0), GLOBAL(&Py_DebugFlag, 0)), - SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir"), NO_GLOBAL), - SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix"), NO_GLOBAL), - SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix"), NO_GLOBAL), - SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10), GLOBAL(&Py_QuietFlag, 0)), - SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir"), NO_GLOBAL), + SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv")), + SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix")), + SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable")), + SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix")), + SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9)), + SPEC(cpu_count, INT, PUBLIC, NO_SYS), + SPEC(lazy_imports, INT, PUBLIC, NO_SYS), + SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix")), + SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable")), + SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1)), + SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS), + SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2)), + SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path")), + SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3)), + SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0)), + SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir")), + SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix")), + SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix")), + SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10)), + SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir")), SPEC(use_environment, BOOL, PUBLIC, - SYS_FLAG_SETTER(7, config_sys_flag_not), GLOBAL(&Py_IgnoreEnvironmentFlag, 1)), - SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8), GLOBAL(&Py_VerboseFlag, 0)), - SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions"), NO_GLOBAL), - SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not), - GLOBAL(&Py_DontWriteBytecodeFlag, 1)), - SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions"), NO_GLOBAL), + SYS_FLAG_SETTER(7, config_sys_flag_not)), + SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8)), + SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions")), + SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not)), + SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions")), // --- Read-only options ----------- #ifdef Py_STATS - SPEC(_pystats, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_pystats, BOOL, READ_ONLY, NO_SYS), #endif - SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS, - GLOBAL(&Py_UnbufferedStdioFlag, 1)), - SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), // sys.flags.dev_mode - SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS), + SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS), + SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS), + SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS), + SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS), // sys.flags.dev_mode + SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS), + SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS), #ifdef Py_GIL_DISABLED - SPEC(enable_gil, INT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(enable_gil, INT, READ_ONLY, NO_SYS), + SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS), #endif - SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(import_time, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(isolated, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_IsolatedFlag, 0)), // sys.flags.isolated + SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS), + SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS), + SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS), + SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS), + SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS), + SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS), + SPEC(import_time, UINT, READ_ONLY, NO_SYS), + SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS), + SPEC(isolated, BOOL, READ_ONLY, NO_SYS), // sys.flags.isolated #ifdef MS_WINDOWS - SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS, - GLOBAL(&Py_LegacyWindowsStdioFlag, 0)), + SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS), #endif - SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(pymalloc_hugepages, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv"), NO_GLOBAL), - SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS, - GLOBAL(&Py_FrozenFlag, 1)), - SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(remote_debug, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(program_name, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS), + SPEC(pymalloc_hugepages, BOOL, READ_ONLY, NO_SYS), + SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), + SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), + SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), + SPEC(remote_debug, BOOL, READ_ONLY, NO_SYS), + SPEC(program_name, WSTR, READ_ONLY, NO_SYS), + SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS), #ifdef Py_DEBUG - SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS), #endif - SPEC(safe_path, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site - SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(stdio_encoding, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(safe_path, BOOL, READ_ONLY, NO_SYS), + SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS), + SPEC(site_import, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_site + SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS), + SPEC(stdio_encoding, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS), + SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), + SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), + SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), #ifdef __APPLE__ - SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS), #endif - SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS, - GLOBAL(&Py_NoUserSiteDirectory, 1)), // sys.flags.no_user_site - SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site + SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), // --- Init-only options ----------- - SPEC(_config_init, UINT, INIT_ONLY, NO_SYS, NO_GLOBAL), - SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), - SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), - SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), - SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), - SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS, NO_GLOBAL), - SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_config_init, UINT, INIT_ONLY, NO_SYS), + SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS), + SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS), + SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS), + SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS), + SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS), + SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS), // Array terminator {NULL, 0, 0, 0, NO_SYS}, @@ -252,16 +239,11 @@ static const PyConfigSpec PYPRECONFIG_SPEC[] = { {NULL, 0, 0, 0, NO_SYS}, }; -// End of ignoring deprecations on global variables -_Py_COMP_DIAG_POP - #undef SPEC #undef SYS_ATTR #undef SYS_FLAG_SETTER #undef SYS_FLAG #undef NO_SYS -#undef GLOBAL -#undef NO_GLOBAL // Forward declarations @@ -611,45 +593,30 @@ static const char usage_envvars[] = /* --- Global configuration variables ----------------------------- */ +// Variable removed from Python limited C API 3.16, but kept in the stable ABI +PyAPI_DATA(int) Py_UTF8Mode; + /* UTF-8 mode (PEP 540): if equal to 1, use the UTF-8 encoding, and change stdin and stdout error handler to "surrogateescape". */ int Py_UTF8Mode = 0; -int Py_DebugFlag = 0; /* Needed by parser.c */ -int Py_VerboseFlag = 0; /* Needed by import.c */ -int Py_QuietFlag = 0; /* Needed by sysmodule.c */ -int Py_InteractiveFlag = 0; /* Previously, was used by Py_FdIsInteractive() */ -int Py_InspectFlag = 0; /* Needed to determine whether to exit at SystemExit */ -int Py_OptimizeFlag = 0; /* Needed by compile.c */ -int Py_NoSiteFlag = 0; /* Suppress 'import site' */ -int Py_BytesWarningFlag = 0; /* Warn on str(bytes) and str(buffer) */ -int Py_FrozenFlag = 0; /* Needed by getpath.c */ -int Py_IgnoreEnvironmentFlag = 0; /* e.g. PYTHONPATH, PYTHONHOME */ -int Py_DontWriteBytecodeFlag = 0; /* Suppress writing bytecode files (*.pyc) */ -int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ -int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ -int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ -int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */ -#ifdef MS_WINDOWS -int Py_LegacyWindowsFSEncodingFlag = 0; /* Uses mbcs instead of utf-8 */ -int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ -#endif static PyObject * _Py_GetGlobalVariablesAsDict(void) { -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - PyObject *dict, *obj; + extern const char *Py_FileSystemDefaultEncoding; + extern const char *Py_FileSystemDefaultEncodeErrors; + extern int Py_HasFileSystemDefaultEncoding; + extern int _Py_HasFileSystemDefaultEncodeErrors; - dict = PyDict_New(); + PyObject *dict = PyDict_New(); if (dict == NULL) { return NULL; } #define SET_ITEM(KEY, EXPR) \ do { \ - obj = (EXPR); \ + PyObject *obj = (EXPR); \ if (obj == NULL) { \ goto fail; \ } \ @@ -674,27 +641,6 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS SET_ITEM_INT(_Py_HasFileSystemDefaultEncodeErrors); SET_ITEM_INT(Py_UTF8Mode); - SET_ITEM_INT(Py_DebugFlag); - SET_ITEM_INT(Py_VerboseFlag); - SET_ITEM_INT(Py_QuietFlag); - SET_ITEM_INT(Py_InteractiveFlag); - SET_ITEM_INT(Py_InspectFlag); - - SET_ITEM_INT(Py_OptimizeFlag); - SET_ITEM_INT(Py_NoSiteFlag); - SET_ITEM_INT(Py_BytesWarningFlag); - SET_ITEM_INT(Py_FrozenFlag); - SET_ITEM_INT(Py_IgnoreEnvironmentFlag); - SET_ITEM_INT(Py_DontWriteBytecodeFlag); - SET_ITEM_INT(Py_NoUserSiteDirectory); - SET_ITEM_INT(Py_UnbufferedStdioFlag); - SET_ITEM_INT(Py_HashRandomizationFlag); - SET_ITEM_INT(Py_IsolatedFlag); - -#ifdef MS_WINDOWS - SET_ITEM_INT(Py_LegacyWindowsFSEncodingFlag); - SET_ITEM_INT(Py_LegacyWindowsStdioFlag); -#endif return dict; @@ -706,7 +652,6 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS #undef SET_ITEM #undef SET_ITEM_INT #undef SET_ITEM_STR -_Py_COMP_DIAG_POP } char* @@ -1834,38 +1779,6 @@ config_read_preconfig(PyConfig *config) } -/* Set Py_xxx global configuration variables from 'config' configuration. */ -static void -config_set_global_vars(const PyConfig *config) -{ - const PyConfigSpec *spec = PYCONFIG_SPEC; - for (; spec->name != NULL; spec++) { - if (spec->global_var.ptr == NULL) { - continue; - } - assert(spec->type == PyConfig_MEMBER_INT - || spec->type == PyConfig_MEMBER_UINT - || spec->type == PyConfig_MEMBER_BOOL); - int *member = config_get_spec_member(config, spec); - int value = *member; - if (value == -1) { - continue; - } - if (spec->global_var.not) { - value = !value; - } - *spec->global_var.ptr = value; - } - -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - /* Random or non-zero hash seed */ - Py_HashRandomizationFlag = (config->use_hash_seed == 0 || - config->hash_seed != 0); -_Py_COMP_DIAG_POP -} - - static const wchar_t* config_get_xoption(const PyConfig *config, wchar_t *name) { @@ -3012,8 +2925,6 @@ config_init_stdio(const PyConfig *config) PyStatus _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) { - config_set_global_vars(config); - if (config->configure_c_stdio) { config_init_stdio(config); } @@ -5043,16 +4954,6 @@ PyConfig_Set(const char *name, PyObject *value) Py_UNREACHABLE(); } - // Set the global variable - if (spec->global_var.ptr != NULL) { - assert(has_int_value); - int value = int_value; - if (spec->global_var.not) { - value = !value; - } - *spec->global_var.ptr = value; - } - if (spec->sys.attr != NULL) { // Set the sys attribute, but don't set PyInterpreterState.config // to keep the code simple. diff --git a/Python/preconfig.c b/Python/preconfig.c index 16594e545abaedf..844ac8e6372fc52 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -13,10 +13,18 @@ /* Forward declarations */ static void preconfig_copy(PyPreConfig *config, const PyPreConfig *config2); +extern int Py_UTF8Mode; /* --- File system encoding/errors -------------------------------- */ +// Variables removed from Python limited C API 3.16, but kept in the stable ABI +PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; +PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; +PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; + +// The default encoding used by the platform file system APIs. +// If non-NULL, this is different than the default encoding for strings. const char *Py_FileSystemDefaultEncoding = NULL; int Py_HasFileSystemDefaultEncoding = 0; const char *Py_FileSystemDefaultEncodeErrors = NULL; @@ -25,8 +33,6 @@ int _Py_HasFileSystemDefaultEncodeErrors = 0; void _Py_ClearFileSystemEncoding(void) { -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); Py_FileSystemDefaultEncoding = NULL; @@ -35,7 +41,6 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors); Py_FileSystemDefaultEncodeErrors = NULL; } -_Py_COMP_DIAG_POP } @@ -60,14 +65,11 @@ _Py_SetFileSystemEncoding(const char *encoding, const char *errors) _Py_ClearFileSystemEncoding(); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS Py_FileSystemDefaultEncoding = encoding2; Py_HasFileSystemDefaultEncoding = 0; Py_FileSystemDefaultEncodeErrors = errors2; _Py_HasFileSystemDefaultEncodeErrors = 0; -_Py_COMP_DIAG_POP return 0; } @@ -470,39 +472,18 @@ preconfig_get_global_var(PyPreConfig *config) return; } -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS if (Py_UTF8Mode > 0) { config->utf8_mode = Py_UTF8Mode; } -_Py_COMP_DIAG_POP } static void -preconfig_set_global_vars(const PyPreConfig *config) +preconfig_set_global_var(const PyPreConfig *config) { -#define COPY_FLAG(ATTR, VAR) \ - if (config->ATTR >= 0) { \ - VAR = config->ATTR; \ - } -#define COPY_NOT_FLAG(ATTR, VAR) \ - if (config->ATTR >= 0) { \ - VAR = !config->ATTR; \ + if (config->utf8_mode >= 0) { + Py_UTF8Mode = config->utf8_mode; } - -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - COPY_FLAG(isolated, Py_IsolatedFlag); - COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); -#ifdef MS_WINDOWS - COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); -#endif - COPY_FLAG(utf8_mode, Py_UTF8Mode); -_Py_COMP_DIAG_POP - -#undef COPY_FLAG -#undef COPY_NOT_FLAG } @@ -935,7 +916,7 @@ _PyPreConfig_Write(const PyPreConfig *src_config) } } - preconfig_set_global_vars(&config); + preconfig_set_global_var(&config); if (config.configure_locale) { if (config.coerce_c_locale) { diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 60ff561776b76dd..2ad801c671855cf 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -778,6 +778,7 @@ Modules/expat/xmlrole.c - condSect1 - Modules/expat/xmlrole.c - condSect2 - Modules/expat/xmlrole.c - declClose - Modules/expat/xmlrole.c - error - +Python/preconfig.c - Py_UTF8Mode - ## other Modules/_io/_iomodule.c - _PyIO_Module - diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index b7aa0e5de7ed99b..5c491e3cfc6211c 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -7,206 +7,8 @@ import argparse from _colorize import get_colors, can_colorize -CODE = '''\ -import time -import os -import sys -import math - -def slow_fibonacci(n): - """Intentionally slow recursive fibonacci - should show up prominently in profiler""" - if n <= 1: - return n - return slow_fibonacci(n-1) + slow_fibonacci(n-2) - -def medium_computation(): - """Medium complexity function""" - result = 0 - for i in range(1000): - result += math.sqrt(i) * math.sin(i) - return result - -def fast_loop(): - """Fast simple loop""" - total = 0 - for i in range(100): - total += i - return total - -def string_operations(): - """String manipulation that should be visible in profiler""" - text = "hello world " * 100 - words = text.split() - return " ".join(reversed(words)) - -def nested_calls(): - """Nested function calls to test call stack depth""" - def level1(): - def level2(): - def level3(): - return medium_computation() - return level3() - return level2() - return level1() - -def main_loop(): - """Main computation loop with different execution paths""" - iteration = 0 - - while True: - iteration += 1 - - # Different execution paths with different frequencies - if iteration % 50 == 0: - # Expensive operation - should show high per-call time - result = slow_fibonacci(20) - - elif iteration % 10 == 0: - # Medium operation - result = nested_calls() - - elif iteration % 5 == 0: - # String operations - result = string_operations() - - else: - # Fast operation - most common - result = fast_loop() - - # Small delay to make sampling more interesting - time.sleep(0.001) - -if __name__ == "__main__": - main_loop() -''' - -DEEP_STATIC_CODE = """\ -import time -def factorial(n): - if n <= 1: - time.sleep(10000) - return 1 - return n * factorial(n-1) - -factorial(900) -""" +from snippets import CODE_EXAMPLES, CODE -CODE_WITH_TONS_OF_THREADS = '''\ -import time -import threading -import random -import math - -def cpu_intensive_work(): - """Do some CPU intensive calculations""" - result = 0 - for _ in range(10000): - result += math.sin(random.random()) * math.cos(random.random()) - return result - -def io_intensive_work(): - """Simulate IO intensive work with sleeps""" - time.sleep(0.1) - -def mixed_workload(): - """Mix of CPU and IO work""" - while True: - if random.random() < 0.3: - cpu_intensive_work() - else: - io_intensive_work() - -def create_threads(n): - """Create n threads doing mixed workloads""" - threads = [] - for _ in range(n): - t = threading.Thread(target=mixed_workload, daemon=True) - t.start() - threads.append(t) - return threads - -# Start with 5 threads -active_threads = create_threads(5) -thread_count = 5 - -# Main thread manages threads and does work -while True: - # Randomly add or remove threads - if random.random() < 0.1: # 10% chance each iteration - if random.random() < 0.5 and thread_count < 100: - # Add 1-5 new threads - new_count = random.randint(1, 5) - new_threads = create_threads(new_count) - active_threads.extend(new_threads) - thread_count += new_count - elif thread_count > 10: - # Remove 1-3 threads - remove_count = random.randint(1, 5) - # The threads will terminate naturally since they're daemons - active_threads = active_threads[remove_count:] - thread_count -= remove_count - - cpu_intensive_work() - time.sleep(0.05) -''' - -ASYNC_CODE = '''\ -import asyncio -import contextlib -import math - -def compute_slice(seed): - result = 0.0 - for i in range(2000): - result += math.sin(seed + i) * math.sqrt(i + 1) - return result - -async def leaf_task(seed): - total = 0.0 - while True: - total += compute_slice(seed) - await asyncio.sleep(0) - -async def parent_task(seed): - child = asyncio.create_task(leaf_task(seed + 1000), name=f"leaf-{seed}") - try: - while True: - compute_slice(seed) - await asyncio.sleep(0.001) - finally: - child.cancel() - with contextlib.suppress(asyncio.CancelledError): - await child - -async def main(): - tasks = [ - asyncio.create_task(parent_task(i), name=f"parent-{i}") - for i in range(8) - ] - await asyncio.gather(*tasks) - -if __name__ == "__main__": - asyncio.run(main()) -''' - -CODE_EXAMPLES = { - "basic": { - "code": CODE, - "description": "Mixed workload with fibonacci, computations, and string operations", - }, - "deep_static": { - "code": DEEP_STATIC_CODE, - "description": "Deep recursive call stack with 900+ frames (factorial)", - }, - "threads": { - "code": CODE_WITH_TONS_OF_THREADS, - "description": "Tons of threads doing mixed CPU/IO work", - }, - "asyncio": { - "code": ASYNC_CODE, - "description": "Asyncio tasks with active and awaited coroutine chains", - }, -} OPERATIONS = { "stack_trace": { diff --git a/Tools/inspection/oracle_external_inspection.py b/Tools/inspection/oracle_external_inspection.py new file mode 100644 index 000000000000000..ec5060a1c9a4a51 --- /dev/null +++ b/Tools/inspection/oracle_external_inspection.py @@ -0,0 +1,436 @@ +"""Compare external inspection modes against a reference mode ("Oracle"). + +This script reports the following validation metrics: + +- impossible: Number of stacks matching a known impossible pattern. The + classifiers are not exhaustive. + +- raw_tvd: Total variation distance between this mode's stack distribution + and the reference's distribution. Stacks classified as impossible are + excluded. + +- tvd_excess: raw_tvd minus tvd_floor. + +- tvd_floor: IID heuristic for the TVD expected from finite sampling of both + distributions. This is not a lower bound. +""" + +import argparse +import contextlib +import math +import os +import random +import subprocess +import statistics +import sys +import tempfile +import time +from collections import Counter + +import _remote_debugging + +from snippets import CASES, _get_lineno + + +TRANSIENT_ERRORS = (OSError, RuntimeError, UnicodeDecodeError) + +MODES = { + "live-cache": (False, True), + "live-nocache": (False, False), + "blocking-cache": (True, True), + "blocking-nocache": (True, False), +} + + +def collapse_cache(mode): + blocking, _ = MODES[mode] + return "blocking-nocache" if blocking else "live-nocache" + + +def tvd(left, right): + lt, rt = sum(left.values()), sum(right.values()) + if not lt or not rt: + return None + return 0.5 * sum( + abs(left[k] / lt - right[k] / rt) for k in set(left) | set(right) + ) + + +def tvd_floor(reference_obs, n_live): + n_ref = sum(reference_obs.values()) + if not n_ref or not n_live: + return None + spread = sum( + math.sqrt(p * (1 - p)) + for p in (c / n_ref for c in reference_obs.values()) + ) + return ( + 0.5 + * math.sqrt(2 / math.pi) + * math.sqrt(1 / n_live + 1 / n_ref) + * spread + ) + + +def print_run_info(args, cases): + print(sys.version.replace("\n", " ")) + print( + f"cases={','.join(cases)} runs={args.runs} " + f"duration={args.duration} " + f"rate_khz={args.rate_khz} warmup={args.warmup} " + f"poisson_sampling={args.poisson_sampling}" + ) + + +def terminate_process(proc): + if proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +@contextlib.contextmanager +def target_process(code, warmup): + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as tmp: + tmp.write(code) + tmp.flush() + tmp_name = tmp.name + proc = None + try: + proc = subprocess.Popen( + [sys.executable, tmp_name], + stdout=subprocess.DEVNULL, + ) + time.sleep(warmup) + if proc.poll() is not None: + raise RuntimeError( + f"target exited unexpectedly with code {proc.returncode}" + ) + yield proc + finally: + with contextlib.suppress(Exception): + if proc is not None: + terminate_process(proc) + with contextlib.suppress(OSError): + os.unlink(tmp_name) + + +def get_trace(unwinder, blocking, op="get_stack_trace"): + call = getattr(unwinder, op) + if not blocking: + return call() + unwinder.pause_threads() + try: + return call() + finally: + unwinder.resume_threads() + + +def iter_units(raw, op): + if op == "get_stack_trace": + for interp in raw: + for thread in interp.threads: + yield ( + thread.thread_id, + None, + thread.status, + thread.frame_info, + ) + else: + for awaited_info in raw: + for task in awaited_info.awaited_by: + frames = [ + frame + for coro in task.coroutine_stack + for frame in coro.call_stack + ] + if frames: + yield (task.task_id, task.task_name, None, frames) + + +def run_mode(case, mode_name, args): + code, classify, *rest = case + op = rest[0] if rest else "get_stack_trace" + classify_units = op != "get_stack_trace" + blocking, cache_frames = MODES[mode_name] + result = { + "attempts": 0, + "samples": 0, + "stacks": 0, + "errors": 0, + "observations": Counter(), + "impossible": 0, + "work_time": 0.0, + } + with target_process(code, args.warmup) as proc: + unwinder = _remote_debugging.RemoteUnwinder( + proc.pid, + all_threads=True, + cache_frames=cache_frames, + ) + rate_hz = args.rate_khz * 1000 + period = 1.0 / rate_hz if rate_hz else 0 + next_sample = time.perf_counter() + deadline = next_sample + args.duration + + while time.perf_counter() < deadline: + if period: + if args.poisson_sampling: + next_sample += random.expovariate(rate_hz) + if next_sample >= deadline: + break + now = time.perf_counter() + if next_sample > now: + time.sleep(next_sample - now) + if not args.poisson_sampling: + next_sample += period + + if time.perf_counter() >= deadline: + break + + result["attempts"] += 1 + work_start = time.perf_counter() + try: + trace = get_trace(unwinder, blocking, op) + except TRANSIENT_ERRORS: + trace = None + result["errors"] += 1 + result["work_time"] += time.perf_counter() - work_start + + if not trace: + continue + + result["samples"] += 1 + for unit in iter_units(trace, op): + frames = unit[3] + impossible = classify is not None and ( + classify(unit) if classify_units else classify(frames) + ) + result["stacks"] += 1 + if impossible: + result["impossible"] += 1 + else: + result["observations"][ + ";".join( + f"{frame.funcname}:{_get_lineno(frame)}" + for frame in frames + ) + ] += 1 + return result + + +def result_metrics(result, reference_obs, is_reference, op): + skip_tvd = is_reference or op != "get_stack_trace" + n_live = sum(result["observations"].values()) + raw_tvd = None if skip_tvd else tvd(result["observations"], reference_obs) + # IID heuristic, not a calibrated noise bound. + floor = None if skip_tvd else tvd_floor(reference_obs, n_live) + return { + "samples": result["samples"], + "stacks": result["stacks"], + "empty": result["attempts"] - result["samples"] - result["errors"], + "impossible": result["impossible"], + "error_percent": ( + 100.0 * result["errors"] / result["attempts"] + if result["attempts"] + else 0.0 + ), + "impossible_percent": ( + 100.0 * result["impossible"] / result["stacks"] + if result["stacks"] + else 0.0 + ), + "avg_us": ( + 1e6 * result["work_time"] / result["attempts"] + if result["attempts"] + else 0.0 + ), + "raw_tvd": raw_tvd, + "tvd_floor": floor, + "tvd_excess": ( + None if (raw_tvd is None or floor is None) else raw_tvd - floor + ), + } + + +def fmt_stat(values, precision): + vals = [value for value in values if value is not None] + if not vals: + return "n/a" + mean = statistics.mean(vals) + if len(vals) > 1: + return f"{mean:.{precision}f}ยฑ{statistics.stdev(vals):.{precision}f}" + return f"{mean:.{precision}f}" + + +def fmt_floor(values): + vals = [value for value in values if value is not None] + return "n/a" if not vals else f"{statistics.median(vals):.3f}" + + +def print_results( + case_name, run_results, modes, reference_mode, op, has_classify +): + is_sync = op == "get_stack_trace" + ref_keys = len(run_results[0][reference_mode]["observations"]) + rows = {} + for mode in modes: + metrics = [ + result_metrics( + results[mode], + results[reference_mode]["observations"], + mode == reference_mode, + op, + ) + for results in run_results + ] + rows[mode] = { + "mode": mode, + "samples": sum(item["samples"] for item in metrics), + "stacks": sum(item["stacks"] for item in metrics), + "empty": sum(item["empty"] for item in metrics), + "impossible": sum(item["impossible"] for item in metrics), + "errors": fmt_stat([item["error_percent"] for item in metrics], 2), + "us": fmt_stat([item["avg_us"] for item in metrics], 2), + "impossible_pct": fmt_stat( + [item["impossible_percent"] for item in metrics], 2 + ), + "raw_tvd": "ref" + if mode == reference_mode + else fmt_stat([item["raw_tvd"] for item in metrics], 3), + "tvd_excess": "ref" + if mode == reference_mode + else fmt_stat([item["tvd_excess"] for item in metrics], 3), + "tvd_floor": "-" + if mode == reference_mode + else fmt_floor([item["tvd_floor"] for item in metrics]), + } + + show_stacks = any(row["stacks"] != row["samples"] for row in rows.values()) + ordered = [reference_mode] + [m for m in modes if m != reference_mode] + + columns = [("mode", "<18", "mode"), ("samples", ">9", "samples")] + if show_stacks: + columns.append(("stacks", ">9", "stacks")) + if not is_sync: + columns.append(("empty", ">9", "empty")) + columns += [("ยตs", ">12", "us"), ("errors%", ">12", "errors")] + if has_classify: + columns += [ + ("impossible", ">10", "impossible"), + ("impossible%", ">12", "impossible_pct"), + ] + if is_sync: + columns += [ + ("raw_tvd", ">12", "raw_tvd"), + ("tvd_excess", ">12", "tvd_excess"), + ("tvd_floor", ">10", "tvd_floor"), + ] + + print(f"\n{case_name} ({op}) ref_keys={ref_keys}") + print(" ".join(f"{label:{spec}}" for label, spec, _ in columns)) + for mode in ordered: + row = rows[mode] + print(" ".join(f"{row[key]:{spec}}" for _, spec, key in columns)) + + +def parse_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--snippet", + action="append", + help="snippet name or Python file; may be passed more than once", + ) + parser.add_argument( + "--mode", + choices=sorted(MODES), + action="append", + help="mode to run; may be passed more than once; omit to run all modes", + ) + parser.add_argument( + "--reference-mode", + choices=sorted(MODES), + default="blocking-nocache", + help="mode used as the distribution reference", + ) + parser.add_argument( + "--duration", + type=float, + default=3.0, + help="seconds to sample each mode", + ) + parser.add_argument( + "--runs", + type=int, + default=1, + help="number of independent runs per case", + ) + parser.add_argument( + "--rate-khz", + type=float, + default=100.0, + help="target sampling rate in kHz; 0 samples as fast as possible", + ) + parser.add_argument( + "--warmup", + type=float, + default=0.7, + help="seconds to let the target run before sampling", + ) + parser.add_argument( + "--poisson-sampling", + action="store_true", + help=( + "sample with exponential inter-arrival times instead of a fixed " + "period" + ), + ) + args = parser.parse_args() + if args.runs < 1: + parser.error("--runs must be greater than zero") + return args + + +def main(): + args = parse_args() + cases = list(args.snippet) if args.snippet else sorted(CASES) + modes = list(args.mode) if args.mode else sorted(MODES) + if args.reference_mode not in modes: + modes.append(args.reference_mode) + + print_run_info(args, cases) + + for name in cases: + if name in CASES: + case = CASES[name] + else: + with open(name, encoding="utf-8") as file: + case = (file.read(), None) + op = case[2] if len(case) > 2 else "get_stack_trace" + case_ref = args.reference_mode + case_modes = list(modes) + if op != "get_stack_trace": + case_ref = collapse_cache(case_ref) + case_modes = list( + dict.fromkeys(collapse_cache(m) for m in case_modes) + ) + if case_ref not in case_modes: + case_modes.append(case_ref) + run_results = [ + {mode: run_mode(case, mode, args) for mode in case_modes} + for _ in range(args.runs) + ] + print_results( + name, run_results, case_modes, case_ref, op, case[1] is not None + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Tools/inspection/snippets.py b/Tools/inspection/snippets.py new file mode 100644 index 000000000000000..47173c4fd125438 --- /dev/null +++ b/Tools/inspection/snippets.py @@ -0,0 +1,601 @@ +"""Scripts and classifiers for external inspection validation. + +Classifiers detect only recognized impossible patterns. They do not validate +entire stacks. False does not imply that the stack as a whole is valid. +""" +import os + + +def _get_lineno(frame, default=None): + if frame is None: + return default + loc = getattr(frame, "location", None) + return getattr(loc, "lineno", default) if loc is not None else default + + +CODE = '''\ +import time +import os +import sys +import math + +def slow_fibonacci(n): + """Intentionally slow recursive fibonacci - should show up prominently in profiler""" + if n <= 1: + return n + return slow_fibonacci(n-1) + slow_fibonacci(n-2) + +def medium_computation(): + """Medium complexity function""" + result = 0 + for i in range(1000): + result += math.sqrt(i) * math.sin(i) + return result + +def fast_loop(): + """Fast simple loop""" + total = 0 + for i in range(100): + total += i + return total + +def string_operations(): + """String manipulation that should be visible in profiler""" + text = "hello world " * 100 + words = text.split() + return " ".join(reversed(words)) + +def nested_calls(): + """Nested function calls to test call stack depth""" + def level1(): + def level2(): + def level3(): + return medium_computation() + return level3() + return level2() + return level1() + +def main_loop(): + """Main computation loop with different execution paths""" + iteration = 0 + + while True: + iteration += 1 + + # Different execution paths with different frequencies + if iteration % 50 == 0: + # Expensive operation - should show high per-call time + result = slow_fibonacci(20) + + elif iteration % 10 == 0: + # Medium operation + result = nested_calls() + + elif iteration % 5 == 0: + # String operations + result = string_operations() + + else: + # Fast operation - most common + result = fast_loop() + + # Small delay to make sampling more interesting + time.sleep(0.001) + +if __name__ == "__main__": + main_loop() +''' + +DEEP_STATIC_CODE = """\ +import time +def factorial(n): + if n <= 1: + time.sleep(10000) + return 1 + return n * factorial(n-1) + +factorial(900) +""" + +CODE_WITH_TONS_OF_THREADS = '''\ +import time +import threading +import random +import math + +def cpu_intensive_work(): + """Do some CPU intensive calculations""" + result = 0 + for _ in range(10000): + result += math.sin(random.random()) * math.cos(random.random()) + return result + +def io_intensive_work(): + """Simulate IO intensive work with sleeps""" + time.sleep(0.1) + +def mixed_workload(): + """Mix of CPU and IO work""" + while True: + if random.random() < 0.3: + cpu_intensive_work() + else: + io_intensive_work() + +def create_threads(n): + """Create n threads doing mixed workloads""" + threads = [] + for _ in range(n): + t = threading.Thread(target=mixed_workload, daemon=True) + t.start() + threads.append(t) + return threads + +# Start with 5 threads +active_threads = create_threads(5) + +# Main thread manages threads and does work +while True: + # Randomly add threads up to the limit + if random.random() < 0.1: # 10% chance each iteration + if random.random() < 0.5 and len(active_threads) < 100: + new_count = min( + random.randint(1, 5), + 100 - len(active_threads), + ) + new_threads = create_threads(new_count) + active_threads.extend(new_threads) + + cpu_intensive_work() + time.sleep(0.05) +''' + +ASYNC_CODE = '''\ +import asyncio +import contextlib +import math + +def compute_slice(seed): + result = 0.0 + for i in range(2000): + result += math.sin(seed + i) * math.sqrt(i + 1) + return result + +async def leaf_task(seed): + total = 0.0 + while True: + total += compute_slice(seed) + await asyncio.sleep(0) + +async def parent_task(seed): + child = asyncio.create_task(leaf_task(seed + 1000), name=f"leaf-{seed}") + try: + while True: + compute_slice(seed) + await asyncio.sleep(0.001) + finally: + child.cancel() + with contextlib.suppress(asyncio.CancelledError): + await child + +async def main(): + tasks = [ + asyncio.create_task(parent_task(i), name=f"parent-{i}") + for i in range(8) + ] + await asyncio.gather(*tasks) + +if __name__ == "__main__": + asyncio.run(main()) +''' + + +FLAT_ALTERNATING_CODE = """\ +def leaf_a(): return sum(range(50)) +def leaf_b(): return sum(range(50)) +def hot_a(): return leaf_a() +def hot_b(): return leaf_b() +while True: + hot_a(); hot_b() +""" + + +def _expected_lines(code): + expected = {} + for number, line in enumerate(code.splitlines(), 1): + stripped = line.strip() + if stripped.startswith("def "): + expected[stripped[4:].split("(")[0].strip()] = number + return expected + + +FLAT_ALTERNATING_LINES = _expected_lines(FLAT_ALTERNATING_CODE) + + +def classify_flat(frames): + present = {} + for frame in frames: + if frame.funcname in FLAT_ALTERNATING_LINES: + present[frame.funcname] = _get_lineno(frame, -1) + if not present: + return False + for name, lineno in present.items(): + if lineno != FLAT_ALTERNATING_LINES[name]: + return True + hot_a, hot_b = "hot_a" in present, "hot_b" in present + leaf_a, leaf_b = "leaf_a" in present, "leaf_b" in present + any_a = leaf_a or hot_a + any_b = leaf_b or hot_b + return (any_a and any_b) or (leaf_a and not hot_a) or (leaf_b and not hot_b) + + +NESTED_ALTERNATING_CODE = """\ +def burn_a(): + total = 0 + for i in range(20000): + total += i + return total + +def burn_b(): + total = 0 + for i in range(20000): + total += i + return total + +def a_leaf(): + return burn_a() + +def b_leaf(): + return burn_b() + +def a_parent(): + return a_leaf() + +def b_parent(): + return b_leaf() + +while True: + a_parent() + b_parent() +""" + + +NESTED_ALTERNATING_BRANCHES = { + "a": ["a_parent", "a_leaf", "burn_a"], + "b": ["b_parent", "b_leaf", "burn_b"], +} + + +def classify_nested(frames): + frame_names = [frame.funcname for frame in frames] + names = set(frame_names) + present = [ + family + for family, chain in NESTED_ALTERNATING_BRANCHES.items() + if names.intersection(chain) + ] + if len(present) > 1: + return True + if not present: + return False + chain = NESTED_ALTERNATING_BRANCHES[present[0]] + active = [name for name in chain if name in names] + depth = chain.index(active[-1]) + if len(active) != depth + 1: + return True + indices = [frame_names.index(name) for name in reversed(active)] + return indices != sorted(indices) + + +SHARED_LEAF_CODE = """\ +def shared_leaf(long_run): + total = 0 + if long_run: + for i in range(50000): + total += i + else: + for i in range(200): + total += i + return total + +def a_wrapper(): + return shared_leaf(False) + +def b_wrapper(): + return shared_leaf(True) + +while True: + a_wrapper() + b_wrapper() +""" + + +def _branch_lines(code, marker): + for number, line in enumerate(code.splitlines(), 1): + if marker in line: + return {number, number + 1} + return set() + + +SHARED_LEAF_LONG_LINES = _branch_lines(SHARED_LEAF_CODE, "range(50000)") +SHARED_LEAF_SHORT_LINES = _branch_lines(SHARED_LEAF_CODE, "range(200)") + + +def classify_shared(frames): + frame_names = [frame.funcname for frame in frames] + names = set(frame_names) + if "a_wrapper" in names and "b_wrapper" in names: + return True + if "shared_leaf" not in names: + return False + index = frame_names.index("shared_leaf") + parent = frame_names[index + 1] if index + 1 < len(frame_names) else None + if parent not in ("a_wrapper", "b_wrapper"): + return True + lineno = _get_lineno(frames[index], -1) + if lineno in SHARED_LEAF_LONG_LINES: + return parent != "b_wrapper" + if lineno in SHARED_LEAF_SHORT_LINES: + return parent != "a_wrapper" + return False + + +GEN_ALTERNATING_CODE = """\ +def agen(n): + total = 0 + for i in range(n): + total += i + yield i + +def bgen(n): + total = 0 + for i in range(n): + total += i + yield i + +def drv_a(): + for _ in agen(60): + pass + +def drv_b(): + for _ in bgen(60): + pass + +while True: + drv_a() + drv_b() +""" + + +def classify_gen(frames): + names = {frame.funcname for frame in frames} + return ("agen" in names and "drv_b" in names) or ( + "bgen" in names and "drv_a" in names + ) + + +DEEP_RECURSION_CODE = """\ +def leaf(): + total = 0 + for i in range(40): + total += i + +def a(n): + return a(n - 1) if n else leaf() + +def b(n): + return b(n - 1) if n else leaf() + +while True: + a(300) + b(300) +""" + + +def classify_recursion(frames): + names = {frame.funcname for frame in frames} + return "a" in names and "b" in names + + +ASYNC_RUNNING_TASK_CODE = """\ +import asyncio + + +def leaf_hot(n): + return sum(range(n)) + + +def leaf_rare(n): + return sum(range(n)) + + +async def run_hot(): + while True: + leaf_hot(50000) + await asyncio.sleep(0) + + +async def run_rare(k): + while True: + leaf_rare(500) + await asyncio.sleep(0) + + +async def main(): + tasks = [asyncio.create_task(run_hot(), name="hot")] + for k in range(8): + tasks.append(asyncio.create_task(run_rare(k), name=f"rare{k}")) + await asyncio.gather(*tasks) + + +asyncio.run(main()) +""" + + +def _name_tag(label): + label = (label or "").lower() + return "hot" if "hot" in label else "rare" if "rare" in label else None + + +def _frame_tag(frames): + fns = {frame.funcname for frame in frames} + hot = bool(fns & {"run_hot", "leaf_hot"}) + rare = bool(fns & {"run_rare", "leaf_rare"}) + return ( + "mixed" + if (hot and rare) + else "hot" + if hot + else "rare" + if rare + else None + ) + + +def classify_async_running_task(unit): + name = _name_tag(unit[1]) + frame = _frame_tag(unit[3]) + return name is not None and frame is not None and frame != name + + +CODE_OBJECT_REUSE_CODE = """\ +SRC_A = "def func_a(n):\\n total=0\\n for i in range(n): total+=i*i\\n return total\\n" +SRC_B = "def func_b(n):\\n total=0\\n for i in range(n): total+=i*i\\n return total\\n" +WORK = 60000 + + +def build_a(): + ns = {} + code = compile(SRC_A, "A_file.py", "exec") + exec(code, ns) + return ns["func_a"], code + + +def build_b(): + ns = {} + code = compile(SRC_B, "B_file.py", "exec") + exec(code, ns) + return ns["func_b"], code + + +while True: + fa, ca = build_a() + fa(WORK) # call_a + del fa, ca + fb, cb = build_b() + fb(WORK) # call_b + del fb, cb +""" + + +def _marker_line(code, marker): + for number, line in enumerate(code.splitlines(), 1): + if marker in line: + return number + return None + + +CALL_A_LINE = _marker_line(CODE_OBJECT_REUSE_CODE, "# call_a") +CALL_B_LINE = _marker_line(CODE_OBJECT_REUSE_CODE, "# call_b") + + +def classify_code_object_reuse(frames): + real = [f for f in frames if f.funcname != ""] + leaf = next((f for f in real if f.funcname in ("func_a", "func_b")), None) + if leaf is None: + return False + base = os.path.basename(leaf.filename) + fn = leaf.funcname + if (fn == "func_a" and base == "B_file.py") or ( + fn == "func_b" and base == "A_file.py" + ): + return True + index = real.index(leaf) + caller = real[index + 1] if index + 1 < len(real) else None + line = _get_lineno(caller) + if line == CALL_A_LINE and fn == "func_b": + return True + if line == CALL_B_LINE and fn == "func_a": + return True + return False + + +OVERSIZED_CHUNK_CODE = """\ +NLOCALS = 1800 + +def make(name, tag, hotbody): + params = ", ".join(f"x{i}=0" for i in range(NLOCALS)) + src = ( + f"def hot_{tag}():\\n{hotbody}\\n" + f"def {name}({params}):\\n return hot_{tag}()\\n" + ) + exec(compile(src, f"{tag}.py", "exec"), globals()) + +make("big_a", "a", " s=0\\n for i in range(2000):\\n s+=i*3\\n return s") +make("big_b", "b", " s=1\\n for i in range(2000):\\n s^=(i<<1)\\n return s") + +while True: + big_a() + big_b() +""" + + +OVERSIZED_A_FUNCS = {"big_a", "hot_a"} +OVERSIZED_B_FUNCS = {"big_b", "hot_b"} + + +def classify_oversized_chunk(frames): + saw_a = saw_b = False + for frame in frames: + base = os.path.basename(frame.filename) + fn = frame.funcname + if base == "a.py": + if fn in OVERSIZED_A_FUNCS: + saw_a = True + elif fn in OVERSIZED_B_FUNCS: + return True + elif base == "b.py": + if fn in OVERSIZED_B_FUNCS: + saw_b = True + elif fn in OVERSIZED_A_FUNCS: + return True + return saw_a and saw_b + + +CODE_EXAMPLES = { + "basic": { + "code": CODE, + "description": "Mixed workload with fibonacci, computations, and string operations", + }, + "deep_static": { + "code": DEEP_STATIC_CODE, + "description": "Deep recursive call stack with 900+ frames (factorial)", + }, + "threads": { + "code": CODE_WITH_TONS_OF_THREADS, + "description": "Tons of threads doing mixed CPU/IO work", + }, + "asyncio": { + "code": ASYNC_CODE, + "description": "Asyncio tasks with active and awaited coroutine chains", + }, +} + +CASES = { + "basic": (CODE, None), + "deep_static": (DEEP_STATIC_CODE, None), + "threads": (CODE_WITH_TONS_OF_THREADS, None), + "asyncio": (ASYNC_CODE, None, "get_async_stack_trace"), + "flat_alternating": (FLAT_ALTERNATING_CODE, classify_flat), + "nested_alternating": (NESTED_ALTERNATING_CODE, classify_nested), + "shared_leaf": (SHARED_LEAF_CODE, classify_shared), + "gen_alternating": (GEN_ALTERNATING_CODE, classify_gen), + "deep_recursion": (DEEP_RECURSION_CODE, classify_recursion), + "async_running_task": ( + ASYNC_RUNNING_TASK_CODE, + classify_async_running_task, + "get_async_stack_trace", + ), + "code_object_reuse": (CODE_OBJECT_REUSE_CODE, classify_code_object_reuse), + "oversized_chunk": (OVERSIZED_CHUNK_CODE, classify_oversized_chunk), +} diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 1dc33520e5387d9..ce079adafcf4c7c 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -125,6 +125,7 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"), str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"), diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 044366c3aac1405..b2978d3a0baa756 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -1,32 +1,23 @@ -import ast +"""Prepare a complete C parser description and emit it.""" + import os.path -import re -from collections.abc import Callable -from dataclasses import dataclass, field -from enum import Enum -from typing import IO, Any +from typing import IO from pegen import grammar -from pegen.grammar import ( - Alt, - Cut, - Forced, - Gather, - GrammarVisitor, - Group, - Leaf, - Lookahead, - NamedItem, - NameLeaf, - NegativeLookahead, - Opt, - PositiveLookahead, - Repeat0, - Repeat1, - Rhs, - Rule, - StringLeaf, +from pegen.c_generator_calls import ( + CCallMakerVisitor as CCallMakerVisitor, +) +from pegen.c_generator_calls import ( + FunctionCall as FunctionCall, +) +from pegen.c_generator_calls import ( + NodeTypes as NodeTypes, +) +from pegen.c_generator_calls import ( + consuming_rules as consuming_rules, ) +from pegen.c_generator_file import CParserEmitter +from pegen.c_generator_model import CParser, CRule, CRuleSignature from pegen.parser_generator import ParserGenerator EXTENSION_PREFIX = """\ @@ -66,345 +57,7 @@ """ -class NodeTypes(Enum): - NAME_TOKEN = 0 - NUMBER_TOKEN = 1 - STRING_TOKEN = 2 - GENERIC_TOKEN = 3 - KEYWORD = 4 - SOFT_KEYWORD = 5 - CUT_OPERATOR = 6 - F_STRING_CHUNK = 7 - - -BASE_NODETYPES = { - "NAME": NodeTypes.NAME_TOKEN, - "NUMBER": NodeTypes.NUMBER_TOKEN, - "STRING": NodeTypes.STRING_TOKEN, - "SOFT_KEYWORD": NodeTypes.SOFT_KEYWORD, -} - - -@dataclass -class FunctionCall: - function: str - arguments: list[Any] = field(default_factory=list) - assigned_variable: str | None = None - assigned_variable_type: str | None = None - return_type: str | None = None - nodetype: NodeTypes | None = None - force_true: bool = False - comment: str | None = None - - def __str__(self) -> str: - parts = [] - parts.append(self.function) - if self.arguments: - parts.append(f"({', '.join(map(str, self.arguments))})") - if self.force_true: - parts.append(", !p->error_indicator") - if self.assigned_variable: - if self.assigned_variable_type: - parts = [ - "(", - self.assigned_variable, - " = ", - "(", - self.assigned_variable_type, - ")", - *parts, - ")", - ] - else: - parts = ["(", self.assigned_variable, " = ", *parts, ")"] - if self.comment: - parts.append(f" // {self.comment}") - return "".join(parts) - - -class CCallMakerVisitor(GrammarVisitor): - def __init__( - self, - parser_generator: ParserGenerator, - exact_tokens: dict[str, int], - non_exact_tokens: set[str], - ): - self.gen = parser_generator - self.exact_tokens = exact_tokens - self.non_exact_tokens = non_exact_tokens - self.cache: dict[str, str] = {} - self.cleanup_statements: list[str] = [] - - def keyword_helper(self, keyword: str) -> FunctionCall: - return FunctionCall( - assigned_variable="_keyword", - function="_PyPegen_expect_token", - arguments=["p", self.gen.keywords[keyword]], - return_type="Token *", - nodetype=NodeTypes.KEYWORD, - comment=f"token='{keyword}'", - ) - - def soft_keyword_helper(self, value: str) -> FunctionCall: - return FunctionCall( - assigned_variable="_keyword", - function="_PyPegen_expect_soft_keyword", - arguments=["p", value], - return_type="expr_ty", - nodetype=NodeTypes.SOFT_KEYWORD, - comment=f"soft_keyword='{value}'", - ) - - def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall: - name = node.value - if name in self.non_exact_tokens: - if name in BASE_NODETYPES: - return FunctionCall( - assigned_variable=f"{name.lower()}_var", - function=f"_PyPegen_{name.lower()}_token", - arguments=["p"], - nodetype=BASE_NODETYPES[name], - return_type="expr_ty", - comment=name, - ) - return FunctionCall( - assigned_variable=f"{name.lower()}_var", - function="_PyPegen_expect_token", - arguments=["p", name], - nodetype=NodeTypes.GENERIC_TOKEN, - return_type="Token *", - comment=f"token='{name}'", - ) - - type = None - rule = self.gen.all_rules.get(name.lower()) - if rule is not None: - type = "asdl_seq *" if rule.is_loop() or rule.is_gather() else rule.type - - return FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type=type, - comment=f"{node}", - ) - - def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: - val = ast.literal_eval(node.value) - if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword - if node.value.endswith("'"): - return self.keyword_helper(val) - else: - return self.soft_keyword_helper(node.value) - else: - assert val in self.exact_tokens, f"{node.value} is not a known literal" - type = self.exact_tokens[val] - return FunctionCall( - assigned_variable="_literal", - function="_PyPegen_expect_token", - arguments=["p", type], - nodetype=NodeTypes.GENERIC_TOKEN, - return_type="Token *", - comment=f"token='{val}'", - ) - - def visit_NamedItem(self, node: NamedItem) -> FunctionCall: - call = self.generate_call(node.item) - if node.name: - call.assigned_variable = node.name - if node.type: - call.assigned_variable_type = node.type - return call - - def assert_no_undefined_behavior( - self, call: FunctionCall, wrapper: str, expected_rtype: str | None, - ) -> None: - if call.return_type != expected_rtype: - raise RuntimeError( - f"{call.function} return type is incompatible with {wrapper}: " - f"expect: {expected_rtype}, actual: {call.return_type}" - ) - - def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: - call = self.generate_call(node.node) - comment = None - if call.nodetype is NodeTypes.NAME_TOKEN: - function = "_PyPegen_lookahead_for_expr" - self.assert_no_undefined_behavior(call, function, "expr_ty") - elif call.nodetype is NodeTypes.STRING_TOKEN: - # _PyPegen_string_token() returns 'void *' instead of 'Token *'; - # in addition, the overall function call would return 'expr_ty'. - assert call.function == "_PyPegen_string_token" - function = "_PyPegen_lookahead" - self.assert_no_undefined_behavior(call, function, "expr_ty") - elif call.nodetype == NodeTypes.SOFT_KEYWORD: - function = "_PyPegen_lookahead_with_string" - self.assert_no_undefined_behavior(call, function, "expr_ty") - elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}: - function = "_PyPegen_lookahead_with_int" - self.assert_no_undefined_behavior(call, function, "Token *") - comment = f"token={node.node}" - elif call.return_type == "expr_ty": - function = "_PyPegen_lookahead_for_expr" - elif call.return_type == "stmt_ty": - function = "_PyPegen_lookahead_for_stmt" - else: - function = "_PyPegen_lookahead" - self.assert_no_undefined_behavior(call, function, None) - return FunctionCall( - function=function, - arguments=[positive, call.function, *call.arguments], - return_type="int", - comment=comment, - ) - - def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall: - return self.lookahead_call_helper(node, 1) - - def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall: - return self.lookahead_call_helper(node, 0) - - def visit_Forced(self, node: Forced) -> FunctionCall: - call = self.generate_call(node.node) - if isinstance(node.node, Leaf): - assert isinstance(node.node, Leaf) - val = ast.literal_eval(node.node.value) - assert val in self.exact_tokens, f"{node.node.value} is not a known literal" - type = self.exact_tokens[val] - return FunctionCall( - assigned_variable="_literal", - function="_PyPegen_expect_forced_token", - arguments=["p", type, f'"{val}"'], - nodetype=NodeTypes.GENERIC_TOKEN, - return_type="Token *", - comment=f"forced_token='{val}'", - ) - if isinstance(node.node, Group): - call = self.visit(node.node.rhs) - call.assigned_variable = None - call.comment = None - return FunctionCall( - assigned_variable="_literal", - function="_PyPegen_expect_forced_result", - arguments=["p", str(call), f'"{node.node.rhs!s}"'], - return_type="void *", - comment=f"forced_token=({node.node.rhs!s})", - ) - else: - raise NotImplementedError(f"Forced tokens don't work with {node.node} nodes") - - def visit_Opt(self, node: Opt) -> FunctionCall: - call = self.generate_call(node.node) - return FunctionCall( - assigned_variable="_opt_var", - function=call.function, - arguments=call.arguments, - force_true=True, - comment=f"{node}", - ) - - def _generate_artificial_rule_call( - self, - node: Any, - prefix: str, - rule_generation_func: Callable[[], str], - return_type: str | None = None, - ) -> FunctionCall: - node_str = f"{node}" - key = f"{prefix}_{node_str}" - if key in self.cache: - name = self.cache[key] - else: - name = rule_generation_func() - self.cache[key] = name - - return FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type=return_type, - comment=node_str, - ) - - def visit_Rhs(self, node: Rhs) -> FunctionCall: - if node.can_be_inlined: - return self.generate_call(node.alts[0].items[0]) - - return self._generate_artificial_rule_call( - node, - "rhs", - lambda: self.gen.artificial_rule_from_rhs(node), - ) - - def visit_Repeat0(self, node: Repeat0) -> FunctionCall: - return self._generate_artificial_rule_call( - node, - "repeat0", - lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), - "asdl_seq *", - ) - - def visit_Repeat1(self, node: Repeat1) -> FunctionCall: - return self._generate_artificial_rule_call( - node, - "repeat1", - lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), - "asdl_seq *", - ) - - def visit_Gather(self, node: Gather) -> FunctionCall: - return self._generate_artificial_rule_call( - node, - "gather", - lambda: self.gen.artificial_rule_from_gather(node), - "asdl_seq *", - ) - - def visit_Group(self, node: Group) -> FunctionCall: - return self.generate_call(node.rhs) - - def visit_Cut(self, node: Cut) -> FunctionCall: - return FunctionCall( - assigned_variable="_cut_var", - return_type="int", - function="1", - nodetype=NodeTypes.CUT_OPERATOR, - ) - - def generate_call(self, node: Any) -> FunctionCall: - return super().visit(node) - - -def consuming_rules(rules: dict[str, Rule]) -> set[str]: - """Conservatively prove which rules consume a token whenever they succeed.""" - consuming: set[str] = set() - - def consumes(node: Any) -> bool: - if isinstance(node, NamedItem): - return consumes(node.item) - if isinstance(node, NameLeaf): - return node.value not in rules or node.value in consuming - if isinstance(node, StringLeaf): - return True - if isinstance(node, Group): - return consumes(node.rhs) - if isinstance(node, Rhs): - return bool(node.alts) and all(any(consumes(i) for i in alt.items) for alt in node.alts) - if isinstance(node, (Forced, Repeat1, Gather)): - return consumes(node.node) - # Predicates, cuts, optional items, and zero-or-more items can succeed - # without consuming. Actions are assumed not to rewrite parser marks. - return False - - while True: - added = {name for name, rule in rules.items() - if name not in consuming and consumes(rule.rhs)} - if not added: - return consuming - consuming.update(added) - - -class CParserGenerator(ParserGenerator, GrammarVisitor): +class CParserGenerator(ParserGenerator): def __init__( self, grammar: grammar.Grammar, @@ -419,551 +72,63 @@ def __init__( self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor( self, exact_tokens, non_exact_tokens ) - self._varname_counter = 0 + self._collected = False self.debug = debug self.skip_actions = skip_actions - self.cleanup_statements: list[str] = [] - self.consuming = consuming_rules(self.rules) - self.prefix_calls: dict[int, tuple[str, str, str]] = {} - - def add_level(self) -> None: - self.print("if (p->level++ == MAXSTACK || _PyPegen_stack_exhausted(p)) {") - with self.indent(): - self.print("_Pypegen_stack_overflow(p);") - self.print("}") - - def remove_level(self) -> None: - self.print("p->level--;") - - def add_return(self, ret_val: str) -> None: - for stmt in self.cleanup_statements: - self.print(stmt) - self.remove_level() - self.print(f"return {ret_val};") - - def unique_varname(self, name: str = "tmpvar") -> str: - new_var = name + "_" + str(self._varname_counter) - self._varname_counter += 1 - return new_var - - def call_with_errorcheck_return(self, call_text: str, returnval: str) -> None: - error_var = self.unique_varname() - self.print(f"int {error_var} = {call_text};") - self.print(f"if ({error_var}) {{") - with self.indent(): - self.add_return(returnval) - self.print("}") - - def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None: - error_var = self.unique_varname() - self.print(f"int {error_var} = {call_text};") - self.print(f"if ({error_var}) {{") - with self.indent(): - self.print(f"goto {goto_target};") - self.print("}") - - def out_of_memory_return( - self, - expr: str, - cleanup_code: str | None = None, - ) -> None: - self.print(f"if ({expr}) {{") - with self.indent(): - if cleanup_code is not None: - self.print(cleanup_code) - self.print("p->error_indicator = 1;") - self.print("PyErr_NoMemory();") - self.add_return("NULL") - self.print("}") - - def out_of_memory_goto(self, expr: str, goto_target: str) -> None: - self.print(f"if ({expr}) {{") - with self.indent(): - self.print("PyErr_NoMemory();") - self.print(f"goto {goto_target};") - self.print("}") def generate(self, filename: str) -> None: + parser = self.prepare(filename) + CParserEmitter(parser, self.file).emit() + + def prepare(self, filename: str) -> CParser: self.collect_rules() - basename = os.path.basename(filename) - self.print(f"// @generated by pegen from {basename}") - header = self.grammar.metas.get("header", EXTENSION_PREFIX) - if header: - self.print(header.rstrip("\n")) - subheader = self.grammar.metas.get("subheader", "") - if subheader: - self.print(subheader) - self._setup_keywords() - self._setup_soft_keywords() - for i, (rulename, rule) in enumerate(self.all_rules.items(), 1000): - comment = " // Left-recursive" if rule.left_recursive else "" - self.print(f"#define {rulename}_type {i}{comment}") - self.print() - for rulename, rule in self.all_rules.items(): - if rule.is_loop() or rule.is_gather(): - type = "asdl_seq *" - elif rule.type: - type = rule.type + " " - else: - type = "void *" - self.print(f"static {type}{rulename}_rule(Parser *p);") - self.print() - for rulename, rule in list(self.all_rules.items()): - self.print() - if rule.left_recursive: - self.print("// Left-recursive") - self.visit(rule) + lowerer = self.callmakervisitor.make_lowerer() + rules = tuple( + lowerer.prepare_rule(rule, skip_actions=self.skip_actions) + for rule in self.all_rules.values() + ) + headers = [] + if header := self.grammar.metas.get("header", EXTENSION_PREFIX): + headers.append(header.rstrip("\n")) + if subheader := self.grammar.metas.get("subheader", ""): + headers.append(subheader) + return CParser( + source_name=os.path.basename(filename), + headers=tuple(headers), + keyword_groups=self._prepare_keywords(), + soft_keywords=tuple(sorted(self.soft_keywords)), + rules=rules, + trailer=self._prepare_trailer(rules), + debug=self.debug, + ) + + def collect_rules(self) -> None: + # Keyword generation also uses this entry point without emitting C. + if not self._collected: + super().collect_rules() + self._collected = True + + def _prepare_keywords(self) -> tuple[tuple[tuple[str, int], ...], ...]: + if not self.keywords: + return () + groups: list[list[tuple[str, int]]] = [ + [] for _ in range(max(map(len, self.keywords)) + 1) + ] + for keyword, token_type in self.keywords.items(): + groups[len(keyword)].append((keyword, token_type)) + return tuple(tuple(group) for group in groups) + + def _prepare_trailer(self, rules: tuple[CRule, ...]) -> str | None: if self.skip_actions: mode = 0 else: - mode = int(self.rules["start"].type == "mod_ty") if "start" in self.rules else 1 - if mode == 1 and self.grammar.metas.get("bytecode"): - mode += 1 + start = next((rule.signature for rule in rules if rule.signature.name == "start"), None) + match start: + case None | CRuleSignature(return_type="mod_ty"): + mode = 2 if self.grammar.metas.get("bytecode") else 1 + case _: + mode = 0 modulename = self.grammar.metas.get("modulename", "parse") - trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX) - if trailer: - self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)) - - def _group_keywords_by_length(self) -> dict[int, list[tuple[str, int]]]: - groups: dict[int, list[tuple[str, int]]] = {} - for keyword_str, keyword_type in self.keywords.items(): - length = len(keyword_str) - if length in groups: - groups[length].append((keyword_str, keyword_type)) - else: - groups[length] = [(keyword_str, keyword_type)] - return groups - - def _setup_keywords(self) -> None: - n_keyword_lists = ( - len(max(self.keywords.keys(), key=len)) + 1 if len(self.keywords) > 0 else 0 - ) - self.print(f"static const int n_keyword_lists = {n_keyword_lists};") - groups = self._group_keywords_by_length() - self.print("static KeywordToken *reserved_keywords[] = {") - with self.indent(): - num_groups = max(groups) + 1 if groups else 1 - for keywords_length in range(num_groups): - if keywords_length not in groups.keys(): - self.print("(KeywordToken[]) {{NULL, -1}},") - else: - self.print("(KeywordToken[]) {") - with self.indent(): - for keyword_str, keyword_type in groups[keywords_length]: - self.print(f'{{"{keyword_str}", {keyword_type}}},') - self.print("{NULL, -1},") - self.print("},") - self.print("};") - - def _setup_soft_keywords(self) -> None: - soft_keywords = sorted(self.soft_keywords) - self.print("static char *soft_keywords[] = {") - with self.indent(): - for keyword in soft_keywords: - self.print(f'"{keyword}",') - self.print("NULL,") - self.print("};") - - def _set_up_token_start_metadata_extraction(self) -> None: - self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {") - with self.indent(): - self.print("p->error_indicator = 1;") - self.add_return("NULL") - self.print("}") - self.print("int _start_lineno = p->tokens[_mark]->lineno;") - self.print("UNUSED(_start_lineno); // Only used by EXTRA macro") - self.print("int _start_col_offset = p->tokens[_mark]->col_offset;") - self.print("UNUSED(_start_col_offset); // Only used by EXTRA macro") - - def _set_up_token_end_metadata_extraction(self) -> None: - self.print("Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);") - self.print("if (_token == NULL) {") - with self.indent(): - self.add_return("NULL") - self.print("}") - self.print("int _end_lineno = _token->end_lineno;") - self.print("UNUSED(_end_lineno); // Only used by EXTRA macro") - self.print("int _end_col_offset = _token->end_col_offset;") - self.print("UNUSED(_end_col_offset); // Only used by EXTRA macro") - - def _check_for_errors(self) -> None: - self.print("if (p->error_indicator) {") - with self.indent(): - self.add_return("NULL") - self.print("}") - - def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: - self.print("{") - with self.indent(): - self.add_level() - self.print(f"{result_type} _res = NULL;") - self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") - with self.indent(): - self.add_return("_res") - self.print("}") - self.print("int _mark = p->mark;") - self.print("int _resmark = p->mark;") - self.print(f"Memo *_memo = _PyPegen_insert_memo_direct(p, _mark, {node.name}_type);") - self.print("if (_memo == NULL) {") - with self.indent(): - self.add_return("NULL") - self.print("}") - self.print("while (1) {") - with self.indent(): - self.print("_memo->node = _res;") - self.print("_memo->mark = p->mark;") - self.print("p->mark = _mark;") - self.print(f"void *_raw = {node.name}_raw(p);") - self.print("if (p->error_indicator) {") - with self.indent(): - self.add_return("NULL") - self.print("}") - self.print("if (_raw == NULL || p->mark <= _resmark)") - with self.indent(): - self.print("break;") - self.print("_resmark = p->mark;") - self.print("_res = _raw;") - self.print("}") - self.print("p->mark = _resmark;") - self.add_return("_res") - self.print("}") - self.print(f"static {result_type}") - self.print(f"{node.name}_raw(Parser *p)") - - def _should_memoize(self, node: Rule) -> bool: - return "memo" in node.flags and not node.left_recursive - - def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None: - memoize = self._should_memoize(node) - - with self.indent(): - self.add_level() - self._check_for_errors() - self.print(f"{result_type} _res = NULL;") - if memoize: - self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") - with self.indent(): - self.add_return("_res") - self.print("}") - self.print("int _mark = p->mark;") - self.prepare_prefix_calls(rhs) - if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): - self._set_up_token_start_metadata_extraction() - self.visit( - rhs, - is_loop=False, - is_gather=node.is_gather(), - rulename=node.name, - ) - if self.debug: - self.print(f'D(fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark));') - self.print("_res = NULL;") - self.print(" done:") - with self.indent(): - if memoize: - self.print(f"_PyPegen_insert_memo(p, _mark, {node.name}_type, _res);") - self.add_return("_res") - - def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: - memoize = self._should_memoize(node) - is_repeat1 = node.name.startswith("_loop1") - - with self.indent(): - self.add_level() - self._check_for_errors() - self.print("void *_res = NULL;") - if memoize: - self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") - with self.indent(): - self.add_return("_res") - self.print("}") - self.print("int _mark = p->mark;") - if memoize: - self.print("int _start_mark = p->mark;") - self.print("void **_children = PyMem_Malloc(sizeof(void *));") - self.out_of_memory_return("!_children") - self.print("Py_ssize_t _children_capacity = 1;") - self.print("Py_ssize_t _n = 0;") - if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): - self._set_up_token_start_metadata_extraction() - self.visit( - rhs, - is_loop=True, - is_gather=node.is_gather(), - rulename=node.name, - ) - if is_repeat1: - self.print("if (_n == 0 || p->error_indicator) {") - with self.indent(): - self.print("PyMem_Free(_children);") - self.add_return("NULL") - self.print("}") - self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") - self.out_of_memory_return("!_seq", cleanup_code="PyMem_Free(_children);") - self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") - self.print("PyMem_Free(_children);") - if memoize and node.name: - self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") - self.add_return("_seq") - - def prepare_prefix_calls(self, rhs: Rhs) -> None: - # Reuse a memoized, consuming prefix only within a consecutive group. - # Suffix parsing starts after the prefix and cannot revisit its start - # through ordinary grammar backtracking. Diagnostic calls are unchanged. - def candidate(alt: Alt) -> Rule | None: - if not alt.items or not isinstance(alt.items[0].item, NameLeaf): - return None - rule = self.rules.get(alt.items[0].item.value) - if rule is None or rule.name not in self.consuming: - return None - if self._should_memoize(rule) or (rule.left_recursive and rule.leader): - return rule - return None - - i = 0 - while i < len(rhs.alts): - rule = candidate(rhs.alts[i]) - j = i + 1 - while rule is not None and j < len(rhs.alts) and candidate(rhs.alts[j]) is rule: - j += 1 - if rule is not None and j - i > 1: - name = self.unique_varname("_prefix") - result, end, valid = name + "_result", name + "_end", name + "_valid" - self.print(f"{rule.type or 'void *'} {result} = NULL;") - self.print(f"int {end} = 0, {valid} = 0;") - for alt in rhs.alts[i:j]: - self.prefix_calls[id(alt.items[0])] = result, end, valid - i = j - - def visit_Rule(self, node: Rule) -> None: - self.prefix_calls = {} - is_loop = node.is_loop() - is_gather = node.is_gather() - rhs = node.flatten() - if is_loop or is_gather: - result_type = "asdl_seq *" - elif node.type: - result_type = node.type - else: - result_type = "void *" - - for line in str(node).splitlines(): - self.print(f"// {line}") - if node.left_recursive and node.leader: - self.print(f"static {result_type} {node.name}_raw(Parser *);") - - self.print(f"static {result_type}") - self.print(f"{node.name}_rule(Parser *p)") - - if node.left_recursive and node.leader: - self._set_up_rule_memoization(node, result_type) - - self.print("{") - - if node.name.endswith("without_invalid"): - with self.indent(): - self.print("int _prev_call_invalid = p->call_invalid_rules;") - self.print("p->call_invalid_rules = 0;") - self.cleanup_statements.append("p->call_invalid_rules = _prev_call_invalid;") - - if is_loop: - self._handle_loop_rule_body(node, rhs) - else: - self._handle_default_rule_body(node, rhs, result_type) - - if node.name.endswith("without_invalid"): - self.cleanup_statements.pop() - - self.print("}") - - def visit_NamedItem(self, node: NamedItem) -> None: - call = self.callmakervisitor.generate_call(node) - if id(node) in self.prefix_calls: - result, end, valid = self.prefix_calls[id(node)] - original = f"{call.function}({', '.join(map(str, call.arguments))})" - call.function = ( - f"((!p->call_invalid_rules && {valid}) ? " - f"(p->mark = {end}, {result}) : " - f"({result} = {original}, {end} = p->mark, {valid} = 1, {result}))" - ) - call.arguments = [] - if call.assigned_variable: - call.assigned_variable = self.dedupe(call.assigned_variable) - self.print(call) - - def visit_Rhs( - self, node: Rhs, is_loop: bool, is_gather: bool, rulename: str | None - ) -> None: - if is_loop: - assert len(node.alts) == 1 - for alt in node.alts: - self.visit(alt, is_loop=is_loop, is_gather=is_gather, rulename=rulename) - - def join_conditions(self, keyword: str, node: Any) -> None: - self.print(f"{keyword} (") - with self.indent(): - first = True - for item in node.items: - if first: - first = False - else: - self.print("&&") - self.visit(item) - self.print(")") - - def emit_action(self, node: Alt, cleanup_code: str | None = None) -> None: - self.print(f"_res = {node.action};") - - self.print("if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {") - with self.indent(): - self.print("p->error_indicator = 1;") - if cleanup_code: - self.print(cleanup_code) - self.add_return("NULL") - self.print("}") - - if self.debug: - self.print( - f'D(fprintf(stderr, "Hit with action [%d-%d]: %s\\n", _mark, p->mark, "{node}"));' - ) - - def emit_default_action(self, is_gather: bool, node: Alt) -> None: - if len(self.local_variable_names) > 1: - if is_gather: - assert len(self.local_variable_names) == 2 - self.print( - f"_res = _PyPegen_seq_insert_in_front(p, " - f"{self.local_variable_names[0]}, {self.local_variable_names[1]});" - ) - else: - if self.debug: - self.print( - f'D(fprintf(stderr, "Hit without action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));' - ) - self.print( - f"_res = _PyPegen_dummy_name(p, {', '.join(self.local_variable_names)});" - ) - else: - if self.debug: - self.print( - f'D(fprintf(stderr, "Hit with default action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));' - ) - self.print(f"_res = {self.local_variable_names[0]};") - - def emit_dummy_action(self) -> None: - self.print("_res = _PyPegen_dummy_name(p);") - - def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: str | None) -> None: - self.join_conditions(keyword="if", node=node) - self.print("{") - # We have parsed successfully all the conditions for the option. - with self.indent(): - node_str = str(node).replace('"', '\\"') - self.print( - f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' - ) - # Prepare to emit the rule action and do so - if node.action and "EXTRA" in node.action: - self._set_up_token_end_metadata_extraction() - if self.skip_actions: - self.emit_dummy_action() - elif node.action: - self.emit_action(node) - else: - self.emit_default_action(is_gather, node) - - # As the current option has parsed correctly, do not continue with the rest. - self.print("goto done;") - self.print("}") - - def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: str | None) -> None: - # Condition of the main body of the alternative - self.join_conditions(keyword="while", node=node) - self.print("{") - # We have parsed successfully one item! - with self.indent(): - # Prepare to emit the rule action and do so - if node.action and "EXTRA" in node.action: - self._set_up_token_end_metadata_extraction() - if self.skip_actions: - self.emit_dummy_action() - elif node.action: - self.emit_action(node, cleanup_code="PyMem_Free(_children);") - else: - self.emit_default_action(is_gather, node) - - # Add the result of rule to the temporary buffer of children. This buffer - # will populate later an asdl_seq with all elements to return. - self.print("if (_n == _children_capacity) {") - with self.indent(): - self.print("_children_capacity *= 2;") - self.print( - "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" - ) - self.out_of_memory_return("!_new_children", cleanup_code="PyMem_Free(_children);") - self.print("_children = _new_children;") - self.print("}") - self.print("_children[_n++] = _res;") - self.print("_mark = p->mark;") - self.print("}") - - def visit_Alt( - self, node: Alt, is_loop: bool, is_gather: bool, rulename: str | None - ) -> None: - if len(node.items) == 1 and str(node.items[0]).startswith("invalid_"): - self.print(f"if (p->call_invalid_rules) {{ // {node}") - else: - self.print(f"{{ // {node}") - with self.indent(): - self._check_for_errors() - node_str = str(node).replace('"', '\\"') - self.print( - f'D(fprintf(stderr, "%*c> {rulename}[%d-%d]: %s\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' - ) - # Prepare variable declarations for the alternative - vars = self.collect_vars(node) - for v, var_type in sorted(item for item in vars.items() if item[0] is not None): - if not var_type: - var_type = "void *" - else: - var_type += " " - if v == "_cut_var": - v += " = 0" # cut_var must be initialized - self.print(f"{var_type}{v};") - if v and v.startswith("_opt_var"): - self.print(f"UNUSED({v}); // Silence compiler warnings") - - with self.local_variable_context(): - if is_loop: - self.handle_alt_loop(node, is_gather, rulename) - else: - self.handle_alt_normal(node, is_gather, rulename) - - self.print("p->mark = _mark;") - node_str = str(node).replace('"', '\\"') - self.print( - f"D(fprintf(stderr, \"%*c%s {rulename}[%d-%d]: %s failed!\\n\", p->level, ' ',\n" - f' p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "{node_str}"));' - ) - if "_cut_var" in vars: - self.print("if (_cut_var) {") - with self.indent(): - self.add_return("NULL") - self.print("}") - self.print("}") - - def collect_vars(self, node: Alt) -> dict[str | None, str | None]: - types = {} - with self.local_variable_context(): - for item in node.items: - name, type = self.add_var(item) - types[name] = type - return types - - def add_var(self, node: NamedItem) -> tuple[str | None, str | None]: - call = self.callmakervisitor.generate_call(node.item) - name = node.name if node.name else call.assigned_variable - if name is not None: - name = self.dedupe(name) - return_type = call.return_type if node.type is None else node.type - return name, return_type + if trailer := self.grammar.metas.get("trailer", EXTENSION_SUFFIX): + return trailer.rstrip("\n") % dict(mode=mode, modulename=modulename) + return None diff --git a/Tools/peg_generator/pegen/c_generator_calls.py b/Tools/peg_generator/pegen/c_generator_calls.py new file mode 100644 index 000000000000000..dc3c0cca1304121 --- /dev/null +++ b/Tools/peg_generator/pegen/c_generator_calls.py @@ -0,0 +1,553 @@ +"""Discover calls and helper rules, then prepare rules from resolved calls.""" + +import ast +import re +from collections.abc import Mapping +from dataclasses import replace +from types import MappingProxyType +from typing import TYPE_CHECKING, Any + +from pegen.c_generator_model import ( + CAction, + CAlternative, + CBindingKind, + CPrefix, + CRule, + CRuleSignature, + CVariable, +) +from pegen.c_generator_model import ( + FunctionCall as FunctionCall, +) +from pegen.c_generator_model import ( + NodeTypes as NodeTypes, +) +from pegen.grammar import ( + Alt, + Cut, + Forced, + Gather, + GrammarVisitor, + Group, + Item, + Leaf, + Lookahead, + NamedItem, + NameLeaf, + NegativeLookahead, + Opt, + PositiveLookahead, + Repeat0, + Repeat1, + Rhs, + Rule, + RuleKind, + StringLeaf, +) + +if TYPE_CHECKING: + from pegen.parser_generator import ParserGenerator + + +BASE_NODETYPES = { + "NAME": NodeTypes.NAME_TOKEN, + "NUMBER": NodeTypes.NUMBER_TOKEN, + "STRING": NodeTypes.STRING_TOKEN, + "SOFT_KEYWORD": NodeTypes.SOFT_KEYWORD, +} + +_HelperNode = Rhs | Repeat0 | Repeat1 | Gather + + +def rule_signature(rule: Rule) -> CRuleSignature: + return_type = rule.type if rule.kind is RuleKind.NORMAL else "asdl_seq *" + return CRuleSignature(rule.name, rule.kind, return_type) + + +def bind_call(node: NamedItem, call: FunctionCall) -> FunctionCall: + if not node.name and not node.type: + return call + return replace( + call, + assigned_variable=node.name or call.assigned_variable, + assigned_variable_type=node.type or call.assigned_variable_type, + binding_kind=CBindingKind.NORMAL if node.name else call.binding_kind, + ) + + +def consuming_rules(rules: dict[str, Rule]) -> set[str]: + """Conservatively prove which rules consume a token whenever they succeed.""" + consuming: set[str] = set() + + def consumes(node: Any) -> bool: + if isinstance(node, NamedItem): + return consumes(node.item) + if isinstance(node, NameLeaf): + return node.value not in rules or node.value in consuming + if isinstance(node, StringLeaf): + return True + if isinstance(node, Group): + return consumes(node.rhs) + if isinstance(node, Rhs): + return bool(node.alts) and all(any(consumes(i) for i in alt.items) for alt in node.alts) + if isinstance(node, (Forced, Repeat1, Gather)): + return consumes(node.node) + # Predicates, cuts, optional items, and zero-or-more items can succeed + # without consuming. Actions are assumed not to rewrite parser marks. + return False + + while True: + added = {name for name, rule in rules.items() + if name not in consuming and consumes(rule.rhs)} + if not added: + return consuming + consuming.update(added) + + +class CCallMakerVisitor(GrammarVisitor): + def __init__( + self, + parser_generator: "ParserGenerator", + exact_tokens: dict[str, int], + non_exact_tokens: set[str], + ): + self._registry = parser_generator + self._keywords = parser_generator.keywords + self._exact_tokens = exact_tokens + self._non_exact_tokens = non_exact_tokens + self._helper_cache: dict[tuple[type, str], str] = {} + self._calls: dict[NamedItem, tuple[Item, FunctionCall]] = {} + + def visit(self, node: Any, *args: Any, **kwargs: Any) -> FunctionCall: + match node: + case NamedItem(item=item): + call = self.visit(item) + self._calls[node] = (item, call) + return bind_call(node, call) + case NameLeaf(): + return self._name_call(node) + case StringLeaf(): + return self._string_call(node) + case PositiveLookahead(): + return self._lookahead_call(node, 1) + case NegativeLookahead(): + return self._lookahead_call(node, 0) + case Forced(): + return self._forced_call(node) + case Opt(): + return self._optional_call(node) + case Rhs(can_be_inlined=True): + return self.visit(node.alts[0].items[0]) + case Rhs() | Repeat0() | Repeat1() | Gather(): + return self._helper_call(node) + case Group(rhs=rhs): + return self.visit(rhs) + case Cut(): + return FunctionCall( + assigned_variable="_cut_var", + return_type="int", + function="1", + nodetype=NodeTypes.CUT_OPERATOR, + binding_kind=CBindingKind.CUT, + ) + case _: + return self.generic_visit(node, *args, **kwargs) + + def _keyword_call(self, keyword: str) -> FunctionCall: + return FunctionCall( + assigned_variable="_keyword", + function="_PyPegen_expect_token", + arguments=("p", self._keywords[keyword]), + return_type="Token *", + nodetype=NodeTypes.KEYWORD, + comment=f"token='{keyword}'", + ) + + def _soft_keyword_call(self, value: str) -> FunctionCall: + return FunctionCall( + assigned_variable="_keyword", + function="_PyPegen_expect_soft_keyword", + arguments=("p", value), + return_type="expr_ty", + nodetype=NodeTypes.SOFT_KEYWORD, + comment=f"soft_keyword='{value}'", + ) + + def _name_call(self, node: NameLeaf) -> FunctionCall: + name = node.value + if name in self._non_exact_tokens: + if name in BASE_NODETYPES: + return FunctionCall( + assigned_variable=f"{name.lower()}_var", + function=f"_PyPegen_{name.lower()}_token", + arguments=("p",), + nodetype=BASE_NODETYPES[name], + return_type="expr_ty", + comment=name, + ) + return FunctionCall( + assigned_variable=f"{name.lower()}_var", + function="_PyPegen_expect_token", + arguments=("p", name), + nodetype=NodeTypes.GENERIC_TOKEN, + return_type="Token *", + comment=f"token='{name}'", + ) + + type = None + if (signature := self._lookup_rule(name)) is not None: + type = signature.return_type + + return FunctionCall( + assigned_variable=f"{name}_var", + function=f"{name}_rule", + arguments=("p",), + return_type=type, + comment=f"{node}", + ) + + def _string_call(self, node: StringLeaf) -> FunctionCall: + val = ast.literal_eval(node.value) + if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword + if node.value.endswith("'"): + return self._keyword_call(val) + else: + return self._soft_keyword_call(node.value) + else: + assert val in self._exact_tokens, f"{node.value} is not a known literal" + type = self._exact_tokens[val] + return FunctionCall( + assigned_variable="_literal", + function="_PyPegen_expect_token", + arguments=("p", type), + nodetype=NodeTypes.GENERIC_TOKEN, + return_type="Token *", + comment=f"token='{val}'", + ) + + def _assert_compatible_return_type( + self, call: FunctionCall, wrapper: str, expected_rtype: str | None, + ) -> None: + if call.return_type != expected_rtype: + raise RuntimeError( + f"{call.function} return type is incompatible with {wrapper}: " + f"expect: {expected_rtype}, actual: {call.return_type}" + ) + + def _lookahead_call(self, node: Lookahead, positive: int) -> FunctionCall: + call = self.visit(node.node) + comment = None + match call: + case FunctionCall(nodetype=NodeTypes.NAME_TOKEN): + function = "_PyPegen_lookahead_for_expr" + self._assert_compatible_return_type(call, function, "expr_ty") + case FunctionCall(nodetype=NodeTypes.STRING_TOKEN): + # _PyPegen_string_token() returns 'void *' instead of 'Token *'; + # in addition, the overall function call would return 'expr_ty'. + assert call.function == "_PyPegen_string_token" + function = "_PyPegen_lookahead" + self._assert_compatible_return_type(call, function, "expr_ty") + case FunctionCall(nodetype=NodeTypes.SOFT_KEYWORD): + function = "_PyPegen_lookahead_with_string" + self._assert_compatible_return_type(call, function, "expr_ty") + case FunctionCall(nodetype=NodeTypes.GENERIC_TOKEN | NodeTypes.KEYWORD): + function = "_PyPegen_lookahead_with_int" + self._assert_compatible_return_type(call, function, "Token *") + comment = f"token={node.node}" + case FunctionCall(return_type="expr_ty"): + function = "_PyPegen_lookahead_for_expr" + case FunctionCall(return_type="stmt_ty"): + function = "_PyPegen_lookahead_for_stmt" + case _: + function = "_PyPegen_lookahead" + self._assert_compatible_return_type(call, function, None) + return FunctionCall( + function=function, + arguments=(positive, call.function, *call.arguments), + return_type="int", + comment=comment, + ) + + def _forced_call(self, node: Forced) -> FunctionCall: + call = self.visit(node.node) + match node.node: + case Leaf(value=value): + val = ast.literal_eval(value) + assert val in self._exact_tokens, f"{value} is not a known literal" + return FunctionCall( + assigned_variable="_literal", + function="_PyPegen_expect_forced_token", + arguments=("p", self._exact_tokens[val], f'"{val}"'), + nodetype=NodeTypes.GENERIC_TOKEN, + return_type="Token *", + comment=f"forced_token='{val}'", + ) + case Group(rhs=rhs): + return FunctionCall( + assigned_variable="_literal", + function="_PyPegen_expect_forced_result", + arguments=("p", call.expression(), f'"{rhs!s}"'), + return_type="void *", + comment=f"forced_token=({rhs!s})", + ) + case _: + raise NotImplementedError(f"Forced tokens don't work with {node.node} nodes") + + def _optional_call(self, node: Opt) -> FunctionCall: + call = self.visit(node.node) + return FunctionCall( + assigned_variable="_opt_var", + function=call.function, + arguments=call.arguments, + force_true=True, + comment=f"{node}", + binding_kind=CBindingKind.OPTIONAL, + ) + + def _helper_call( + self, + node: _HelperNode, + ) -> FunctionCall: + node_str = f"{node}" + signature = self._resolve_artificial_rule(node) + name = signature.name + return FunctionCall( + assigned_variable=f"{name}_var", + function=f"{name}_rule", + arguments=("p",), + return_type=signature.return_type, + comment=node_str, + ) + + def _lookup_rule(self, name: str) -> CRuleSignature | None: + if (rule := self._registry.all_rules.get(name.lower())) is not None: + return rule_signature(rule) + return None + + def _resolve_artificial_rule(self, node: _HelperNode) -> CRuleSignature: + # Preserve helper reuse and numbering from the fixed-point traversal. + key = (type(node), str(node)) + if (name := self._helper_cache.get(key)) is None: + match node: + case Rhs(): + name = self._registry.artificial_rule_from_rhs(node) + case Repeat0(node=child): + name = self._registry.artificial_rule_from_repeat(child, is_repeat1=False) + case Repeat1(node=child): + name = self._registry.artificial_rule_from_repeat(child, is_repeat1=True) + case Gather(): + name = self._registry.artificial_rule_from_gather(node) + self._helper_cache[key] = name + return rule_signature(self._registry.all_rules[name]) + + def make_lowerer(self) -> "CCallLowerer": + return CCallLowerer( + calls=self._calls, + rules=self._registry.all_rules, + original_rules=self._registry.rules, + signatures={ + name: rule_signature(rule) for name, rule in self._registry.all_rules.items() + }, + ) + + +class CCallLowerer: + """Resolve bindings, actions and control flow without registering rules. + + Discovery and lowering operate on the same, unchanged grammar. + """ + + def __init__( + self, + *, + calls: Mapping[NamedItem, tuple[Item, FunctionCall]], + rules: Mapping[str, Rule], + original_rules: dict[str, Rule], + signatures: Mapping[str, CRuleSignature], + ): + self._calls = MappingProxyType(dict(calls)) + self._signatures = MappingProxyType(dict(signatures)) + self._prefixes: dict[str, tuple[CPrefix, ...]] = {} + self._prefix_calls: dict[NamedItem, CPrefix] = {} + consuming = consuming_rules(original_rules) + counter = 0 + + def candidate(alt: Alt) -> Rule | None: + if not alt.items or not isinstance(alt.items[0].item, NameLeaf): + return None + rule = original_rules.get(alt.items[0].item.value) + if rule is None or rule.name not in consuming: + return None + if ("memo" in rule.flags and not rule.left_recursive) or ( + rule.left_recursive and rule.leader + ): + return rule + return None + + for rule in rules.values(): + if rule.kind in {RuleKind.LOOP0, RuleKind.LOOP1}: + continue + # Reuse a consuming prefix only within a consecutive group. + # Diagnostic calls still invoke the original rule. + prefixes = [] + alts = rule.flatten().alts + i = 0 + while i < len(alts): + prefix_rule = candidate(alts[i]) + j = i + 1 + while prefix_rule is not None and j < len(alts) and candidate(alts[j]) is prefix_rule: + j += 1 + if prefix_rule is not None and j - i > 1: + prefix = CPrefix(f"_prefix_{counter}", prefix_rule.type or "void *") + counter += 1 + prefixes.append(prefix) + for alt in alts[i:j]: + self._prefix_calls[alt.items[0]] = prefix + i = j + self._prefixes[rule.name] = tuple(prefixes) + + def prepare_rule(self, rule: Rule, *, skip_actions: bool = False) -> CRule: + if (signature := self._signatures.get(rule.name)) is None: + raise RuntimeError(f"Rule {rule.name!r} was not discovered") + rhs = rule.flatten() + if signature.kind in {RuleKind.LOOP0, RuleKind.LOOP1}: + assert len(rhs.alts) == 1 + return CRule( + signature=signature, + text=str(rule), + alternatives=tuple( + self.prepare_alt(alt, kind=signature.kind, skip_actions=skip_actions) + for alt in rhs.alts + ), + left_recursive=rule.left_recursive, + leader=rule.leader, + memoize="memo" in rule.flags and not rule.left_recursive, + disable_invalid_rules=rule.name.endswith("without_invalid"), + prefixes=self._prefixes.get(rule.name, ()), + ) + + def prepare_alt( + self, + node: Alt, + *, + kind: RuleKind = RuleKind.NORMAL, + skip_actions: bool = False, + ) -> CAlternative: + calls: list[FunctionCall] = [] + variables: dict[str, CVariable] = {} + cut_variable = None + for item in node.items: + recorded = self._calls.get(item) + if recorded is None or recorded[0] is not item.item: + raise RuntimeError(f"Item {item} was not discovered") + call = bind_call(item, recorded[1]) + if (prefix := self._prefix_calls.get(item)) is not None: + result, end, valid = prefix.result, prefix.end, prefix.valid + original = call.expression() + call = replace( + call, + function=( + f"((!p->call_invalid_rules && {valid}) ? " + f"(p->mark = {end}, {result}) : " + f"({result} = {original}, {end} = p->mark, {valid} = 1, {result}))" + ), + arguments=(), + ) + if original_name := call.assigned_variable: + name = original_name + counter = 0 + while name in variables: + counter += 1 + name = f"{original_name}_{counter}" + if name != original_name: + call = replace(call, assigned_variable=name) + initializer = ( + "0" if call.binding_kind is CBindingKind.CUT and cut_variable is None + else None + ) + if initializer is not None: + cut_variable = name + variables[name] = CVariable( + name=name, + type=call.return_type if item.type is None else item.type, + initializer=initializer, + unused=call.binding_kind is CBindingKind.OPTIONAL, + ) + calls.append(call) + return CAlternative( + text=str(node), + action=self._prepare_action(node, list(variables), kind, skip_actions), + calls=tuple(calls), + variables=tuple(variables.values()), + cut_variable=cut_variable, + requires_invalid_rules=self._requires_invalid_rules(node), + uses_locations=bool(node.action and "EXTRA" in node.action), + ) + + def _requires_invalid_rules(self, node: Alt) -> bool: + match node.items: + case [NamedItem(item=item)]: + pass + case _: + return False + # Preserve the source convention for bare, optional and repeated + # invalid references, including an invalid gather separator. + while True: + match item: + case Rhs(alts=[Alt(items=[NamedItem(item=child)])]): + item = child + case Gather(separator=separator): + item = separator + case Opt(node=child) | Repeat0(node=child) | Repeat1(node=child): + # A compound optional may match empty, so an invalid + # reference inside it must not gate the whole alternative. + if self._is_compound(child): + return False + item = child + case NameLeaf(value=name): + return name.startswith("invalid_") + case _: + return False + + def _is_compound(self, item: Item) -> bool: + match item: + case Rhs(alts=alts): + return len(alts) > 1 or any( + len(alt.items) > 1 or any(self._is_compound(part.item) for part in alt.items) + for alt in alts + ) + case Group(rhs=rhs): + return self._is_compound(rhs) + case Gather(separator=separator, node=child): + return self._is_compound(separator) or self._is_compound(child) + case ( + Opt(node=child) | Repeat0(node=child) | Repeat1(node=child) + | Forced(node=child) | Lookahead(node=child) + ): + return self._is_compound(child) + case StringLeaf(value=value): + return " " in value + case _: + return False + + @staticmethod + def _prepare_action( + node: Alt, names: list[str], kind: RuleKind, skip_actions: bool, + ) -> CAction: + if skip_actions: + return CAction("_PyPegen_dummy_name(p)") + if action := node.action: + return CAction( + action, checked=True, debug_message="Hit with action [%d-%d]: %s", + ) + match names: + case [first, rest] if kind is RuleKind.GATHER: + return CAction(f"_PyPegen_seq_insert_in_front(p, {first}, {rest})") + case [_, _, *_]: + assert kind is not RuleKind.GATHER + return CAction( + f"_PyPegen_dummy_name(p, {', '.join(names)})", + debug_message="Hit without action [%d:%d]: %s", + ) + case _: + return CAction(names[0], debug_message="Hit with default action [%d:%d]: %s") diff --git a/Tools/peg_generator/pegen/c_generator_file.py b/Tools/peg_generator/pegen/c_generator_file.py new file mode 100644 index 000000000000000..4ce9345d5f5c823 --- /dev/null +++ b/Tools/peg_generator/pegen/c_generator_file.py @@ -0,0 +1,82 @@ +"""Emit a complete C parser from an immutable plan and local output state.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import IO + +from pegen.c_generator_model import CParser +from pegen.c_generator_rules import CRuleEmitter + + +class CParserEmitter: + def __init__(self, parser: CParser, file: IO[str] | None): + self._parser = parser + self._file = file + self._level = 0 + + def emit(self) -> None: + self._emit_headers() + self._emit_keywords() + self._emit_declarations() + self._emit_rules() + if self._parser.trailer is not None: + self.print(self._parser.trailer) + + def _emit_headers(self) -> None: + self.print(f"// @generated by pegen from {self._parser.source_name}") + for header in self._parser.headers: + self.print(header) + + def _emit_keywords(self) -> None: + groups = self._parser.keyword_groups + self.print(f"static const int n_keyword_lists = {len(groups)};") + self.print("static KeywordToken *reserved_keywords[] = {") + with self.indent(): + for group in groups or ((),): + if not group: + self.print("(KeywordToken[]) {{NULL, -1}},") + else: + self.print("(KeywordToken[]) {") + with self.indent(): + for keyword, token_type in group: + self.print(f'{{"{keyword}", {token_type}}},') + self.print("{NULL, -1},") + self.print("},") + self.print("};") + self.print("static char *soft_keywords[] = {") + with self.indent(): + for keyword in self._parser.soft_keywords: + self.print(f'"{keyword}",') + self.print("NULL,") + self.print("};") + + def _emit_declarations(self) -> None: + for rule_id, rule in enumerate(self._parser.rules, 1000): + comment = " // Left-recursive" if rule.left_recursive else "" + self.print(f"#define {rule.signature.name}_type {rule_id}{comment}") + self.print() + for rule in self._parser.rules: + self.print(rule.signature.declaration()) + self.print() + + def _emit_rules(self) -> None: + for rule in self._parser.rules: + self.print() + if rule.left_recursive: + self.print("// Left-recursive") + CRuleEmitter(self, rule, debug=self._parser.debug).emit() + + def print(self, *args: object) -> None: + if not args: + print(file=self._file) + else: + print(" " * self._level, end="", file=self._file) + print(*args, file=self._file) + + @contextmanager + def indent(self) -> Iterator[None]: + self._level += 1 + try: + yield + finally: + self._level -= 1 diff --git a/Tools/peg_generator/pegen/c_generator_model.py b/Tools/peg_generator/pegen/c_generator_model.py new file mode 100644 index 000000000000000..7b25977d70efb5f --- /dev/null +++ b/Tools/peg_generator/pegen/c_generator_model.py @@ -0,0 +1,142 @@ +"""Immutable C parser descriptions shared by lowering and emission.""" + +from dataclasses import dataclass +from enum import Enum, auto + +from pegen.grammar import RuleKind + + +class NodeTypes(Enum): + NAME_TOKEN = 0 + NUMBER_TOKEN = 1 + STRING_TOKEN = 2 + GENERIC_TOKEN = 3 + KEYWORD = 4 + SOFT_KEYWORD = 5 + CUT_OPERATOR = 6 + F_STRING_CHUNK = 7 + + +class CBindingKind(Enum): + NORMAL = auto() + OPTIONAL = auto() + CUT = auto() + + +@dataclass(frozen=True, slots=True) +class FunctionCall: + function: str + arguments: tuple[str | int, ...] = () + assigned_variable: str | None = None + assigned_variable_type: str | None = None + return_type: str | None = None + nodetype: NodeTypes | None = None + force_true: bool = False + comment: str | None = None + binding_kind: CBindingKind = CBindingKind.NORMAL + + def expression(self) -> str: + """Render the invocation without its alternative-local binding or comment.""" + expression = self.function + if arguments := self.arguments: + expression += f"({', '.join(map(str, arguments))})" + if self.force_true: + expression += ", !p->error_indicator" + return expression + + def __str__(self) -> str: + expression = self.expression() + if variable := self.assigned_variable: + cast = f"({self.assigned_variable_type})" if self.assigned_variable_type else "" + expression = f"({variable} = {cast}{expression})" + if comment := self.comment: + expression += f" // {comment}" + return expression + + +@dataclass(frozen=True, slots=True) +class CRuleSignature: + name: str + kind: RuleKind + return_type: str | None + + @property + def c_return_type(self) -> str: + return self.return_type or "void *" + + def declaration(self) -> str: + separator = " " if self.kind is RuleKind.NORMAL and self.return_type else "" + return f"static {self.c_return_type}{separator}{self.name}_rule(Parser *p);" + + +@dataclass(frozen=True, slots=True) +class CVariable: + name: str + type: str | None + initializer: str | None = None + unused: bool = False + + +@dataclass(frozen=True, slots=True) +class CAction: + expression: str + checked: bool = False + debug_message: str | None = None + + +@dataclass(frozen=True, slots=True) +class CAlternative: + text: str + action: CAction + calls: tuple[FunctionCall, ...] + variables: tuple[CVariable, ...] + cut_variable: str | None + requires_invalid_rules: bool + uses_locations: bool + + +@dataclass(frozen=True, slots=True) +class CPrefix: + name: str + type: str + + @property + def result(self) -> str: + return self.name + "_result" + + @property + def end(self) -> str: + return self.name + "_end" + + @property + def valid(self) -> str: + return self.name + "_valid" + + +@dataclass(frozen=True, slots=True) +class CRule: + signature: CRuleSignature + text: str + alternatives: tuple[CAlternative, ...] + left_recursive: bool + leader: bool + memoize: bool + disable_invalid_rules: bool + prefixes: tuple[CPrefix, ...] = () + + @property + def uses_locations(self) -> bool: + return any(alt.uses_locations for alt in self.alternatives) + + +@dataclass(frozen=True, slots=True) +class CParser: + """Complete file-emission input, independent of compilation state.""" + + source_name: str + headers: tuple[str, ...] + keyword_groups: tuple[tuple[tuple[str, int], ...], ...] + soft_keywords: tuple[str, ...] + rules: tuple[CRule, ...] + trailer: str | None + debug: bool diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py new file mode 100644 index 000000000000000..6dd598beb1b39c4 --- /dev/null +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -0,0 +1,363 @@ +"""Render prepared C rules with rule-local output and cleanup state.""" + +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from typing import Protocol + +from pegen.c_generator_model import CAlternative, CRule +from pegen.grammar import RuleKind + + +class CWriter(Protocol): + def print(self, *args: object) -> None: + ... + + def indent(self) -> AbstractContextManager[None]: + ... + + +@dataclass(frozen=True, slots=True) +class _CReturnEmitter: + writer: CWriter + cleanups: tuple[str, ...] = () + + def with_cleanup(self, cleanup: str) -> "_CReturnEmitter": + return _CReturnEmitter(self.writer, (cleanup, *self.cleanups)) + + def emit(self, value: str) -> None: + for cleanup in self.cleanups: + self.writer.print(cleanup) + self.writer.print("p->level--;") + self.writer.print(f"return {value};") + + def check_memory(self, expr: str) -> None: + self.writer.print(f"if ({expr}) {{") + with self.writer.indent(): + self.no_memory() + self.writer.print("}") + + def no_memory(self) -> None: + self.writer.print("p->error_indicator = 1;") + self.writer.print("PyErr_NoMemory();") + self.emit("NULL") + + +class _LoopBuffer: + """Temporary repetition storage and the exits that release it.""" + + _release = "PyMem_Free(_children);" + + def __init__(self, writer: CWriter, returns: _CReturnEmitter): + self._print = writer.print + self._indent = writer.indent + self._returns = returns + self.error_returns = returns.with_cleanup(self._release) + + def initialize(self) -> None: + self._print("void **_children = PyMem_Malloc(sizeof(void *));") + self._returns.check_memory("!_children") + self._print("Py_ssize_t _children_capacity = 1;") + self._print("Py_ssize_t _n = 0;") + + def append(self, value: str) -> None: + self._print("if (_n == _children_capacity) {") + with self._indent(): + self._print("_children_capacity *= 2;") + self._print( + "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" + ) + self._check_memory("!_new_children") + self._print("_children = _new_children;") + self._print("}") + self._print(f"_children[_n++] = {value};") + + def finish(self, *, require_one: bool) -> str: + if require_one: + self._print("if (_n == 0 || p->error_indicator) {") + with self._indent(): + self.error_returns.emit("NULL") + self._print("}") + self._print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") + self._check_memory("!_seq") + self._print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") + self._print(self._release) + return "_seq" + + def _check_memory(self, expr: str) -> None: + self._print(f"if ({expr}) {{") + with self._indent(): + self._print(self._release) + self._returns.no_memory() + self._print("}") + + +class CRuleEmitter: + def __init__(self, writer: CWriter, rule: CRule, *, debug: bool = False): + self._writer = writer + self._print = writer.print + self._indent = writer.indent + self._rule = rule + self._debug = debug + self._returns = _CReturnEmitter(writer) + + def emit(self) -> None: + rule = self._rule + signature = rule.signature + result_type = signature.c_return_type + for line in rule.text.splitlines(): + self._print(f"// {line}") + if rule.left_recursive and rule.leader: + self._print(f"static {result_type} {signature.name}_raw(Parser *);") + self._print(f"static {result_type}") + self._print(f"{signature.name}_rule(Parser *p)") + if rule.left_recursive and rule.leader: + self._emit_left_recursive_wrapper() + self._print("{") + with self._invalid_rule_context(): + match signature.kind: + case RuleKind.LOOP0 | RuleKind.LOOP1: + self._emit_loop_body() + case RuleKind.NORMAL | RuleKind.GATHER: + self._emit_rule_body() + self._print("}") + + @contextmanager + def _invalid_rule_context(self) -> Iterator[None]: + if not self._rule.disable_invalid_rules: + yield + return + with self._indent(): + self._print("int _prev_call_invalid = p->call_invalid_rules;") + self._print("p->call_invalid_rules = 0;") + previous_returns = self._returns + self._returns = previous_returns.with_cleanup( + "p->call_invalid_rules = _prev_call_invalid;" + ) + try: + yield + finally: + self._returns = previous_returns + + def _emit_left_recursive_wrapper(self) -> None: + signature = self._rule.signature + result_type = signature.c_return_type + self._print("{") + with self._indent(): + self._emit_recursion_check() + self._print(f"{result_type} _res = NULL;") + self._print(f"if (_PyPegen_is_memoized(p, {signature.name}_type, &_res)) {{") + with self._indent(): + self._returns.emit("_res") + self._print("}") + self._print("int _mark = p->mark;") + self._print("int _resmark = p->mark;") + self._print(f"Memo *_memo = _PyPegen_insert_memo_direct(p, _mark, {signature.name}_type);") + self._print("if (_memo == NULL) {") + with self._indent(): + self._returns.emit("NULL") + self._print("}") + self._print("while (1) {") + with self._indent(): + self._print("_memo->node = _res;") + self._print("_memo->mark = p->mark;") + self._print("p->mark = _mark;") + self._print(f"void *_raw = {signature.name}_raw(p);") + self._print("if (p->error_indicator) {") + with self._indent(): + self._returns.emit("NULL") + self._print("}") + self._print("if (_raw == NULL || p->mark <= _resmark)") + with self._indent(): + self._print("break;") + self._print("_resmark = p->mark;") + self._print("_res = _raw;") + self._print("}") + self._print("p->mark = _resmark;") + self._returns.emit("_res") + self._print("}") + self._print(f"static {result_type}") + self._print(f"{signature.name}_raw(Parser *p)") + + def _emit_rule_body(self) -> None: + signature = self._rule.signature + memoize = self._rule.memoize + result_type = signature.c_return_type + + with self._indent(): + self._emit_recursion_check() + self._emit_error_check() + self._print(f"{result_type} _res = NULL;") + if memoize: + self._print(f"if (_PyPegen_is_memoized(p, {signature.name}_type, &_res)) {{") + with self._indent(): + self._returns.emit("_res") + self._print("}") + self._print("int _mark = p->mark;") + for prefix in self._rule.prefixes: + self._print(f"{prefix.type} {prefix.result} = NULL;") + self._print(f"int {prefix.end} = 0, {prefix.valid} = 0;") + if self._rule.uses_locations: + self._emit_token_start_metadata() + for alt in self._rule.alternatives: + with self._alternative(alt): + self._emit_normal_alt(alt) + if self._debug: + self._print(f'D(fprintf(stderr, "Fail at %d: {signature.name}\\n", p->mark));') + self._print("_res = NULL;") + self._print(" done:") + with self._indent(): + if memoize: + self._print(f"_PyPegen_insert_memo(p, _mark, {signature.name}_type, _res);") + self._returns.emit("_res") + + def _emit_loop_body(self) -> None: + rule = self._rule + signature = rule.signature + buffer = _LoopBuffer(self._writer, self._returns) + with self._indent(): + self._emit_recursion_check() + self._emit_error_check() + self._print("void *_res = NULL;") + if rule.memoize: + self._print(f"if (_PyPegen_is_memoized(p, {signature.name}_type, &_res)) {{") + with self._indent(): + self._returns.emit("_res") + self._print("}") + self._print("int _mark = p->mark;") + if rule.memoize: + self._print("int _start_mark = p->mark;") + buffer.initialize() + if rule.uses_locations: + self._emit_token_start_metadata() + alt, = rule.alternatives + with self._alternative(alt): + self._emit_loop_alt(alt, buffer) + result = buffer.finish(require_one=signature.kind is RuleKind.LOOP1) + if rule.memoize: + self._print(f"_PyPegen_insert_memo(p, _start_mark, {signature.name}_type, {result});") + self._returns.emit(result) + + @contextmanager + def _alternative(self, alt: CAlternative) -> Iterator[None]: + rulename = self._rule.signature.name + if alt.requires_invalid_rules: + self._print(f"if (p->call_invalid_rules) {{ // {alt.text}") + else: + self._print(f"{{ // {alt.text}") + with self._indent(): + self._emit_error_check() + node_str = alt.text.replace('"', '\\"') + self._print( + f'D(fprintf(stderr, "%*c> {rulename}[%d-%d]: %s\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' + ) + for variable in sorted(alt.variables, key=lambda var: var.name): + ctype = variable.type + " " if variable.type else "void *" + initializer = ( + f" = {variable.initializer}" if variable.initializer is not None else "" + ) + self._print(f"{ctype}{variable.name}{initializer};") + if variable.unused: + self._print(f"UNUSED({variable.name}); // Silence compiler warnings") + + yield + + self._print("p->mark = _mark;") + self._print( + f"D(fprintf(stderr, \"%*c%s {rulename}[%d-%d]: %s failed!\\n\", p->level, ' ',\n" + f' p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "{node_str}"));' + ) + if alt.cut_variable is not None: + self._print(f"if ({alt.cut_variable}) {{") + with self._indent(): + self._returns.emit("NULL") + self._print("}") + self._print("}") + + def _emit_conditions(self, keyword: str, alt: CAlternative) -> None: + self._print(f"{keyword} (") + with self._indent(): + for index, call in enumerate(alt.calls): + if index: + self._print("&&") + self._print(call) + self._print(")") + + def _emit_normal_alt(self, alt: CAlternative) -> None: + rulename = self._rule.signature.name + self._emit_conditions(keyword="if", alt=alt) + self._print("{") + with self._indent(): + node_str = alt.text.replace('"', '\\"') + self._print( + f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' + ) + self._emit_alt_action(alt, self._returns) + + self._print("goto done;") + self._print("}") + + def _emit_loop_alt(self, alt: CAlternative, buffer: _LoopBuffer) -> None: + self._emit_conditions(keyword="while", alt=alt) + self._print("{") + with self._indent(): + self._emit_alt_action(alt, buffer.error_returns) + buffer.append("_res") + self._print("_mark = p->mark;") + self._print("}") + + def _emit_alt_action(self, alt: CAlternative, error_returns: _CReturnEmitter) -> None: + # Location failures and explicit-action failures have distinct cleanup + # paths in the generated parser. Keep their return contexts separate. + if alt.uses_locations: + self._emit_token_end_metadata() + if not alt.action.checked: + self._emit_action_debug(alt) + self._print(f"_res = {alt.action.expression};") + if alt.action.checked: + self._print("if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {") + with self._indent(): + self._print("p->error_indicator = 1;") + error_returns.emit("NULL") + self._print("}") + self._emit_action_debug(alt) + + def _emit_action_debug(self, alt: CAlternative) -> None: + if self._debug and alt.action.debug_message: + self._print( + f'D(fprintf(stderr, "{alt.action.debug_message}\\n", _mark, p->mark, "{alt.text}"));' + ) + + def _emit_token_start_metadata(self) -> None: + self._print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {") + with self._indent(): + self._print("p->error_indicator = 1;") + self._returns.emit("NULL") + self._print("}") + self._print("int _start_lineno = p->tokens[_mark]->lineno;") + self._print("UNUSED(_start_lineno); // Only used by EXTRA macro") + self._print("int _start_col_offset = p->tokens[_mark]->col_offset;") + self._print("UNUSED(_start_col_offset); // Only used by EXTRA macro") + + def _emit_token_end_metadata(self) -> None: + self._print("Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);") + self._print("if (_token == NULL) {") + with self._indent(): + self._returns.emit("NULL") + self._print("}") + self._print("int _end_lineno = _token->end_lineno;") + self._print("UNUSED(_end_lineno); // Only used by EXTRA macro") + self._print("int _end_col_offset = _token->end_col_offset;") + self._print("UNUSED(_end_col_offset); // Only used by EXTRA macro") + + def _emit_error_check(self) -> None: + self._print("if (p->error_indicator) {") + with self._indent(): + self._returns.emit("NULL") + self._print("}") + + def _emit_recursion_check(self) -> None: + self._print("if (p->level++ == MAXSTACK || _PyPegen_stack_exhausted(p)) {") + with self._indent(): + self._print("_Pypegen_stack_overflow(p);") + self._print("}") diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index d3c2eca6615a9fb..f81e64aa5837782 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Iterable, Iterator, Set +from enum import Enum, auto from typing import Any @@ -57,20 +58,36 @@ def __iter__(self) -> Iterator[Rule]: SIMPLE_STR = True +class RuleKind(Enum): + NORMAL = auto() + LOOP0 = auto() + LOOP1 = auto() + GATHER = auto() + + class Rule: - def __init__(self, name: str, type: str | None, rhs: Rhs, flags: frozenset[str] | None = None): + def __init__( + self, + name: str, + type: str | None, + rhs: Rhs, + flags: frozenset[str] | None = None, + *, + kind: RuleKind = RuleKind.NORMAL, + ): self.name = name self.type = type self.rhs = rhs self.flags = flags or frozenset() + self.kind = kind self.left_recursive = False self.leader = False def is_loop(self) -> bool: - return self.name.startswith("_loop") + return self.kind in (RuleKind.LOOP0, RuleKind.LOOP1) def is_gather(self) -> bool: - return self.name.startswith("_gather") + return self.kind is RuleKind.GATHER def __str__(self) -> str: if SIMPLE_STR or self.type is None: @@ -84,22 +101,18 @@ def __str__(self) -> str: return "\n".join(lines) def __repr__(self) -> str: - return f"Rule({self.name!r}, {self.type!r}, {self.rhs!r})" + kind = f", kind=RuleKind.{self.kind.name}" if self.kind is not RuleKind.NORMAL else "" + return f"Rule({self.name!r}, {self.type!r}, {self.rhs!r}{kind})" def __iter__(self) -> Iterator[Rhs]: yield self.rhs def flatten(self) -> Rhs: # If it's a single parenthesized group, flatten it. - rhs = self.rhs - if ( - not self.is_loop() - and len(rhs.alts) == 1 - and len(rhs.alts[0].items) == 1 - and isinstance(rhs.alts[0].items[0].item, Group) - ): - rhs = rhs.alts[0].items[0].item.rhs - return rhs + match self.rhs: + case Rhs(alts=[Alt(items=[NamedItem(item=Group(rhs=rhs))])]) if not self.is_loop(): + return rhs + return self.rhs class Leaf: @@ -147,12 +160,11 @@ def __iter__(self) -> Iterator[list[Alt]]: @property def can_be_inlined(self) -> bool: - if len(self.alts) != 1 or len(self.alts[0].items) != 1: - return False - # If the alternative has an action we cannot inline - if getattr(self.alts[0], "action", None) is not None: - return False - return True + match self.alts: + case [Alt(items=[_], action=None)]: + return True + case _: + return False class Alt: diff --git a/Tools/peg_generator/pegen/grammar_analysis.py b/Tools/peg_generator/pegen/grammar_analysis.py new file mode 100644 index 000000000000000..5b716cf209e000a --- /dev/null +++ b/Tools/peg_generator/pegen/grammar_analysis.py @@ -0,0 +1,155 @@ +"""Nullable and left-recursion analysis for source grammar rules.""" + +from collections.abc import Iterable, Set +from typing import Any + +from pegen import sccutils +from pegen.grammar import ( + Alt, + Cut, + Forced, + Gather, + GrammarVisitor, + Group, + NamedItem, + NameLeaf, + Opt, + Repeat0, + Repeat1, + Rhs, + Rule, + StringLeaf, +) + + +class NullableVisitor(GrammarVisitor): + def __init__(self, rules: dict[str, Rule]) -> None: + self.rules = rules + self.visited: set[Any] = set() + self.nullables: set[Rule | NamedItem] = set() + + def visit(self, node: Any, *args: Any, **kwargs: Any) -> bool | None: + match node: + case Rule(rhs=rhs): + if node in self.visited: + return False + self.visited.add(node) + if self.visit(rhs): + self.nullables.add(node) + return node in self.nullables + case NamedItem(item=item): + if self.visit(item): + self.nullables.add(node) + return node in self.nullables + case Rhs(alts=alts): + return any(self.visit(alt) for alt in alts) + case Alt(items=items): + return all(self.visit(item) for item in items) + case Forced() | Opt() | Repeat0(): + return True + case Repeat1() | Gather() | Cut(): + return False + case Group(rhs=rhs): + return self.visit(rhs) + case NameLeaf(value=name): + if (rule := self.rules.get(name)) is not None: + return self.visit(rule) + # Token or unknown; never empty. + return False + case StringLeaf(value=value): + # The string token '' is considered empty. + return not value + case _: + return self.generic_visit(node, *args, **kwargs) + + +def compute_nullables(rules: dict[str, Rule]) -> set[Any]: + """Compute which rules in a grammar are nullable. + + Thanks to TatSu (tatsu/leftrec.py) for inspiration. + """ + nullable_visitor = NullableVisitor(rules) + for rule in rules.values(): + nullable_visitor.visit(rule) + return nullable_visitor.nullables + + +class InitialNamesVisitor(GrammarVisitor): + def __init__(self, rules: dict[str, Rule]) -> None: + self.rules = rules + self.nullables = compute_nullables(rules) + + def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> set[Any]: + names: set[str] = set() + for value in node: + if isinstance(value, list): + for item in value: + names |= self.visit(item, *args, **kwargs) + else: + names |= self.visit(value, *args, **kwargs) + return names + + def visit(self, node: Any, *args: Any, **kwargs: Any) -> set[Any]: + match node: + case Alt(items=items): + names: set[str] = set() + for item in items: + names |= self.visit(item) + if item not in self.nullables: + break + return names + case Forced() | Cut() | StringLeaf(): + return set() + case NameLeaf(value=name): + return {name} + case _: + return self.generic_visit(node, *args, **kwargs) + + +def compute_left_recursives( + rules: dict[str, Rule] +) -> tuple[dict[str, Set[str]], list[Set[str]]]: + graph = make_first_graph(rules) + sccs = list(sccutils.strongly_connected_components(graph.keys(), graph)) + for scc in sccs: + if len(scc) > 1: + for name in scc: + rules[name].left_recursive = True + # Try to find a leader such that all cycles go through it. + leaders = set(scc) + for start in scc: + for cycle in sccutils.find_cycles_in_scc(graph, scc, start): + # print("Cycle:", " -> ".join(cycle)) + leaders -= scc - set(cycle) + if not leaders: + raise ValueError( + f"SCC {scc} has no leadership candidate (no element is included in all cycles)" + ) + # print("Leaders:", leaders) + leader = min(leaders) # Pick an arbitrary leader from the candidates. + rules[leader].leader = True + else: + name = min(scc) # The only element. + if name in graph[name]: + rules[name].left_recursive = True + rules[name].leader = True + return graph, sccs + + +def make_first_graph(rules: dict[str, Rule]) -> dict[str, Set[str]]: + """Compute the graph of left-invocations. + + There's an edge from A to B if A may invoke B at its initial + position. + + Note that this requires the nullable flags to have been computed. + """ + initial_name_visitor = InitialNamesVisitor(rules) + graph: dict[str, Set[str]] = {} + vertices: set[str] = set() + for rulename, rhs in rules.items(): + graph[rulename] = names = initial_name_visitor.visit(rhs) + vertices |= names + for vertex in vertices: + graph.setdefault(vertex, set()) + return graph diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 81314b0cc073f9c..f7783f1e7810b0e 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -3,30 +3,38 @@ import re import sys from abc import abstractmethod -from collections.abc import Iterable, Iterator, Set -from typing import IO, Any +from collections.abc import Iterator +from typing import IO -from pegen import sccutils from pegen.grammar import ( Alt, - Cut, - Forced, Gather, Grammar, GrammarError, GrammarVisitor, - Group, - Lookahead, NamedItem, NameLeaf, - Opt, Plain, - Repeat0, - Repeat1, Rhs, Rule, + RuleKind, StringLeaf, ) +from pegen.grammar_analysis import ( + InitialNamesVisitor as InitialNamesVisitor, +) +from pegen.grammar_analysis import ( + NullableVisitor as NullableVisitor, +) +from pegen.grammar_analysis import ( + compute_left_recursives as compute_left_recursives, +) +from pegen.grammar_analysis import ( + compute_nullables as compute_nullables, +) +from pegen.grammar_analysis import ( + make_first_graph as make_first_graph, +) class RuleCollectorVisitor(GrammarVisitor): @@ -93,12 +101,7 @@ def __init__(self, grammar: Grammar, tokens: set[str], file: IO[str] | None): self.keywords: dict[str, int] = {} self.soft_keywords: set[str] = set() self.rules = grammar.rules - self.validate_rule_names() - if "trailer" not in grammar.metas and "start" not in self.rules: - raise GrammarError("Grammar without a trailer must have a 'start' rule") - checker = RuleCheckingVisitor(self.rules, self.tokens) - for rule in self.rules.values(): - checker.visit(rule) + self._validate_grammar() self.file = file self.level = 0 self.first_graph, self.first_sccs = compute_left_recursives(self.rules) @@ -107,6 +110,14 @@ def __init__(self, grammar: Grammar, tokens: set[str], file: IO[str] | None): self.all_rules: dict[str, Rule] = self.rules.copy() # Rules + temporal rules self._local_variable_stack: list[list[str]] = [] + def _validate_grammar(self) -> None: + self.validate_rule_names() + if "trailer" not in self.grammar.metas and "start" not in self.rules: + raise GrammarError("Grammar without a trailer must have a 'start' rule") + checker = RuleCheckingVisitor(self.rules, self.tokens) + for rule in self.rules.values(): + checker.visit(rule) + def validate_rule_names(self) -> None: for rule in self.rules: if rule.startswith("_"): @@ -175,10 +186,14 @@ def artificial_rule_from_repeat(self, node: Plain, is_repeat1: bool) -> str: self.counter += 1 if is_repeat1: prefix = "_loop1_" + kind = RuleKind.LOOP1 else: prefix = "_loop0_" + kind = RuleKind.LOOP0 name = f"{prefix}{self.counter}" - self.all_rules[name] = Rule(name, None, Rhs([Alt([NamedItem(None, node)])])) + self.all_rules[name] = Rule( + name, None, Rhs([Alt([NamedItem(None, node)])]), kind=kind + ) return name def artificial_rule_from_gather(self, node: Gather) -> str: @@ -192,6 +207,7 @@ def artificial_rule_from_gather(self, node: Gather) -> str: extra_function_name, None, Rhs([extra_function_alt]), + kind=RuleKind.LOOP0, ) self.counter += 1 name = f"_gather_{self.counter}" @@ -202,6 +218,7 @@ def artificial_rule_from_gather(self, node: Gather) -> str: name, None, Rhs([alt]), + kind=RuleKind.GATHER, ) return name @@ -213,168 +230,3 @@ def dedupe(self, name: str) -> str: name = f"{origname}_{counter}" self.local_variable_names.append(name) return name - - -class NullableVisitor(GrammarVisitor): - def __init__(self, rules: dict[str, Rule]) -> None: - self.rules = rules - self.visited: set[Any] = set() - self.nullables: set[Rule | NamedItem] = set() - - def visit_Rule(self, rule: Rule) -> bool: - if rule in self.visited: - return False - self.visited.add(rule) - if self.visit(rule.rhs): - self.nullables.add(rule) - return rule in self.nullables - - def visit_Rhs(self, rhs: Rhs) -> bool: - for alt in rhs.alts: - if self.visit(alt): - return True - return False - - def visit_Alt(self, alt: Alt) -> bool: - for item in alt.items: - if not self.visit(item): - return False - return True - - def visit_Forced(self, force: Forced) -> bool: - return True - - def visit_LookAhead(self, lookahead: Lookahead) -> bool: - return True - - def visit_Opt(self, opt: Opt) -> bool: - return True - - def visit_Repeat0(self, repeat: Repeat0) -> bool: - return True - - def visit_Repeat1(self, repeat: Repeat1) -> bool: - return False - - def visit_Gather(self, gather: Gather) -> bool: - return False - - def visit_Cut(self, cut: Cut) -> bool: - return False - - def visit_Group(self, group: Group) -> bool: - return self.visit(group.rhs) - - def visit_NamedItem(self, item: NamedItem) -> bool: - if self.visit(item.item): - self.nullables.add(item) - return item in self.nullables - - def visit_NameLeaf(self, node: NameLeaf) -> bool: - if node.value in self.rules: - return self.visit(self.rules[node.value]) - # Token or unknown; never empty. - return False - - def visit_StringLeaf(self, node: StringLeaf) -> bool: - # The string token '' is considered empty. - return not node.value - - -def compute_nullables(rules: dict[str, Rule]) -> set[Any]: - """Compute which rules in a grammar are nullable. - - Thanks to TatSu (tatsu/leftrec.py) for inspiration. - """ - nullable_visitor = NullableVisitor(rules) - for rule in rules.values(): - nullable_visitor.visit(rule) - return nullable_visitor.nullables - - -class InitialNamesVisitor(GrammarVisitor): - def __init__(self, rules: dict[str, Rule]) -> None: - self.rules = rules - self.nullables = compute_nullables(rules) - - def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> set[Any]: - names: set[str] = set() - for value in node: - if isinstance(value, list): - for item in value: - names |= self.visit(item, *args, **kwargs) - else: - names |= self.visit(value, *args, **kwargs) - return names - - def visit_Alt(self, alt: Alt) -> set[Any]: - names: set[str] = set() - for item in alt.items: - names |= self.visit(item) - if item not in self.nullables: - break - return names - - def visit_Forced(self, force: Forced) -> set[Any]: - return set() - - def visit_LookAhead(self, lookahead: Lookahead) -> set[Any]: - return set() - - def visit_Cut(self, cut: Cut) -> set[Any]: - return set() - - def visit_NameLeaf(self, node: NameLeaf) -> set[Any]: - return {node.value} - - def visit_StringLeaf(self, node: StringLeaf) -> set[Any]: - return set() - - -def compute_left_recursives( - rules: dict[str, Rule] -) -> tuple[dict[str, Set[str]], list[Set[str]]]: - graph = make_first_graph(rules) - sccs = list(sccutils.strongly_connected_components(graph.keys(), graph)) - for scc in sccs: - if len(scc) > 1: - for name in scc: - rules[name].left_recursive = True - # Try to find a leader such that all cycles go through it. - leaders = set(scc) - for start in scc: - for cycle in sccutils.find_cycles_in_scc(graph, scc, start): - # print("Cycle:", " -> ".join(cycle)) - leaders -= scc - set(cycle) - if not leaders: - raise ValueError( - f"SCC {scc} has no leadership candidate (no element is included in all cycles)" - ) - # print("Leaders:", leaders) - leader = min(leaders) # Pick an arbitrary leader from the candidates. - rules[leader].leader = True - else: - name = min(scc) # The only element. - if name in graph[name]: - rules[name].left_recursive = True - rules[name].leader = True - return graph, sccs - - -def make_first_graph(rules: dict[str, Rule]) -> dict[str, Set[str]]: - """Compute the graph of left-invocations. - - There's an edge from A to B if A may invoke B at its initial - position. - - Note that this requires the nullable flags to have been computed. - """ - initial_name_visitor = InitialNamesVisitor(rules) - graph = {} - vertices: set[str] = set() - for rulename, rhs in rules.items(): - graph[rulename] = names = initial_name_visitor.visit(rhs) - vertices |= names - for vertex in vertices: - graph.setdefault(vertex, set()) - return graph