Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

gh-92651: Remove the Include/token.h header file #92652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
May 11, 2022
Merged

gh-92651: Remove the Include/token.h header file #92652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
12fc168
gh-92651: Remove the Include/token.h header file
vstinner May 11, 2022
e6273d2
Update make_h() default value
vstinner May 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions 5 Doc/whatsnew/3.12.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,8 @@ Deprecated

Removed
-------

* Remove the ``token.h`` header file. There was never any public tokenizer C
API. The ``token.h`` header file was only designed to be used by Python
internals.
(Contributed by Victor Stinner in :gh:`92651`.)
19 changes: 11 additions & 8 deletions 19 Include/token.h → Include/internal/pycore_token.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
/* Auto-generated by Tools/scripts/generate_token.py */

/* Token types */
#ifndef Py_LIMITED_API
#ifndef Py_TOKEN_H
#define Py_TOKEN_H
#ifndef Py_INTERNAL_TOKEN_H
#define Py_INTERNAL_TOKEN_H
#ifdef __cplusplus
extern "C" {
#endif

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */

#define ENDMARKER 0
Expand Down Expand Up @@ -85,13 +88,13 @@ extern "C" {
(x) == DEDENT)


// Symbols exported for test_peg_generator
PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
PyAPI_FUNC(int) PyToken_OneChar(int);
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
PyAPI_FUNC(int) _PyToken_OneChar(int);
PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);

#ifdef __cplusplus
}
#endif
#endif /* !Py_TOKEN_H */
#endif /* Py_LIMITED_API */
#endif // !Py_INTERNAL_TOKEN_H
6 changes: 3 additions & 3 deletions 6 Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1325,11 +1325,11 @@ regen-token:
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \
$(srcdir)/Grammar/Tokens \
$(srcdir)/Doc/library/token-list.inc
# Regenerate Include/token.h from Grammar/Tokens
# Regenerate Include/internal/pycore_token.h from Grammar/Tokens
# using Tools/scripts/generate_token.py
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \
$(srcdir)/Grammar/Tokens \
$(srcdir)/Include/token.h
$(srcdir)/Include/internal/pycore_token.h
# Regenerate Parser/token.c from Grammar/Tokens
# using Tools/scripts/generate_token.py
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \
Expand Down Expand Up @@ -1521,7 +1521,6 @@ PYTHON_HEADERS= \
$(srcdir)/Include/structmember.h \
$(srcdir)/Include/structseq.h \
$(srcdir)/Include/sysmodule.h \
$(srcdir)/Include/token.h \
$(srcdir)/Include/traceback.h \
$(srcdir)/Include/tracemalloc.h \
$(srcdir)/Include/tupleobject.h \
Expand Down Expand Up @@ -1632,6 +1631,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_structseq.h \
$(srcdir)/Include/internal/pycore_symtable.h \
$(srcdir)/Include/internal/pycore_sysmodule.h \
$(srcdir)/Include/internal/pycore_token.h \
$(srcdir)/Include/internal/pycore_traceback.h \
$(srcdir)/Include/internal/pycore_tuple.h \
$(srcdir)/Include/internal/pycore_typeobject.h \
Expand Down
3 changes: 3 additions & 0 deletions 3 Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Remove the ``token.h`` header file. There was never any public tokenizer C
API. The ``token.h`` header file was only designed to be used by Python
internals. Patch by Victor Stinner.
2 changes: 1 addition & 1 deletion 2 PCbuild/pythoncore.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@
<ClInclude Include="..\Include\internal\pycore_structseq.h" />
<ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
<ClInclude Include="..\Include\internal\pycore_symtable.h" />
<ClInclude Include="..\Include\internal\pycore_token.h" />
<ClInclude Include="..\Include\internal\pycore_traceback.h" />
<ClInclude Include="..\Include\internal\pycore_tuple.h" />
<ClInclude Include="..\Include\internal\pycore_typeobject.h" />
Expand Down Expand Up @@ -291,7 +292,6 @@
<ClInclude Include="..\Include\structseq.h" />
<ClInclude Include="..\Include\symtable.h" />
<ClInclude Include="..\Include\sysmodule.h" />
<ClInclude Include="..\Include\token.h" />
<ClInclude Include="..\Include\traceback.h" />
<ClInclude Include="..\Include\tracemalloc.h" />
<ClInclude Include="..\Include\tupleobject.h" />
Expand Down
6 changes: 3 additions & 3 deletions 6 PCbuild/pythoncore.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,9 +213,6 @@
<ClInclude Include="..\Include\sysmodule.h">
<Filter>Include</Filter>
</ClInclude>
<ClInclude Include="..\Include\token.h">
<Filter>Include</Filter>
</ClInclude>
<ClInclude Include="..\Include\traceback.h">
<Filter>Include</Filter>
</ClInclude>
Expand Down Expand Up @@ -633,6 +630,9 @@
<ClInclude Include="..\Include\internal\pycore_symtable.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_token.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_traceback.h">
<Filter>Include\internal</Filter>
</ClInclude>
Expand Down
2 changes: 1 addition & 1 deletion 2 PCbuild/regen.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
<Format>rst</Format>
</_TokenOutputs>
<_TokenOutputs Include="$(PySourcePath)Include\token.h">
<_TokenOutputs Include="$(PySourcePath)Include\internal\pycore_token.h">
<Format>h</Format>
</_TokenOutputs>
<_TokenOutputs Include="$(PySourcePath)Parser\token.c">
Expand Down
2 changes: 1 addition & 1 deletion 2 Parser/pegen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <token.h>
#include <pycore_ast.h>
#include <pycore_token.h>

#if 0
#define PyPARSE_YIELD_IS_KEYWORD 0x0001
Expand Down
8 changes: 4 additions & 4 deletions 8 Parser/token.c
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions 6 Parser/tokenizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1992,10 +1992,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
/* Check for two-character token */
{
int c2 = tok_nextc(tok);
int token = PyToken_TwoChars(c, c2);
int token = _PyToken_TwoChars(c, c2);
if (token != OP) {
int c3 = tok_nextc(tok);
int token3 = PyToken_ThreeChars(c, c2, c3);
int token3 = _PyToken_ThreeChars(c, c2, c3);
if (token3 != OP) {
token = token3;
}
Expand Down Expand Up @@ -2059,7 +2059,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
/* Punctuation character */
*p_start = tok->start;
*p_end = tok->cur;
return PyToken_OneChar(c);
return _PyToken_OneChar(c);
}

int
Expand Down
2 changes: 1 addition & 1 deletion 2 Parser/tokenizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ extern "C" {

/* Tokenizer interface */

#include "token.h" /* For token types */
#include "pycore_token.h" /* For token types */

#define MAXINDENT 100 /* Max indentation level */
#define MAXLEVEL 200 /* Max parentheses level */
Expand Down
1 change: 0 additions & 1 deletion 1 Python/pythonrun.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
#include "pycore_sysmodule.h" // _PySys_Audit()
#include "pycore_traceback.h" // _PyTraceBack_Print_Indented()

#include "token.h" // INDENT
#include "errcode.h" // E_EOF
#include "marshal.h" // PyMarshal_ReadLongFromFile()

Expand Down
29 changes: 16 additions & 13 deletions 29 Tools/scripts/generate_token.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,16 @@ def update_file(file, content):
/* Auto-generated by Tools/scripts/generate_token.py */

/* Token types */
#ifndef Py_LIMITED_API
#ifndef Py_TOKEN_H
#define Py_TOKEN_H
#ifndef Py_INTERNAL_TOKEN_H
#define Py_INTERNAL_TOKEN_H
#ifdef __cplusplus
extern "C" {
#endif

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */

%s\
Expand All @@ -75,19 +78,19 @@ def update_file(file, content):
(x) == DEDENT)


// Symbols exported for test_peg_generator
PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
PyAPI_FUNC(int) PyToken_OneChar(int);
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
PyAPI_FUNC(int) _PyToken_OneChar(int);
PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);

#ifdef __cplusplus
}
#endif
#endif /* !Py_TOKEN_H */
#endif /* Py_LIMITED_API */
#endif // !Py_INTERNAL_TOKEN_H
"""

def make_h(infile, outfile='Include/token.h'):
def make_h(infile, outfile='Include/internal/pycore_token.h'):
tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)

defines = []
Expand All @@ -106,7 +109,7 @@ def make_h(infile, outfile='Include/token.h'):
/* Auto-generated by Tools/scripts/generate_token.py */

#include "Python.h"
#include "token.h"
#include "pycore_token.h"

/* Token names */

Expand All @@ -117,21 +120,21 @@ def make_h(infile, outfile='Include/token.h'):
/* Return the token corresponding to a single character */

int
PyToken_OneChar(int c1)
_PyToken_OneChar(int c1)
{
%s\
return OP;
}

int
PyToken_TwoChars(int c1, int c2)
_PyToken_TwoChars(int c1, int c2)
{
%s\
return OP;
}

int
PyToken_ThreeChars(int c1, int c2, int c3)
_PyToken_ThreeChars(int c1, int c2, int c3)
{
%s\
return OP;
Expand Down
Morty Proxy This is a proxified and sanitized view of the page, visit original site.