From 5b3eb2f608bb390d7b838fd76675c640891d6387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Thu, 23 May 2019 22:18:19 +0300 Subject: [PATCH 1/7] bpo-20443: make co_filename absolute --- Lib/test/test_code.py | 7 +++++++ Lib/test/test_trace.py | 2 +- Lib/test/test_warnings/__init__.py | 4 ++-- Python/compile.c | 3 +++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 9bf290d8d5a1f7..351250411766ce 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -131,6 +131,7 @@ import unittest import weakref import opcode +import os try: import ctypes except ImportError: @@ -212,6 +213,12 @@ class List(list): obj = List([1, 2, 3]) self.assertEqual(obj[0], "Foreign getitem: 1") + @cpython_only + def test_filename_abspath(self): + def x(): + pass + self.assertEqual(x.__code__.co_filename, os.path.abspath(__file__)) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index afe79026766192..d6c7ebf28c7fa9 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -522,7 +522,7 @@ def f(): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {TESTFN} ({filename})', stdout) + self.assertIn(f'6 100% {TESTFN} ({os.path.abspath(filename)})', stdout) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 86c2f226ebcf1f..167b4176e1a1ca 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -932,7 +932,7 @@ def run(*args): f = None ResourceWarning: Enable tracemalloc to get the object allocation traceback ''') - expected = expected.format(fname=support.TESTFN).strip() + expected = expected.format(fname=os.path.abspath(support.TESTFN)).strip() self.assertEqual(stderr, expected) # tracemalloc enabled @@ -946,7 +946,7 @@ def run(*args): File "{fname}", lineno 3 f = open(__file__) ''') - expected = expected.format(fname=support.TESTFN).strip() + expected = expected.format(fname=os.path.abspath(support.TESTFN)).strip() self.assertEqual(stderr, expected) diff --git a/Python/compile.c b/Python/compile.c index b20548c777246c..fc125bbbd523c3 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -325,6 +325,9 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, } if (!compiler_init(&c)) return NULL; + char* path = realpath(PyUnicode_DATA(filename), NULL); + if (path != NULL) + filename = PyUnicode_FromString(path); Py_INCREF(filename); c.c_filename = filename; c.c_arena = arena; From 60008f18ca2de984f6ccec9256a732613bc87a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Thu, 23 May 2019 22:22:08 +0300 Subject: [PATCH 2/7] add news entry --- .../Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst new file mode 100644 index 00000000000000..f75b742d120200 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst @@ -0,0 +1,2 @@ +Now code objects ``co_filename`` attribute is always an absolute path -- by +Batuhan Taskaya From d090f40eb1ea913f14e9557a88375a338c48809c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Thu, 23 May 2019 22:35:52 +0300 Subject: [PATCH 3/7] add realpath available check --- Python/compile.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index fc125bbbd523c3..36d1d3289be289 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -325,9 +325,14 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, } if (!compiler_init(&c)) return NULL; + + #if defined(HAVE_REALPATH) char* path = realpath(PyUnicode_DATA(filename), NULL); - if (path != NULL) + if (path != NULL) { filename = PyUnicode_FromString(path); + } + #endif + Py_INCREF(filename); c.c_filename = filename; c.c_arena = arena; From 6dbc4f57ebb557ff914e5cf3058ca5d3b2fe3d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Thu, 23 May 2019 23:26:09 +0300 Subject: [PATCH 4/7] windows support? --- Python/compile.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 36d1d3289be289..7545dbbf98004a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -22,7 +22,7 @@ */ #include "Python.h" - +#include "osdefs.h" #include "Python-ast.h" #include "pycore_pystate.h" /* _PyInterpreterState_GET_UNSAFE() */ #include "ast.h" @@ -30,6 +30,7 @@ #include "symtable.h" #include "opcode.h" #include "wordcode_helpers.h" +#include #define DEFAULT_BLOCK_SIZE 16 #define DEFAULT_BLOCKS 8 @@ -326,13 +327,18 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, if (!compiler_init(&c)) return NULL; + wchar_t *path = PyUnicode_AsWideCharString(filename, NULL); #if defined(HAVE_REALPATH) - char* path = realpath(PyUnicode_DATA(filename), NULL); - if (path != NULL) { - filename = PyUnicode_FromString(path); - } + wchar_t fullpath[MAXPATHLEN]; + path = _Py_wrealpath(path, fullpath, Py_ARRAY_LENGTH(fullpath)); + #elif defined(MS_WINDOWS) + wchar_t fullpath[MAX_PATH]; + wchar_t *ptemp; + if (GetFullPathNameW(path, Py_ARRAY_LENGTH(fullpath), fullpath, &ptemp)) + path = fullpath; #endif - + if (path != NULL) + filename = PyUnicode_FromWideChar(path, -1); Py_INCREF(filename); c.c_filename = filename; c.c_arena = arena; From 62107b3b3e0ee78959920e73d6ce392c9140bdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Fri, 24 May 2019 19:12:09 +0300 Subject: [PATCH 5/7] Add future flag --- Doc/library/__future__.rst | 4 ++++ Include/code.h | 1 + Include/compile.h | 5 ++++- Lib/__future__.py | 6 ++++++ Lib/test/test_code.py | 5 ----- Lib/test/test_future.py | 4 ++++ Lib/test/test_future6.py | 14 ++++++++++++++ Lib/test/test_trace.py | 2 +- Lib/test/test_warnings/__init__.py | 4 ++-- Python/compile.c | 31 ++++++++++++++++-------------- Python/future.c | 2 ++ 11 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 Lib/test/test_future6.py diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index e3d749e6017847..b6de928db1c0c4 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -93,6 +93,10 @@ language using this mechanism: | annotations | 3.7.0b1 | 4.0 | :pep:`563`: | | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ ++------------------+-------------+--------------+---------------------------------------------+ +| absolute_codepath| 3.8.0b1 | 4.0 | :pep:`XXX`: | +| | | | *Absolute codepaths* | ++------------------+-------------+--------------+---------------------------------------------+ .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. diff --git a/Include/code.h b/Include/code.h index 933de97d078282..d916cb0e3bb696 100644 --- a/Include/code.h +++ b/Include/code.h @@ -84,6 +84,7 @@ typedef struct { #define CO_FUTURE_BARRY_AS_BDFL 0x40000 #define CO_FUTURE_GENERATOR_STOP 0x80000 #define CO_FUTURE_ANNOTATIONS 0x100000 +#define CO_FUTURE_ABSOLUTE_CODEPATH 0x200000 /* This value is found in the co_cell2arg array when the associated cell variable does not correspond to an argument. */ diff --git a/Include/compile.h b/Include/compile.h index 13708678675f7b..a60fac7c1e420e 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -16,7 +16,9 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ - CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) + CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS | \ + CO_FUTURE_ABSOLUTE_CODEPATH) + #define PyCF_MASK_OBSOLETE (CO_NESTED) #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 @@ -48,6 +50,7 @@ typedef struct { #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" #define FUTURE_GENERATOR_STOP "generator_stop" #define FUTURE_ANNOTATIONS "annotations" +#define FUTURE_ABSOLUTE_CODEPATH "absolute_codepath" struct _mod; /* Declare the existence of this type */ #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) diff --git a/Lib/__future__.py b/Lib/__future__.py index e1135685d846ce..d09072e8bd7f75 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -58,6 +58,7 @@ "barry_as_FLUFL", "generator_stop", "annotations", + "absolute_codepath", ] __all__ = ["all_feature_names"] + all_feature_names @@ -76,6 +77,7 @@ CO_FUTURE_BARRY_AS_BDFL = 0x40000 CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime +CO_FUTURE_ABSOLUTE_CODEPATH = 0x200000 class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): @@ -144,3 +146,7 @@ def __repr__(self): annotations = _Feature((3, 7, 0, "beta", 1), (4, 0, 0, "alpha", 0), CO_FUTURE_ANNOTATIONS) + +absolute_codepath = _Feature((3, 8, 0, "beta", 1), + (4, 0, 0, "alpha", 0), + CO_FUTURE_ABSOLUTE_CODEPATH) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 351250411766ce..a230cf5ec5b61c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -213,11 +213,6 @@ class List(list): obj = List([1, 2, 3]) self.assertEqual(obj[0], "Foreign getitem: 1") - @cpython_only - def test_filename_abspath(self): - def x(): - pass - self.assertEqual(x.__code__.co_filename, os.path.abspath(__file__)) def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index dd148b62956298..a16e1fe0cc320b 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -103,6 +103,10 @@ def test_unicode_literals_exec(self): exec("from __future__ import unicode_literals; x = ''", {}, scope) self.assertIsInstance(scope["x"], str) + def test_future5(self): + with support.CleanImport('test_future6'): + from test import test_future6 + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Lib/test/test_future6.py b/Lib/test/test_future6.py new file mode 100644 index 00000000000000..2e19f1f261bfb6 --- /dev/null +++ b/Lib/test/test_future6.py @@ -0,0 +1,14 @@ +from __future__ import absolute_codepath +import os +import unittest + +class Tests(unittest.TestCase): + def test_absolute_codepath(self): + def fn(): + pass + filename = fn.__code__.co_filename + self.assertEqual(filename, os.path.abspath(filename)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index d6c7ebf28c7fa9..afe79026766192 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -522,7 +522,7 @@ def f(): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {TESTFN} ({os.path.abspath(filename)})', stdout) + self.assertIn(f'6 100% {TESTFN} ({filename})', stdout) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 167b4176e1a1ca..86c2f226ebcf1f 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -932,7 +932,7 @@ def run(*args): f = None ResourceWarning: Enable tracemalloc to get the object allocation traceback ''') - expected = expected.format(fname=os.path.abspath(support.TESTFN)).strip() + expected = expected.format(fname=support.TESTFN).strip() self.assertEqual(stderr, expected) # tracemalloc enabled @@ -946,7 +946,7 @@ def run(*args): File "{fname}", lineno 3 f = open(__file__) ''') - expected = expected.format(fname=os.path.abspath(support.TESTFN)).strip() + expected = expected.format(fname=support.TESTFN).strip() self.assertEqual(stderr, expected) diff --git a/Python/compile.c b/Python/compile.c index 7545dbbf98004a..d77aeb7f796153 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -327,20 +327,6 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, if (!compiler_init(&c)) return NULL; - wchar_t *path = PyUnicode_AsWideCharString(filename, NULL); - #if defined(HAVE_REALPATH) - wchar_t fullpath[MAXPATHLEN]; - path = _Py_wrealpath(path, fullpath, Py_ARRAY_LENGTH(fullpath)); - #elif defined(MS_WINDOWS) - wchar_t fullpath[MAX_PATH]; - wchar_t *ptemp; - if (GetFullPathNameW(path, Py_ARRAY_LENGTH(fullpath), fullpath, &ptemp)) - path = fullpath; - #endif - if (path != NULL) - filename = PyUnicode_FromWideChar(path, -1); - Py_INCREF(filename); - c.c_filename = filename; c.c_arena = arena; c.c_future = PyFuture_FromASTObject(mod, filename); if (c.c_future == NULL) @@ -368,6 +354,23 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, goto finally; } + if (c.c_future->ff_features & CO_FUTURE_ABSOLUTE_CODEPATH) { + wchar_t *path = PyUnicode_AsWideCharString(filename, NULL); + #if defined(HAVE_REALPATH) + wchar_t fullpath[MAXPATHLEN]; + path = _Py_wrealpath(path, fullpath, Py_ARRAY_LENGTH(fullpath)); + #elif defined(MS_WINDOWS) + wchar_t fullpath[MAX_PATH]; + wchar_t *ptemp; + if (GetFullPathNameW(path, Py_ARRAY_LENGTH(fullpath), fullpath, &ptemp)) + path = fullpath; + #endif + if (path != NULL) + filename = PyUnicode_FromWideChar(path, -1); + } + Py_INCREF(filename); + c.c_filename = filename; + co = compiler_mod(&c, mod); finally: diff --git a/Python/future.c b/Python/future.c index 1663a38a6fdb3a..3dcaa38ab66ef2 100644 --- a/Python/future.c +++ b/Python/future.c @@ -45,6 +45,8 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) continue; } else if (strcmp(feature, FUTURE_ANNOTATIONS) == 0) { ff->ff_features |= CO_FUTURE_ANNOTATIONS; + } else if (strcmp(feature, FUTURE_ABSOLUTE_CODEPATH) == 0) { + ff->ff_features |= CO_FUTURE_ABSOLUTE_CODEPATH; } else if (strcmp(feature, "braces") == 0) { PyErr_SetString(PyExc_SyntaxError, "not a chance"); From b8be88b1eeb14c12d8e8974d54d9ec24f3373ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Fri, 24 May 2019 19:15:21 +0300 Subject: [PATCH 6/7] update blurb entry --- .../Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst index f75b742d120200..3a458750964102 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-23-22-22-02.bpo-20443.wXFKFh.rst @@ -1,2 +1,2 @@ -Now code objects ``co_filename`` attribute is always an absolute path -- by +Now code objects ``co_filename`` attribute is always an absolute path if a ``absolute_codepath`` future flag specified -- by Batuhan Taskaya From ac6cfed0d185f0c2544a05220be75e89fc2df110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Fri, 24 May 2019 19:17:10 +0300 Subject: [PATCH 7/7] add a pseudo pep number for test pass --- Doc/library/__future__.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index b6de928db1c0c4..c10c022aa24787 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -94,7 +94,7 @@ language using this mechanism: | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ +------------------+-------------+--------------+---------------------------------------------+ -| absolute_codepath| 3.8.0b1 | 4.0 | :pep:`XXX`: | +| absolute_codepath| 3.8.0b1 | 4.0 | :pep:`9999`: | | | | | *Absolute codepaths* | +------------------+-------------+--------------+---------------------------------------------+