From aee1fe61bf4df0596acaeb6709987a1ca2dd38f6 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 21 May 2018 13:26:28 -0400 Subject: [PATCH] Test in a minimal Python configuration This removes most of the optional Python modules and makes sure basic matplotlib functionality still works. --- .travis.yml | 12 + .../sphinxext/tests/test_tinypages.py | 14 + tools/Setup.dist | 392 ++++++++++++++++++ tools/create_minimal_python.py | 73 ++++ 4 files changed, 491 insertions(+) create mode 100644 tools/Setup.dist create mode 100644 tools/create_minimal_python.py diff --git a/.travis.yml b/.travis.yml index e84d9dd02a4d..33d034b3af39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,6 +72,7 @@ env: - PYTHONFAULTHANDLER=1 - PYTEST_ADDOPTS="-rawR --maxfail=50 --timeout=300 --durations=25 --cov-report= --cov=lib -n $NPROC" - RUN_PEP8= + - MINIMAL_PYTHON= matrix: include: @@ -87,6 +88,11 @@ matrix: - PYTEST=pytest==3.4 - PYTEST_COV=pytest-cov==2.3.1 - SPHINX=sphinx==1.3 + # Build a minimal Python configuration where the optional modules have + # been removed. The file in tools/Setup.dist should be updated from the + # CPython repository when the version of Python being tested has + # changed. + - MINIMAL_PYTHON=true - python: 3.5 env: # - PYTHONOPTIMIZE=2 # This currently doesn't work. @@ -182,6 +188,12 @@ install: - | # Install matplotlib python -mpip install -ve . + - | + # If this is meant to be a minimal python, remove optional modules now + if [[ $MINIMAL_PYTHON == 'true' ]] then + python tools/create_minimal_python.py + export PATH=$PWD/minimal_python/bin:$PATH + fi before_script: | if [[ $TRAVIS_OS_NAME != 'osx' ]]; then diff --git a/lib/matplotlib/sphinxext/tests/test_tinypages.py b/lib/matplotlib/sphinxext/tests/test_tinypages.py index 9ec300894760..3cb2225d2084 100644 --- a/lib/matplotlib/sphinxext/tests/test_tinypages.py +++ b/lib/matplotlib/sphinxext/tests/test_tinypages.py @@ -15,6 +15,20 @@ reason="'{} -msphinx' does not return 0".format(sys.executable)) +try: + import multiprocessing + HAS_MULTIPROCESSING = True +except ImportError: + HAS_MULTIPROCESSING = False + + +needs_multiprocessing = pytest.mark.skipif( + not HAS_MULTIPROCESSING, + reason="needs multiprocessing") + + +@needs_sphinx +@needs_multiprocessing def test_tinypages(tmpdir): html_dir = pjoin(str(tmpdir), 'html') doctree_dir = pjoin(str(tmpdir), 'doctrees') diff --git a/tools/Setup.dist b/tools/Setup.dist new file mode 100644 index 000000000000..06ba6adf24e3 --- /dev/null +++ b/tools/Setup.dist @@ -0,0 +1,392 @@ +# -*- makefile -*- +# The file Setup is used by the makesetup script to construct the files +# Makefile and config.c, from Makefile.pre and config.c.in, +# respectively. The file Setup itself is initially copied from +# Setup.dist; once it exists it will not be overwritten, so you can edit +# Setup to your heart's content. Note that Makefile.pre is created +# from Makefile.pre.in by the toplevel configure script. + +# (VPATH notes: Setup and Makefile.pre are in the build directory, as +# are Makefile and config.c; the *.in and *.dist files are in the source +# directory.) + +# Each line in this file describes one or more optional modules. +# Modules enabled here will not be compiled by the setup.py script, +# so the file can be used to override setup.py's behavior. + +# Lines have the following structure: +# +# ... [ ...] [ ...] [ ...] +# +# is anything ending in .c (.C, .cc, .c++ are C++ files) +# is anything starting with -I, -D, -U or -C +# is anything ending in .a or beginning with -l or -L +# is anything else but should be a valid Python +# identifier (letters, digits, underscores, beginning with non-digit) +# +# (As the makesetup script changes, it may recognize some other +# arguments as well, e.g. *.so and *.sl as libraries. See the big +# case statement in the makesetup script.) +# +# Lines can also have the form +# +# = +# +# which defines a Make variable definition inserted into Makefile.in +# +# Finally, if a line contains just the word "*shared*" (without the +# quotes but with the stars), then the following modules will not be +# built statically. The build process works like this: +# +# 1. Build all modules that are declared as static in Modules/Setup, +# combine them into libpythonxy.a, combine that into python. +# 2. Build all modules that are listed as shared in Modules/Setup. +# 3. Invoke setup.py. That builds all modules that +# a) are not builtin, and +# b) are not listed in Modules/Setup, and +# c) can be build on the target +# +# Therefore, modules declared to be shared will not be +# included in the config.c file, nor in the list of objects to be +# added to the library archive, and their linker options won't be +# added to the linker options. Rules to create their .o files and +# their shared libraries will still be added to the Makefile, and +# their names will be collected in the Make variable SHAREDMODS. This +# is used to build modules as shared libraries. (They can be +# installed using "make sharedinstall", which is implied by the +# toplevel "make install" target.) (For compatibility, +# *noconfig* has the same effect as *shared*.) +# +# In addition, *static* explicitly declares the following modules to +# be static. Lines containing "*static*" and "*shared*" may thus +# alternate throughout this file. + +# NOTE: As a standard policy, as many modules as can be supported by a +# platform should be present. The distribution comes with all modules +# enabled that are supported by most platforms and don't require you +# to ftp sources from elsewhere. + + +# Some special rules to define PYTHONPATH. +# Edit the definitions below to indicate which options you are using. +# Don't add any whitespace or comments! + +# Directories where library files get installed. +# DESTLIB is for Python modules; MACHDESTLIB for shared libraries. +DESTLIB=$(LIBDEST) +MACHDESTLIB=$(BINLIBDEST) + +# NOTE: all the paths are now relative to the prefix that is computed +# at run time! + +# Standard path -- don't edit. +# No leading colon since this is the first entry. +# Empty since this is now just the runtime prefix. +DESTPATH= + +# Site specific path components -- should begin with : if non-empty +SITEPATH= + +# Standard path components for test modules +TESTPATH= + +# Path components for machine- or system-dependent modules and shared libraries +MACHDEPPATH=:$(PLATDIR) +EXTRAMACHDEPPATH= + +COREPYTHONPATH=$(DESTPATH)$(SITEPATH)$(TESTPATH)$(MACHDEPPATH)$(EXTRAMACHDEPPATH) +PYTHONPATH=$(COREPYTHONPATH) + + +# The modules listed here can't be built as shared libraries for +# various reasons; therefore they are listed here instead of in the +# normal order. + +# This only contains the minimal set of modules required to run the +# setup.py script in the root of the Python source tree. + +posix posixmodule.c # posix (UNIX) system calls +errno errnomodule.c # posix (UNIX) errno values +pwd pwdmodule.c # this is needed to find out the user's home dir + # if $HOME is not set +_sre _sre.c # Fredrik Lundh's new regular expressions +_codecs _codecsmodule.c # access to the builtin codecs and codec registry +_weakref _weakref.c # weak references +_functools _functoolsmodule.c # Tools for working with functions and callable objects +_operator _operator.c # operator.add() and similar goodies +_collections _collectionsmodule.c # Container types +itertools itertoolsmodule.c # Functions creating iterators for efficient looping +atexit atexitmodule.c # Register functions to be run at interpreter-shutdown +_stat _stat.c # stat.h interface +time timemodule.c # -lm # time operations and variables + +# access to ISO C locale support +_locale _localemodule.c # -lintl + +# Standard I/O baseline +_io -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c + +# The zipimport module is always imported at startup. Having it as a +# builtin module avoids some bootstrapping problems and reduces overhead. +zipimport zipimport.c + +# faulthandler module +faulthandler faulthandler.c + +# debug tool to trace memory blocks allocated by Python +_tracemalloc _tracemalloc.c hashtable.c + +# The rest of the modules listed in this file are all commented out by +# default. Usually they can be detected and built as dynamically +# loaded modules by the new setup.py script added in Python 2.1. If +# you're on a platform that doesn't support dynamic loading, want to +# compile modules statically into the Python binary, or need to +# specify some odd set of compiler switches, you can uncomment the +# appropriate lines below. + +# ====================================================================== + +# The Python symtable module depends on .h files that setup.py doesn't track +_symtable symtablemodule.c + +# Uncommenting the following line tells makesetup that all following +# modules are to be built as shared libraries (see above for more +# detail; also note that *static* reverses this effect): + +#*shared* + +# GNU readline. Unlike previous Python incarnations, GNU readline is +# now incorporated in an optional module, configured in the Setup file +# instead of by a configure script switch. You may have to insert a +# -L option pointing to the directory where libreadline.* lives, +# and you may have to change -ltermcap to -ltermlib or perhaps remove +# it, depending on your system -- see the GNU readline instructions. +# It's okay for this to be a shared library, too. + +#readline readline.c -lreadline -ltermcap + + +# Modules that should always be present (non UNIX dependent): + +#array arraymodule.c # array objects +#cmath cmathmodule.c _math.c # -lm # complex math library functions +#math mathmodule.c _math.c # -lm # math library functions, e.g. sin() +#_struct _struct.c # binary structure packing/unpacking +#_weakref _weakref.c # basic weak reference support +#_testcapi _testcapimodule.c # Python C API test module +#_random _randommodule.c # Random number generator +#_elementtree -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI _elementtree.c # elementtree accelerator +#_pickle _pickle.c # pickle accelerator +#_datetime _datetimemodule.c # datetime accelerator +#_bisect _bisectmodule.c # Bisection algorithms +#_heapq _heapqmodule.c # Heap queue algorithm + +#unicodedata unicodedata.c # static Unicode character database + + +# Modules with some UNIX dependencies -- on by default: +# (If you have a really backward UNIX, select and socket may not be +# supported...) + +#fcntl fcntlmodule.c # fcntl(2) and ioctl(2) +#spwd spwdmodule.c # spwd(3) +#grp grpmodule.c # grp(3) +#select selectmodule.c # select(2); not on ancient System V + +# Memory-mapped files (also works on Win32). +#mmap mmapmodule.c + +# CSV file helper +#_csv _csv.c + +# Socket module helper for socket(2) +#_socket socketmodule.c + +# Socket module helper for SSL support; you must comment out the other +# socket line above, and possibly edit the SSL variable: +#SSL=/usr/local/ssl +#_ssl _ssl.c \ +# -DUSE_SSL -I$(SSL)/include -I$(SSL)/include/openssl \ +# -L$(SSL)/lib -lssl -lcrypto + +# The crypt module is now disabled by default because it breaks builds +# on many systems (where -lcrypt is needed), e.g. Linux (I believe). +# +# First, look at Setup.config; configure may have set this for you. + +#_crypt _cryptmodule.c # -lcrypt # crypt(3); needs -lcrypt on some systems + + +# Some more UNIX dependent modules -- off by default, since these +# are not supported by all UNIX systems: + +#nis nismodule.c -lnsl # Sun yellow pages -- not everywhere +#termios termios.c # Steen Lumholt's termios module +#resource resource.c # Jeremy Hylton's rlimit interface + +#_posixsubprocess _posixsubprocess.c # POSIX subprocess module helper + +# Multimedia modules -- off by default. +# These don't work for 64-bit platforms!!! +# #993173 says audioop works on 64-bit platforms, though. +# These represent audio samples or images as strings: + +#audioop audioop.c # Operations on audio samples + + +# Note that the _md5 and _sha modules are normally only built if the +# system does not have the OpenSSL libs containing an optimized version. + +# The _md5 module implements the RSA Data Security, Inc. MD5 +# Message-Digest Algorithm, described in RFC 1321. + +#_md5 md5module.c + + +# The _sha module implements the SHA checksum algorithms. +# (NIST's Secure Hash Algorithms.) +#_sha1 sha1module.c +#_sha256 sha256module.c +#_sha512 sha512module.c + + +# The _tkinter module. +# +# The command for _tkinter is long and site specific. Please +# uncomment and/or edit those parts as indicated. If you don't have a +# specific extension (e.g. Tix or BLT), leave the corresponding line +# commented out. (Leave the trailing backslashes in! If you +# experience strange errors, you may want to join all uncommented +# lines and remove the backslashes -- the backslash interpretation is +# done by the shell's "read" command and it may not be implemented on +# every system. + +# *** Always uncomment this (leave the leading underscore in!): +# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT \ +# *** Uncomment and edit to reflect where your Tcl/Tk libraries are: +# -L/usr/local/lib \ +# *** Uncomment and edit to reflect where your Tcl/Tk headers are: +# -I/usr/local/include \ +# *** Uncomment and edit to reflect where your X11 header files are: +# -I/usr/X11R6/include \ +# *** Or uncomment this for Solaris: +# -I/usr/openwin/include \ +# *** Uncomment and edit for Tix extension only: +# -DWITH_TIX -ltix8.1.8.2 \ +# *** Uncomment and edit for BLT extension only: +# -DWITH_BLT -I/usr/local/blt/blt8.0-unoff/include -lBLT8.0 \ +# *** Uncomment and edit for PIL (TkImaging) extension only: +# (See http://www.pythonware.com/products/pil/ for more info) +# -DWITH_PIL -I../Extensions/Imaging/libImaging tkImaging.c \ +# *** Uncomment and edit for TOGL extension only: +# -DWITH_TOGL togl.c \ +# *** Uncomment and edit to reflect your Tcl/Tk versions: +# -ltk8.2 -ltcl8.2 \ +# *** Uncomment and edit to reflect where your X11 libraries are: +# -L/usr/X11R6/lib \ +# *** Or uncomment this for Solaris: +# -L/usr/openwin/lib \ +# *** Uncomment these for TOGL extension only: +# -lGL -lGLU -lXext -lXmu \ +# *** Uncomment for AIX: +# -lld \ +# *** Always uncomment this; X11 libraries to link with: +# -lX11 + +# Lance Ellinghaus's syslog module +#syslog syslogmodule.c # syslog daemon interface + + +# Curses support, requiring the System V version of curses, often +# provided by the ncurses library. e.g. on Linux, link with -lncurses +# instead of -lcurses). +# +# First, look at Setup.config; configure may have set this for you. + +#_curses _cursesmodule.c -lcurses -ltermcap +# Wrapper for the panel library that's part of ncurses and SYSV curses. +#_curses_panel _curses_panel.c -lpanel -lncurses + + +# Modules that provide persistent dictionary-like semantics. You will +# probably want to arrange for at least one of them to be available on +# your machine, though none are defined by default because of library +# dependencies. The Python module dbm/__init__.py provides an +# implementation independent wrapper for these; dbm/dumb.py provides +# similar functionality (but slower of course) implemented in Python. + +# The standard Unix dbm module has been moved to Setup.config so that +# it will be compiled as a shared library by default. Compiling it as +# a built-in module causes conflicts with the pybsddb3 module since it +# creates a static dependency on an out-of-date version of db.so. +# +# First, look at Setup.config; configure may have set this for you. + +#_dbm _dbmmodule.c # dbm(3) may require -lndbm or similar + +# Anthony Baxter's gdbm module. GNU dbm(3) will require -lgdbm: +# +# First, look at Setup.config; configure may have set this for you. + +#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm + + +# Helper module for various ascii-encoders +#binascii binascii.c + +# Fred Drake's interface to the Python parser +#parser parsermodule.c + + +# Lee Busby's SIGFPE modules. +# The library to link fpectl with is platform specific. +# Choose *one* of the options below for fpectl: + +# For SGI IRIX (tested on 5.3): +#fpectl fpectlmodule.c -lfpe + +# For Solaris with SunPro compiler (tested on Solaris 2.5 with SunPro C 4.2): +# (Without the compiler you don't have -lsunmath.) +#fpectl fpectlmodule.c -R/opt/SUNWspro/lib -lsunmath -lm + +# For other systems: see instructions in fpectlmodule.c. +#fpectl fpectlmodule.c ... + +# Test module for fpectl. No extra libraries needed. +#fpetest fpetestmodule.c + +# Andrew Kuchling's zlib module. +# This require zlib 1.1.3 (or later). +# See http://www.gzip.org/zlib/ +#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz + +# Interface to the Expat XML parser +# +# Expat was written by James Clark and is now maintained by a group of +# developers on SourceForge; see www.libexpat.org for more +# information. The pyexpat module was written by Paul Prescod after a +# prototype by Jack Jansen. Source of Expat 1.95.2 is included in +# Modules/expat/. Usage of a system shared libexpat.so/expat.dll is +# not advised. +# +# More information on Expat can be found at www.libexpat.org. +# +#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI + +# Hye-Shik Chang's CJKCodecs + +# multibytecodec is required for all the other CJK codec modules +#_multibytecodec cjkcodecs/multibytecodec.c + +#_codecs_cn cjkcodecs/_codecs_cn.c +#_codecs_hk cjkcodecs/_codecs_hk.c +#_codecs_iso2022 cjkcodecs/_codecs_iso2022.c +#_codecs_jp cjkcodecs/_codecs_jp.c +#_codecs_kr cjkcodecs/_codecs_kr.c +#_codecs_tw cjkcodecs/_codecs_tw.c + +# Example -- included for reference only: +# xx xxmodule.c + +# Another example -- the 'xxsubtype' module shows C-level subtyping in action +xxsubtype xxsubtype.c diff --git a/tools/create_minimal_python.py b/tools/create_minimal_python.py new file mode 100644 index 000000000000..83c554b4caec --- /dev/null +++ b/tools/create_minimal_python.py @@ -0,0 +1,73 @@ +from importlib import machinery +import os +import shutil +import sys + +# These are modules which otherwise *should* be removed, but can't since they +# are requirements of pytest. + +# TODO: Find a solution to this, possibly involving running tests in a +# separate, more limited process. + +PYTEST_REQUIREMENTS = [ + '_posixsubprocess', 'select', 'binascii', 'pyexpat', '_socket' +] + +# These are optional parts of the standard library that are required for core +# functionality in matplotlib that can not easily be avoided. + +MATPLOTLIB_HARD_REQUIREMENTS = [ + 'zlib', '_csv' +] + + +KEEP_MODULES = PYTEST_REQUIREMENTS + MATPLOTLIB_HARD_REQUIREMENTS + + +def iterate_modules_to_remove(): + with open(os.path.join( + os.path.dirname(__file__), 'Setup.dist'), 'r') as fd: + for line in fd: + if line.startswith('# Modules with some UNIX dependencies'): + break + + for line in fd: + line = line.strip() + if line.startswith('#') and not line.startswith('# ') and len(line) > 1: + modulename = line[1:].split()[0] + if modulename not in KEEP_MODULES: + yield modulename + + # Some other commonly-missing modules that aren't in Setup.dist + yield '_bz2' + yield '_ctypes' + yield '_lzma' + yield '_lsprof' + yield '_multiprocessing' + yield 'ossaudiodev' + yield '_tkinter' + yield 'tkinter' + yield '_sqlite3' + yield '_thread' + + +def main(): + PYTHON_VERSION = 'python{}.{}'.format(sys.version_info[0], sys.version_info[1]) + + DST_ROOT = os.path.abspath('./minimal_python') + LIB_ROOT = os.path.join(DST_ROOT, 'lib', PYTHON_VERSION) + shutil.rmtree(os.path.join(LIB_ROOT, '__pycache__')) + + SETUP_DIST_PATH = os.path.join(os.path.dirname(__file__), 'Setup.dist') + for modulename in iterate_modules_to_remove(): + for root in ['', 'lib-dynload']: + for ext in machinery.all_suffixes(): + modulepath = os.path.join(LIB_ROOT, root, modulename + ext) + if os.path.isfile(modulepath): + os.remove(modulepath) + elif os.path.isdir(modulepath): + os.rmtree(modulepath) + + +if __name__ == '__main__': + main()