Reduce number of font files opened

This should hopefully address the long-reported "Too many open files" error message (Fix #3315). To reproduce: On a Mac or Windows box with starvation for file handles (Linux has a much higher file handle limit by default), build the docs, then immediately build again. This will trigger the caching bug. The font cache in the mathtext renderer was broken. It was caching a font file once for every *combination* of font properties, including things like size. Therefore, in a complex math expression containing many different sizes of the same font, the font file was opened once for each of those sizes. Font files are opened and kept open (rather than opened, read, and closed) so that FreeType only needs to load the actual glyphs that are used, rather than the entire font. In an era of cheap memory and fast disk, it probably doesn't matter for our current fonts, but once #5214 is merged, we will have larger font files with many more glyphs and this loading time will matter more. The solution here is to do all font file loading in one place and to use `lru_cache` (available since Python 3.2) to do the caching, and to use only the file name and hinting parameters as a cache key. For earlier versions of Python, the functools32 backport package is required. (Or we can discuss whether we want to vendor it).
matplotlib · tacaswell · Oct 30, 2015 · Oct 21, 2015 · Oct 21, 2015 · Oct 21, 2015
commit 349762b8d2fba9638746e6ed1c8d237a925561ec
diff --git a/lib/matplotlib/backends/backend_agg.py b/lib/matplotlib/backends/backend_agg.py
@@ -32,8 +32,8 @@
     FigureManagerBase, FigureCanvasBase
 from matplotlib.cbook import is_string_like, maxdict, restrict_dict
 from matplotlib.figure import Figure
-from matplotlib.font_manager import findfont
-from matplotlib.ft2font import FT2Font, LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING, \
+from matplotlib.font_manager import findfont, get_font
+from matplotlib.ft2font import LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING, \
     LOAD_DEFAULT, LOAD_NO_AUTOHINT
 from matplotlib.mathtext import MathTextParser
 from matplotlib.path import Path
@@ -81,7 +81,6 @@ class RendererAgg(RendererBase):
    # renderer at a time

    lock = threading.RLock()
-    _fontd = maxdict(50)
    def __init__(self, width, height, dpi):
        if __debug__: verbose.report('RendererAgg.__init__', 'debug-annoying')
        RendererBase.__init__(self)
@@ -191,6 +190,7 @@ def draw_text(self, gc, x, y, s, prop, angle, ismath=False, mtext=None):

        flags = get_hinting_flag()
        font = self._get_agg_font(prop)
+
        if font is None: return None
        if len(s) == 1 and ord(s) > 127:
            font.load_char(ord(s), flags=flags)
@@ -272,18 +272,10 @@ def _get_agg_font(self, prop):
        if __debug__: verbose.report('RendererAgg._get_agg_font',
                                     'debug-annoying')

-        key = hash(prop)
-        font = RendererAgg._fontd.get(key)
-
-        if font is None:
-            fname = findfont(prop)
-            font = RendererAgg._fontd.get(fname)
-            if font is None:
-                font = FT2Font(
-                    fname,
-                    hinting_factor=rcParams['text.hinting_factor'])
-                RendererAgg._fontd[fname] = font
-            RendererAgg._fontd[key] = font
+        fname = findfont(prop)
+        font = get_font(
+            fname,
+            hinting_factor=rcParams['text.hinting_factor'])

        font.clear()
        size = prop.get_size_in_points()

diff --git a/lib/matplotlib/backends/backend_pdf.py b/lib/matplotlib/backends/backend_pdf.py
@@ -35,11 +35,11 @@
 from matplotlib.cbook import Bunch, is_string_like, \
    get_realpath_and_stat, is_writable_file_like, maxdict
 from matplotlib.figure import Figure
-from matplotlib.font_manager import findfont, is_opentype_cff_font
+from matplotlib.font_manager import findfont, is_opentype_cff_font, get_font
 from matplotlib.afm import AFM
 import matplotlib.type1font as type1font
 import matplotlib.dviread as dviread
-from matplotlib.ft2font import FT2Font, FIXED_WIDTH, ITALIC, LOAD_NO_SCALE, \
+from matplotlib.ft2font import FIXED_WIDTH, ITALIC, LOAD_NO_SCALE, \
    LOAD_NO_HINTING, KERNING_UNFITTED
 from matplotlib.mathtext import MathTextParser
 from matplotlib.transforms import Affine2D, BboxBase
@@ -757,7 +757,7 @@ def createType1Descriptor(self, t1font, fontfile):
        if 0:
            flags |= 1 << 18

-        ft2font = FT2Font(fontfile)
+        ft2font = get_font(fontfile)

        descriptor = {
            'Type':        Name('FontDescriptor'),
@@ -817,7 +817,7 @@ def _get_xobject_symbol_name(self, filename, symbol_name):
    def embedTTF(self, filename, characters):
        """Embed the TTF font from the named file into the document."""

-        font = FT2Font(filename)
+        font = get_font(filename)
        fonttype = rcParams['pdf.fonttype']

        def cvt(length, upe=font.units_per_EM, nearest=True):
@@ -1526,7 +1526,6 @@ def writeTrailer(self):


 class RendererPdf(RendererBase):
-    truetype_font_cache = maxdict(50)
    afm_font_cache = maxdict(50)

    def __init__(self, file, image_dpi):
@@ -2126,15 +2125,8 @@ def _get_font_afm(self, prop):
        return font

    def _get_font_ttf(self, prop):
-        key = hash(prop)
-        font = self.truetype_font_cache.get(key)
-        if font is None:
-            filename = findfont(prop)
-            font = self.truetype_font_cache.get(filename)
-            if font is None:
-                font = FT2Font(filename)
-                self.truetype_font_cache[filename] = font
-            self.truetype_font_cache[key] = font
+        filename = findfont(prop)
+        font = get_font(filename)
        font.clear()
        font.set_size(prop.get_size_in_points(), 72)
        return font

diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py
@@ -36,10 +36,9 @@
 system_fonts = []
 if sys.platform.startswith('win'):
    from matplotlib import font_manager
-    from matplotlib.ft2font import FT2Font
    for f in font_manager.win32InstalledFonts():
        try:
-            system_fonts.append(FT2Font(str(f)).family_name)
+            system_fonts.append(font_manager.get_font(str(f)).family_name)
        except:
            pass # unknown error, skip this font
 else:

diff --git a/lib/matplotlib/backends/backend_ps.py b/lib/matplotlib/backends/backend_ps.py
@@ -28,8 +28,8 @@ def _fn_name(): return sys._getframe(1).f_code.co_name
    is_writable_file_like, maxdict, file_requires_unicode
 from matplotlib.figure import Figure

-from matplotlib.font_manager import findfont, is_opentype_cff_font
-from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING
+from matplotlib.font_manager import findfont, is_opentype_cff_font, get_font
+from matplotlib.ft2font import KERNING_DEFAULT, LOAD_NO_HINTING
 from matplotlib.ttconv import convert_ttf_to_ps
 from matplotlib.mathtext import MathTextParser
 from matplotlib._mathtext_data import uni2type1
@@ -199,7 +199,6 @@ class RendererPS(RendererBase):
    context instance that controls the colors/styles.
    """

-    fontd = maxdict(50)
    afmfontd = maxdict(50)

    def __init__(self, width, height, pswriter, imagedpi=72):
@@ -393,15 +392,8 @@ def _get_font_afm(self, prop):
        return font

    def _get_font_ttf(self, prop):
-        key = hash(prop)
-        font = self.fontd.get(key)
-        if font is None:
-            fname = findfont(prop)
-            font = self.fontd.get(fname)
-            if font is None:
-                font = FT2Font(fname)
-                self.fontd[fname] = font
-            self.fontd[key] = font
+        fname = findfont(prop)
+        font = get_font(fname)
        font.clear()
        size = prop.get_size_in_points()
        font.set_size(size, 72.0)
@@ -1145,7 +1137,7 @@ def print_figure_impl():
            if not rcParams['ps.useafm']:
                for font_filename, chars in six.itervalues(ps_renderer.used_characters):
                    if len(chars):
-                        font = FT2Font(font_filename)
+                        font = get_font(font_filename)
                        cmap = font.get_charmap()
                        glyph_ids = []
                        for c in chars:

diff --git a/lib/matplotlib/backends/backend_svg.py b/lib/matplotlib/backends/backend_svg.py
@@ -19,8 +19,8 @@
 from matplotlib.cbook import is_string_like, is_writable_file_like, maxdict
 from matplotlib.colors import rgb2hex
 from matplotlib.figure import Figure
-from matplotlib.font_manager import findfont, FontProperties
-from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING
+from matplotlib.font_manager import findfont, FontProperties, get_font
+from matplotlib.ft2font import KERNING_DEFAULT, LOAD_NO_HINTING
 from matplotlib.mathtext import MathTextParser
 from matplotlib.path import Path
 from matplotlib import _path
@@ -326,15 +326,8 @@ def _make_flip_transform(self, transform):
                .translate(0.0, self.height))

    def _get_font(self, prop):
-        key = hash(prop)
-        font = self.fontd.get(key)
-        if font is None:
-            fname = findfont(prop)
-            font = self.fontd.get(fname)
-            if font is None:
-                font = FT2Font(fname)
-                self.fontd[fname] = font
-            self.fontd[key] = font
+        fname = findfont(prop)
+        font = get_font(fname)
        font.clear()
        size = prop.get_size_in_points()
        font.set_size(size, 72.0)
@@ -495,7 +488,7 @@ def _write_svgfonts(self):
        writer = self.writer
        writer.start('defs')
        for font_fname, chars in six.iteritems(self._fonts):
-            font = FT2Font(font_fname)
+            font = get_font(font_fname)
            font.set_size(72, 72)
            sfnt = font.get_sfnt()
            writer.start('font', id=sfnt[(1, 0, 0, 4)])

diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py
@@ -63,6 +63,12 @@
 from matplotlib.fontconfig_pattern import \
    parse_fontconfig_pattern, generate_fontconfig_pattern

+try:
+    from functools import lru_cache
+except ImportError:
+    from functools32 import lru_cache
+
+
 USE_FONTCONFIG = False
 verbose = matplotlib.verbose

@@ -733,7 +739,7 @@ def get_name(self):
        Return the name of the font that best matches the font
        properties.
        """
-        return ft2font.FT2Font(findfont(self)).family_name
+        return get_font(findfont(self)).family_name

    def get_style(self):
        """
@@ -1336,7 +1342,6 @@ def findfont(self, prop, fontext='ttf', directory=None,
            _lookup_cache[fontext].set(prop, result)
        return result

-
 _is_opentype_cff_font_cache = {}
 def is_opentype_cff_font(filename):
    """
@@ -1357,6 +1362,10 @@ def is_opentype_cff_font(filename):
 fontManager = None
 _fmcache = None

+
+get_font = lru_cache(64)(ft2font.FT2Font)
+
+
 # The experimental fontconfig-based backend.
 if USE_FONTCONFIG and sys.platform != 'win32':
    import re

diff --git a/lib/matplotlib/mathtext.py b/lib/matplotlib/mathtext.py
@@ -50,8 +50,8 @@
 from matplotlib.afm import AFM
 from matplotlib.cbook import Bunch, get_realpath_and_stat, \
    is_string_like, maxdict
-from matplotlib.ft2font import FT2Font, FT2Image, KERNING_DEFAULT, LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING
-from matplotlib.font_manager import findfont, FontProperties
+from matplotlib.ft2font import FT2Image, KERNING_DEFAULT, LOAD_FORCE_AUTOHINT, LOAD_NO_HINTING
+from matplotlib.font_manager import findfont, FontProperties, get_font
 from matplotlib._mathtext_data import latex_to_bakoma, \
        latex_to_standard, tex2uni, latex_to_cmex, stix_virtual_fonts
 from matplotlib import get_data_path, rcParams
@@ -563,7 +563,7 @@ def __init__(self, default_font_prop, mathtext_backend):
        self._fonts = {}

        filename = findfont(default_font_prop)
-        default_font = self.CachedFont(FT2Font(filename))
+        default_font = self.CachedFont(get_font(filename))
        self._fonts['default'] = default_font
        self._fonts['regular'] = default_font

@@ -576,10 +576,9 @@ def _get_font(self, font):
            basename = self.fontmap[font]
        else:
            basename = font
-
        cached_font = self._fonts.get(basename)
        if cached_font is None and os.path.exists(basename):
-            font = FT2Font(basename)
+            font = get_font(basename)
            cached_font = self.CachedFont(font)
            self._fonts[basename] = cached_font
            self._fonts[font.postscript_name] = cached_font

diff --git a/lib/matplotlib/tests/__init__.py b/lib/matplotlib/tests/__init__.py
@@ -49,12 +49,6 @@ def setup():
    rcParams['text.hinting'] = False
    rcParams['text.hinting_factor'] = 8

-    # Clear the font caches.  Otherwise, the hinting mode can travel
-    # from one test to another.
-    backend_agg.RendererAgg._fontd.clear()
-    backend_pdf.RendererPdf.truetype_font_cache.clear()
-    backend_svg.RendererSVG.fontd.clear()
-

 def assert_str_equal(reference_str, test_str,
                     format_str=('String {str1} and {str2} do not '

diff --git a/lib/matplotlib/textpath.py b/lib/matplotlib/textpath.py
@@ -13,11 +13,11 @@
 from matplotlib.path import Path
 from matplotlib import rcParams
 import matplotlib.font_manager as font_manager
-from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING
+from matplotlib.ft2font import KERNING_DEFAULT, LOAD_NO_HINTING
 from matplotlib.ft2font import LOAD_TARGET_LIGHT
 from matplotlib.mathtext import MathTextParser
 import matplotlib.dviread as dviread
-from matplotlib.font_manager import FontProperties
+from matplotlib.font_manager import FontProperties, get_font
 from matplotlib.transforms import Affine2D
 from matplotlib.externals.six.moves.urllib.parse import quote as urllib_quote

@@ -54,7 +54,7 @@ def _get_font(self, prop):
        find a ttf font.
        """
        fname = font_manager.findfont(prop)
-        font = FT2Font(fname)
+        font = get_font(fname)
        font.set_size(self.FONT_SCALE, self.DPI)

        return font
@@ -334,7 +334,7 @@ def get_glyphs_tex(self, prop, s, glyph_map=None,
            font_bunch = self.tex_font_map[dvifont.texname]

            if font_and_encoding is None:
-                font = FT2Font(font_bunch.filename)
+                font = get_font(font_bunch.filename)

                for charmap_name, charmap_code in [("ADOBE_CUSTOM",
                                                    1094992451),

diff --git a/lib/mpl_toolkits/tests/__init__.py b/lib/mpl_toolkits/tests/__init__.py
@@ -49,12 +49,6 @@ def setup():
    rcParams['text.hinting'] = False
    rcParams['text.hinting_factor'] = 8

-    # Clear the font caches.  Otherwise, the hinting mode can travel
-    # from one test to another.
-    backend_agg.RendererAgg._fontd.clear()
-    backend_pdf.RendererPdf.truetype_font_cache.clear()
-    backend_svg.RendererSVG.fontd.clear()
-

 def assert_str_equal(reference_str, test_str,
                     format_str=('String {str1} and {str2} do not '

diff --git a/setup.py b/setup.py
@@ -67,6 +67,7 @@
    'Required dependencies and extensions',
    setupext.Numpy(),
    setupext.Dateutil(),
+    setupext.FuncTools32(),
    setupext.Pytz(),
    setupext.Cycler(),
    setupext.Tornado(),

diff --git a/setupext.py b/setupext.py
@@ -1221,6 +1221,29 @@ def get_install_requires(self):
        return [dateutil]


+class FuncTools32(SetupPackage):
+    name = "functools32"
+
+    def check(self):
+        if sys.version_info[:2] < (3, 2):
+            try:
+                import functools32
+            except ImportError:
+                return (
+                    "functools32 was not found. It is required for for"
+                    "python versions prior to 3.2")
+
+            return "using functools32 version %s" % functools32.__version__
+        else:
+            return "Not required"
+
+    def get_install_requires(self):
+        if sys.version_info[:2] < (3, 2):
+            return ['functools32']
+        else:
+            return []
+
+
 class Tornado(OptionalPackage):
    name = "tornado"