diff --git a/examples/text_labels_and_annotations/legend.py b/examples/text_labels_and_annotations/legend.py
index 33d8af3204e4..e2a1d1319324 100644
--- a/examples/text_labels_and_annotations/legend.py
+++ b/examples/text_labels_and_annotations/legend.py
@@ -21,7 +21,11 @@
 ax.plot(a, d, 'k:', label='Data length')
 ax.plot(a, c + d, 'k', label='Total message length')
 
-legend = ax.legend(loc='upper center', shadow=True, fontsize='x-large')
+# Create an arrow with pre-defined label.
+ax.annotate("", xy=(1.5, 4.5), xytext=(1.5, 9.0),
+            arrowprops={'arrowstyle': '<->', 'color': 'C7'}, label='Distance')
+
+legend = ax.legend(loc='upper center', shadow=True, fontsize='large')
 
 # Put a nicer background color on the legend.
 legend.get_frame().set_facecolor('C0')
diff --git a/lib/matplotlib/legend.py b/lib/matplotlib/legend.py
index 7aef9ed7e8b3..e0e774d6e7cf 100644
--- a/lib/matplotlib/legend.py
+++ b/lib/matplotlib/legend.py
@@ -34,11 +34,11 @@
 from matplotlib.font_manager import FontProperties
 from matplotlib.lines import Line2D
 from matplotlib.patches import (Patch, Rectangle, Shadow, FancyBboxPatch,
-                                StepPatch)
+                                StepPatch, FancyArrowPatch)
 from matplotlib.collections import (
     Collection, CircleCollection, LineCollection, PathCollection,
     PolyCollection, RegularPolyCollection)
-from matplotlib.text import Text
+from matplotlib.text import Annotation, Text
 from matplotlib.transforms import Bbox, BboxBase, TransformedBbox
 from matplotlib.transforms import BboxTransformTo, BboxTransformFrom
 from matplotlib.offsetbox import (
@@ -649,7 +649,9 @@ def draw(self, renderer):
             update_func=legend_handler.update_from_first_child),
         tuple: legend_handler.HandlerTuple(),
         PathCollection: legend_handler.HandlerPathCollection(),
-        PolyCollection: legend_handler.HandlerPolyCollection()
+        PolyCollection: legend_handler.HandlerPolyCollection(),
+        FancyArrowPatch: legend_handler.HandlerFancyArrowPatch(),
+        Annotation: legend_handler.HandlerAnnotation()
         }
 
     # (get|set|update)_default_handler_maps are public interfaces to
@@ -799,6 +801,7 @@ def _init_legend_box(self, handles, labels, markerfirst=True):
                                              self._legend_handle_box])
         self._legend_box.set_figure(self.figure)
         self._legend_box.axes = self.axes
+        self._legend_box.set_offset(self._findoffset)
         self.texts = text_list
         self.legendHandles = handle_list
 
diff --git a/lib/matplotlib/legend_handler.py b/lib/matplotlib/legend_handler.py
index c21c6d1212d3..fafb685f3b8d 100644
--- a/lib/matplotlib/legend_handler.py
+++ b/lib/matplotlib/legend_handler.py
@@ -34,7 +34,7 @@ def legend_artist(self, legend, orig_handle, fontsize, handlebox)
 
 from matplotlib import _api, cbook
 from matplotlib.lines import Line2D
-from matplotlib.patches import Rectangle
+from matplotlib.patches import Rectangle, FancyArrowPatch
 import matplotlib.collections as mcoll
 
 
@@ -806,3 +806,40 @@ def create_artists(self, legend, orig_handle,
         self.update_prop(p, orig_handle, legend)
         p.set_transform(trans)
         return [p]
+
+
+class HandlerFancyArrowPatch(HandlerPatch):
+    """
+    Handler for `~.FancyArrowPatch` instances.
+    """
+    def _create_patch(self, legend, orig_handle, xdescent, ydescent, width,
+                      height, fontsize):
+        arrow = FancyArrowPatch([-xdescent,
+                                 -ydescent + height / 2],
+                                [-xdescent + width,
+                                 -ydescent + height / 2],
+                                mutation_scale=width / 3)
+        arrow.set_arrowstyle(orig_handle.get_arrowstyle())
+        return arrow
+
+
+class HandlerAnnotation(HandlerBase):
+    """
+    Handler for `.Annotation` instances.
+    Defers to `HandlerFancyArrowPatch` to draw the annotation arrow (if any).
+    """
+    def create_artists(self, legend, orig_handle, xdescent, ydescent, width,
+                       height, fontsize, trans):
+
+        if orig_handle.arrow_patch is not None:
+            # Arrow without text
+            handler = HandlerFancyArrowPatch()
+            handle = orig_handle.arrow_patch
+        else:
+            # No arrow
+            handler = HandlerPatch()
+            # Dummy patch to copy properties from to rectangle patch
+            handle = Rectangle(width=0, height=0, xy=(0, 0), color='none')
+
+        return handler.create_artists(legend, handle, xdescent, ydescent,
+                                      width, height, fontsize, trans)
diff --git a/lib/matplotlib/tests/test_legend.py b/lib/matplotlib/tests/test_legend.py
index 004b9407fddb..97d5061d9ab2 100644
--- a/lib/matplotlib/tests/test_legend.py
+++ b/lib/matplotlib/tests/test_legend.py
@@ -927,3 +927,30 @@ def test_legend_markers_from_line2d():
 
     assert markers == new_markers == _markers
     assert labels == new_labels
+
+
+def test_annotation_legend():
+    fig, ax = plt.subplots()
+    # Add annotation with arrow and label
+    ax.annotate("", xy=(0.5, 0.5), xytext=(0.5, 0.7),
+                arrowprops={'arrowstyle': '<->'}, label="Bar")
+    legend = ax.legend()
+    assert len(legend.get_texts()) == 1
+    # No arrow, no label
+    ax.annotate("Foo", xy=(0.3, 0.3))
+    legend = ax.legend()
+    assert len(legend.get_texts()) == 1
+    # Arrow, no label
+    ax.annotate("FooBar", xy=(0.7, 0.7), xytext=(0.7, 0.9),
+                arrowprops={'arrowstyle': '->'})
+    legend = ax.legend()
+    assert len(legend.get_texts()) == 1
+    # Add another annotation with arrow and label. now with non-empty text
+    ax.annotate("Foo", xy=(0.1, 0.1), xytext=(0.1, 0.7),
+                arrowprops={'arrowstyle': '<-'}, label="Foo")
+    legend = ax.legend()
+    assert len(legend.get_texts()) == 2
+    # Add annotation without arrow, but with label
+    ax.annotate("Foo", xy=(0.2, 0.2), xytext=(0.2, 0.6), label="Foo")
+    legend = ax.legend()
+    assert len(legend.get_texts()) == 3