diff --git a/doc/users/next_whats_new/widget_dragging.rst b/doc/users/next_whats_new/widget_dragging.rst
new file mode 100644
index 000000000000..3766c680341d
--- /dev/null
+++ b/doc/users/next_whats_new/widget_dragging.rst
@@ -0,0 +1,9 @@
+Dragging selectors
+------------------
+
+The `~matplotlib.widgets.RectangleSelector` and
+`~matplotlib.widgets.EllipseSelector` have a new keyword argument,
+*drag_from_anywhere*, which when set to `True` allows you to click and drag
+from anywhere inside the selector to move it. Previously it was only possible
+to move it by either activating the move modifier button, or clicking on the
+central handle.
diff --git a/lib/matplotlib/tests/test_widgets.py b/lib/matplotlib/tests/test_widgets.py
index 0f098002b177..3452313ddf12 100644
--- a/lib/matplotlib/tests/test_widgets.py
+++ b/lib/matplotlib/tests/test_widgets.py
@@ -44,6 +44,41 @@ def test_rectangle_selector():
     check_rectangle(rectprops=dict(fill=True))
 
 
+@pytest.mark.parametrize('drag_from_anywhere, new_center',
+                         [[True, (60, 75)],
+                          [False, (30, 20)]])
+def test_rectangle_drag(drag_from_anywhere, new_center):
+    ax = get_ax()
+
+    def onselect(epress, erelease):
+        pass
+
+    tool = widgets.RectangleSelector(ax, onselect, interactive=True,
+                                     drag_from_anywhere=drag_from_anywhere)
+    # Create rectangle
+    do_event(tool, 'press', xdata=0, ydata=10, button=1)
+    do_event(tool, 'onmove', xdata=100, ydata=120, button=1)
+    do_event(tool, 'release', xdata=100, ydata=120, button=1)
+    assert tool.center == (50, 65)
+    # Drag inside rectangle, but away from centre handle
+    #
+    # If drag_from_anywhere == True, this will move the rectangle by (10, 10),
+    # giving it a new center of (60, 75)
+    #
+    # If drag_from_anywhere == False, this will create a new rectangle with
+    # center (30, 20)
+    do_event(tool, 'press', xdata=25, ydata=15, button=1)
+    do_event(tool, 'onmove', xdata=35, ydata=25, button=1)
+    do_event(tool, 'release', xdata=35, ydata=25, button=1)
+    assert tool.center == new_center
+    # Check that in both cases, dragging outside the rectangle draws a new
+    # rectangle
+    do_event(tool, 'press', xdata=175, ydata=185, button=1)
+    do_event(tool, 'onmove', xdata=185, ydata=195, button=1)
+    do_event(tool, 'release', xdata=185, ydata=195, button=1)
+    assert tool.center == (180, 190)
+
+
 def test_ellipse():
     """For ellipse, test out the key modifiers"""
     ax = get_ax()
diff --git a/lib/matplotlib/widgets.py b/lib/matplotlib/widgets.py
index 333af5257b5a..76dde645a49d 100644
--- a/lib/matplotlib/widgets.py
+++ b/lib/matplotlib/widgets.py
@@ -2189,7 +2189,8 @@ def __init__(self, ax, onselect, drawtype='box',
                  minspanx=0, minspany=0, useblit=False,
                  lineprops=None, rectprops=None, spancoords='data',
                  button=None, maxdist=10, marker_props=None,
-                 interactive=False, state_modifier_keys=None):
+                 interactive=False, state_modifier_keys=None,
+                 drag_from_anywhere=False):
         r"""
         Parameters
         ----------
@@ -2261,6 +2262,10 @@ def onselect(eclick: MouseEvent, erelease: MouseEvent)
               default: "ctrl".
 
             "square" and "center" can be combined.
+
+        drag_from_anywhere : bool, optional
+            If `True`, the widget can be moved by clicking anywhere within
+            its bounds.
         """
         super().__init__(ax, onselect, useblit=useblit, button=button,
                          state_modifier_keys=state_modifier_keys)
@@ -2268,6 +2273,7 @@ def onselect(eclick: MouseEvent, erelease: MouseEvent)
         self.to_draw = None
         self.visible = True
         self.interactive = interactive
+        self.drag_from_anywhere = drag_from_anywhere
 
         if drawtype == 'none':  # draw a line but make it invisible
             drawtype = 'line'
@@ -2407,8 +2413,9 @@ def _onmove(self, event):
                 y1 = event.ydata
 
         # move existing shape
-        elif (('move' in self.state or self.active_handle == 'C')
-              and self._extents_on_press is not None):
+        elif (('move' in self.state or self.active_handle == 'C' or
+               (self.drag_from_anywhere and self._contains(event))) and
+              self._extents_on_press is not None):
             x0, x1, y0, y1 = self._extents_on_press
             dx = event.xdata - self.eventpress.xdata
             dy = event.ydata - self.eventpress.ydata
@@ -2539,16 +2546,24 @@ def _set_active_handle(self, event):
         if 'move' in self.state:
             self.active_handle = 'C'
             self._extents_on_press = self.extents
-
         # Set active handle as closest handle, if mouse click is close enough.
         elif m_dist < self.maxdist * 2:
+            # Prioritise center handle over other handles
             self.active_handle = 'C'
         elif c_dist > self.maxdist and e_dist > self.maxdist:
-            self.active_handle = None
-            return
+            # Not close to any handles
+            if self.drag_from_anywhere and self._contains(event):
+                # Check if we've clicked inside the region
+                self.active_handle = 'C'
+                self._extents_on_press = self.extents
+            else:
+                self.active_handle = None
+                return
         elif c_dist < e_dist:
+            # Closest to a corner handle
             self.active_handle = self._corner_order[c_idx]
         else:
+            # Closest to an edge handle
             self.active_handle = self._edge_order[e_idx]
 
         # Save coordinates of rectangle at the start of handle movement.
@@ -2560,6 +2575,10 @@ def _set_active_handle(self, event):
             y0, y1 = y1, event.ydata
         self._extents_on_press = x0, x1, y0, y1
 
+    def _contains(self, event):
+        """Return True if event is within the patch."""
+        return self.to_draw.contains(event, radius=0)[0]
+
     @property
     def geometry(self):
         """