scikit-learn · snath-xoc · Apr 4, 2022 · Apr 4, 2022 · Apr 4, 2022 · Apr 4, 2022
diff --git a/doc/whats_new/v1.6.rst b/doc/whats_new/v1.6.rst
@@ -103,6 +103,12 @@ Changelog
    :pr:`123456` by :user:`Joe Bloggs <joeongithub>`.
    where 123455 is the *pull request* number, not the issue number.

+:mod:`sklearn.linear_model`
+..........................
+- |Fix| :func:`_coordinate_descent._alpha_grid` adapted to account for sample weights.
+  :pr:`23045`by :user:`John Hopfensperger <s-banach>`and :pr:`29308`by :user:`Shruti Nath <snath-xoc>`.
+
+
 :mod:`sklearn.base`
 ...................


diff --git a/sklearn/linear_model/_coordinate_descent.py b/sklearn/linear_model/_coordinate_descent.py
@@ -98,6 +98,7 @@ def _alpha_grid(
    eps=1e-3,
    n_alphas=100,
    copy_X=True,
+    sample_weight=None,
 ):
    """Compute the grid of alpha values for elastic net parameter search

@@ -132,6 +133,8 @@ def _alpha_grid(

    copy_X : bool, default=True
        If ``True``, X will be copied; else, it may be overwritten.
+
+    sample_weight : ndarray of shape (n_samples,)
    """
    if l1_ratio == 0:
        raise ValueError(
@@ -140,43 +143,40 @@ def _alpha_grid(
            "your estimator with the appropriate `alphas=` "
            "argument."
        )
-    n_samples = len(y)
-
-    sparse_center = False
-    if Xy is None:
-        X_sparse = sparse.issparse(X)
-        sparse_center = X_sparse and fit_intercept
-        X = check_array(
-            X, accept_sparse="csc", copy=(copy_X and fit_intercept and not X_sparse)
+    if Xy is not None:
+        Xyw = Xy
+    else:
+        X, y, X_offset, _, _ = _preprocess_data(
+            X,
+            y,
+            fit_intercept=fit_intercept,
+            copy=copy_X,
+            sample_weight=sample_weight,
+            check_input=False,
        )
-        if not X_sparse:
-            # X can be touched inplace thanks to the above line
-            X, y, _, _, _ = _preprocess_data(
-                X, y, fit_intercept=fit_intercept, copy=False
-            )
-        Xy = safe_sparse_dot(X.T, y, dense_output=True)
-
-        if sparse_center:
-            # Workaround to find alpha_max for sparse matrices.
-            # since we should not destroy the sparsity of such matrices.
-            _, _, X_offset, _, X_scale = _preprocess_data(
-                X, y, fit_intercept=fit_intercept
-            )
-            mean_dot = X_offset * np.sum(y)
-
-    if Xy.ndim == 1:
-        Xy = Xy[:, np.newaxis]
+        if sample_weight is not None:
+            if y.ndim > 1:
+                yw = y * np.broadcast_to(sample_weight.reshape(-1, 1), y.shape)

-    if sparse_center:
-        if fit_intercept:
-            Xy -= mean_dot[:, np.newaxis]
+            else:
+                yw = y * sample_weight
+        else:
+            yw = y
+        if sparse.issparse(X):
+            Xyw = safe_sparse_dot(X.T, yw, dense_output=True) - np.sum(yw) * X_offset
+        else:
+            Xyw = np.dot(X.T, yw)

-    alpha_max = np.sqrt(np.sum(Xy**2, axis=1)).max() / (n_samples * l1_ratio)
+    if Xyw.ndim == 1:
+        Xyw = Xyw[:, np.newaxis]
+    if sample_weight is not None:
+        n_samples = sample_weight.sum()
+    else:
+        n_samples = X.shape[0]
+    alpha_max = np.sqrt(np.sum(Xyw**2, axis=1)).max() / (n_samples * l1_ratio)

-    if alpha_max <= np.finfo(float).resolution:
-        alphas = np.empty(n_alphas)
-        alphas.fill(np.finfo(float).resolution)
-        return alphas
+    if alpha_max <= np.finfo(np.float64).resolution:
+        return np.full(n_alphas, np.finfo(np.float64).resolution)

    return np.geomspace(alpha_max, alpha_max * eps, num=n_alphas)

@@ -979,7 +979,6 @@ def fit(self, X, y, sample_weight=None, check_input=True):
                accept_sparse="csc",
                order="F",
                dtype=[np.float64, np.float32],
-                force_writeable=True,
                accept_large_sparse=False,
                copy=X_copied,
                multi_output=True,
@@ -1608,7 +1607,6 @@ def fit(self, X, y, sample_weight=None, **params):
            check_X_params = dict(
                accept_sparse="csc",
                dtype=[np.float64, np.float32],
-                force_writeable=True,
                copy=False,
                accept_large_sparse=False,
            )
@@ -1634,7 +1632,6 @@ def fit(self, X, y, sample_weight=None, **params):
                accept_sparse="csc",
                dtype=[np.float64, np.float32],
                order="F",
-                force_writeable=True,
                copy=copy_X,
            )
            X, y = self._validate_data(
@@ -1702,6 +1699,7 @@ def fit(self, X, y, sample_weight=None, **params):
                    eps=self.eps,
                    n_alphas=self.n_alphas,
                    copy_X=self.copy_X,
+                    sample_weight=sample_weight,
                )
                for l1_ratio in l1_ratios
            ]
@@ -2511,7 +2509,6 @@ def fit(self, X, y):
        check_X_params = dict(
            dtype=[np.float64, np.float32],
            order="F",
-            force_writeable=True,
            copy=self.copy_X and self.fit_intercept,
        )
        check_y_params = dict(ensure_2d=False, order="F")

diff --git a/sklearn/linear_model/tests/test_coordinate_descent.py b/sklearn/linear_model/tests/test_coordinate_descent.py
@@ -48,6 +48,7 @@
    assert_almost_equal,
    assert_array_almost_equal,
    assert_array_equal,
+    assert_array_less,
    ignore_warnings,
 )
 from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS
@@ -1319,39 +1320,34 @@ def test_enet_cv_sample_weight_correctness(fit_intercept, sparse_container):
        X = sparse_container(X)
    params = dict(tol=1e-6)

-    # Set alphas, otherwise the two cv models might use different ones.
-    if fit_intercept:
-        alphas = np.linspace(0.001, 0.01, num=91)
-    else:
-        alphas = np.linspace(0.01, 0.1, num=91)
-
-    # We weight the first fold 2 times more.
-    sw[:n_samples] = 2
+    # We weight the first fold n times more.
-    # We weight the first fold n times more.
+    # We re-weight the first cross-validation group with random integer weights.
+    # The samples in the other groups are left with unit weights.
-    # We weight the first fold n times more.
+    # We re-weight the first cross-validation group with random integer weights.
+    # The samples in the other groups are left with unit weights.
+    sw[:n_samples] = rng.randint(0, 5, size=sw[:n_samples].shape[0])
    groups_sw = np.r_[
        np.full(n_samples, 0), np.full(n_samples, 1), np.full(n_samples, 2)
    ]
    splits_sw = list(LeaveOneGroupOut().split(X, groups=groups_sw))
-    reg_sw = ElasticNetCV(
-        alphas=alphas, cv=splits_sw, fit_intercept=fit_intercept, **params
-    )
+    reg_sw = ElasticNetCV(cv=splits_sw, fit_intercept=fit_intercept, **params)
    reg_sw.fit(X, y, sample_weight=sw)

    # We repeat the first fold 2 times and provide splits ourselves
    if sparse_container is not None:
        X = X.toarray()
-    X = np.r_[X[:n_samples], X]
+    X_rep = np.repeat(X, sw.astype(int), axis=0)
+    ##Need to know number of repitions made in total
-    ##Need to know number of repitions made in total
+    # Inspect the total number of random repetitions so as to adjust the size of
+    # the first cross-validation group accordingly.
-    ##Need to know number of repitions made in total
+    # Inspect the total number of random repetitions so as to adjust the size of
+    # the first cross-validation group accordingly.
+    n_reps = X_rep.shape[0] - X.shape[0]
+    X = X_rep
    if sparse_container is not None:
        X = sparse_container(X)
-    y = np.r_[y[:n_samples], y]
+    y = np.repeat(y, sw.astype(int), axis=0)
    groups = np.r_[
-        np.full(2 * n_samples, 0), np.full(n_samples, 1), np.full(n_samples, 2)
+        np.full(n_reps + n_samples, 0), np.full(n_samples, 1), np.full(n_samples, 2)
    ]
    splits = list(LeaveOneGroupOut().split(X, groups=groups))
-    reg = ElasticNetCV(alphas=alphas, cv=splits, fit_intercept=fit_intercept, **params)
+    reg = ElasticNetCV(cv=splits, fit_intercept=fit_intercept, **params)
    reg.fit(X, y)

    # ensure that we chose meaningful alphas, i.e. not boundaries
-    assert alphas[0] < reg.alpha_ < alphas[-1]
+    assert_allclose(reg_sw.alphas_, reg.alphas_)
    assert reg_sw.alpha_ == reg.alpha_
    assert_allclose(reg_sw.coef_, reg.coef_)
    assert reg_sw.intercept_ == pytest.approx(reg.intercept_)
@@ -1392,6 +1388,26 @@ def test_enet_cv_grid_search(sample_weight):
    assert reg.alpha_ == pytest.approx(gs.best_params_["alpha"])


+@pytest.mark.parametrize("sparseX", [False, True])
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("sample_weight", [np.array([10, 1, 10, 1]), None])
+def test_enet_alpha_max_sample_weight(sparseX, fit_intercept, sample_weight):
+    X = np.array([[3.0, 1.0], [2.0, 5.0], [5.0, 3.0], [1.0, 4.0]])
+    beta = np.array([1, 1])
+    y = X @ beta
+    if sparseX:
+        X = sparse.csc_matrix(X)
+    # Test alpha_max makes coefs zero.
+    reg = ElasticNetCV(n_alphas=1, cv=2, eps=1, fit_intercept=fit_intercept)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_almost_equal(reg.coef_, 0)
+    alpha_max = reg.alpha_
+    # Test smaller alpha makes coefs nonzero.
+    reg = ElasticNet(alpha=0.99 * alpha_max, fit_intercept=fit_intercept)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_array_less(1e-3, np.max(np.abs(reg.coef_)))
+
+
 @pytest.mark.parametrize("fit_intercept", [True, False])
 @pytest.mark.parametrize("l1_ratio", [0, 0.5, 1])
 @pytest.mark.parametrize("precompute", [False, True])