Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

feat: add support for pandas series & data frames as inputs for ml models. #1088

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Oct 23, 2024
Merged

feat: add support for pandas series & data frames as inputs for ml models. #1088

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
9469b43
support pandas dataframes and series as model inputs.
sycai Oct 15, 2024
f9a7c09
polish code and add tests
sycai Oct 15, 2024
48166c9
clean up code
sycai Oct 15, 2024
65c7516
fix type hints
sycai Oct 16, 2024
ee906c0
fix lint
sycai Oct 16, 2024
87773a3
fix a bug that was introduced in the last commit
sycai Oct 16, 2024
d3634a1
Merge branch 'main' into b362723869
sycai Oct 16, 2024
a4599b9
use type alias in type hints and update docs
sycai Oct 16, 2024
d91cbc8
add session parameter in the converters
sycai Oct 16, 2024
89caee6
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Oct 16, 2024
6f0ec17
fix default parameter issue
sycai Oct 16, 2024
5642fed
fix type error
sycai Oct 16, 2024
a9f231f
Merge branch 'main' into b362723869
sycai Oct 21, 2024
9b42dc7
Merge branch 'main' into b362723869
sycai Oct 23, 2024
1b4aa4d
Merge branch 'main' into b362723869
sycai Oct 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions 17 bigframes/ml/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@
"""

import abc
from typing import cast, Optional, TypeVar, Union
from typing import cast, Optional, TypeVar

import bigframes_vendored.sklearn.base

from bigframes.ml import core
import bigframes.ml.utils as utils
import bigframes.pandas as bpd


Expand Down Expand Up @@ -157,8 +158,8 @@ class SupervisedTrainablePredictor(TrainablePredictor):

def fit(
self: _T,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
) -> _T:
return self._fit(X, y)

Expand All @@ -172,8 +173,8 @@ class UnsupervisedTrainablePredictor(TrainablePredictor):

def fit(
self: _T,
X: Union[bpd.DataFrame, bpd.Series],
y: Optional[Union[bpd.DataFrame, bpd.Series]] = None,
X: utils.ArrayType,
y: Optional[utils.ArrayType] = None,
) -> _T:
return self._fit(X, y)

Expand Down Expand Up @@ -243,8 +244,8 @@ def transform(self, X):

def fit_transform(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Optional[Union[bpd.DataFrame, bpd.Series]] = None,
X: utils.ArrayType,
y: Optional[utils.ArrayType] = None,
) -> bpd.DataFrame:
return self.fit(X, y).transform(X)

Expand All @@ -264,6 +265,6 @@ def transform(self, y):

def fit_transform(
self,
y: Union[bpd.DataFrame, bpd.Series],
y: utils.ArrayType,
) -> bpd.DataFrame:
return self.fit(y).transform(y)
18 changes: 11 additions & 7 deletions 18 bigframes/ml/cluster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import bigframes_vendored.sklearn.cluster._kmeans
from google.cloud import bigquery
import pandas as pd

import bigframes
from bigframes.core import log_adapter
Expand Down Expand Up @@ -101,7 +102,7 @@ def _bqml_options(self) -> dict:

def _fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y=None, # ignored
transforms: Optional[List[str]] = None,
) -> KMeans:
Expand All @@ -125,17 +126,20 @@ def cluster_centers_(self) -> bpd.DataFrame:

def predict(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before predict")

(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.predict(X)

def detect_anomalies(
self, X: Union[bpd.DataFrame, bpd.Series], *, contamination: float = 0.1
self,
X: Union[bpd.DataFrame, bpd.Series, pd.DataFrame, pd.Series],
*,
contamination: float = 0.1,
) -> bpd.DataFrame:
"""Detect the anomaly data points of the input.

Expand All @@ -156,7 +160,7 @@ def detect_anomalies(
if not self._bqml_model:
raise RuntimeError("A model must be fitted before detect_anomalies")

(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.detect_anomalies(
X, options={"contamination": contamination}
Expand All @@ -181,12 +185,12 @@ def to_gbq(self, model_name: str, replace: bool = False) -> KMeans:

def score(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y=None, # ignored
) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before score")

(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.evaluate(X)
6 changes: 3 additions & 3 deletions 6 bigframes/ml/compose.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ def _compile_to_sql(

def fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y=None, # ignored
) -> ColumnTransformer:
(X,) = utils.convert_to_dataframe(X)
Expand All @@ -347,11 +347,11 @@ def fit(
self._extract_output_names()
return self

def transform(self, X: Union[bpd.DataFrame, bpd.Series]) -> bpd.DataFrame:
def transform(self, X: utils.ArrayType) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("Must be fitted before transform")

(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

df = self._bqml_model.transform(X)
return typing.cast(
Expand Down
13 changes: 8 additions & 5 deletions 13 bigframes/ml/decomposition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def _bqml_options(self) -> dict:

def _fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y=None,
transforms: Optional[List[str]] = None,
) -> PCA:
Expand Down Expand Up @@ -129,16 +129,19 @@ def explained_variance_ratio_(self) -> bpd.DataFrame:
["principal_component_id", "explained_variance_ratio"]
]

def predict(self, X: Union[bpd.DataFrame, bpd.Series]) -> bpd.DataFrame:
def predict(self, X: utils.ArrayType) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before predict")

(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.predict(X)

def detect_anomalies(
self, X: Union[bpd.DataFrame, bpd.Series], *, contamination: float = 0.1
self,
X: utils.ArrayType,
*,
contamination: float = 0.1,
) -> bpd.DataFrame:
"""Detect the anomaly data points of the input.

Expand All @@ -159,7 +162,7 @@ def detect_anomalies(
if not self._bqml_model:
raise RuntimeError("A model must be fitted before detect_anomalies")

(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.detect_anomalies(
X, options={"contamination": contamination}
Expand Down
60 changes: 30 additions & 30 deletions 60 bigframes/ml/ensemble.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from __future__ import annotations

from typing import Dict, List, Literal, Optional, Union
from typing import Dict, List, Literal, Optional

import bigframes_vendored.sklearn.ensemble._forest
import bigframes_vendored.xgboost.sklearn
Expand Down Expand Up @@ -142,8 +142,8 @@ def _bqml_options(self) -> Dict[str, str | int | bool | float | List[str]]:

def _fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
transforms: Optional[List[str]] = None,
) -> XGBRegressor:
X, y = utils.convert_to_dataframe(X, y)
Expand All @@ -158,24 +158,24 @@ def _fit(

def predict(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before predict")
(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.predict(X)

def score(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
):
X, y = utils.convert_to_dataframe(X, y)

if not self._bqml_model:
raise RuntimeError("A model must be fitted before score")

X, y = utils.convert_to_dataframe(X, y, session=self._bqml_model.session)

input_data = (
X.join(y, how="outer") if (X is not None) and (y is not None) else None
)
Expand Down Expand Up @@ -291,8 +291,8 @@ def _bqml_options(self) -> Dict[str, str | int | bool | float | List[str]]:

def _fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
transforms: Optional[List[str]] = None,
) -> XGBClassifier:
X, y = utils.convert_to_dataframe(X, y)
Expand All @@ -305,22 +305,22 @@ def _fit(
)
return self

def predict(self, X: Union[bpd.DataFrame, bpd.Series]) -> bpd.DataFrame:
def predict(self, X: utils.ArrayType) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before predict")
(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.predict(X)

def score(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
):
if not self._bqml_model:
raise RuntimeError("A model must be fitted before score")

X, y = utils.convert_to_dataframe(X, y)
X, y = utils.convert_to_dataframe(X, y, session=self._bqml_model.session)

input_data = (
X.join(y, how="outer") if (X is not None) and (y is not None) else None
Expand Down Expand Up @@ -427,8 +427,8 @@ def _bqml_options(self) -> Dict[str, str | int | bool | float | List[str]]:

def _fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
transforms: Optional[List[str]] = None,
) -> RandomForestRegressor:
X, y = utils.convert_to_dataframe(X, y)
Expand All @@ -443,18 +443,18 @@ def _fit(

def predict(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before predict")
(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.predict(X)

def score(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
):
"""Calculate evaluation metrics of the model.

Expand All @@ -476,7 +476,7 @@ def score(
if not self._bqml_model:
raise RuntimeError("A model must be fitted before score")

X, y = utils.convert_to_dataframe(X, y)
X, y = utils.convert_to_dataframe(X, y, session=self._bqml_model.session)

input_data = (
X.join(y, how="outer") if (X is not None) and (y is not None) else None
Expand Down Expand Up @@ -583,8 +583,8 @@ def _bqml_options(self) -> Dict[str, str | int | bool | float | List[str]]:

def _fit(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
transforms: Optional[List[str]] = None,
) -> RandomForestClassifier:
X, y = utils.convert_to_dataframe(X, y)
Expand All @@ -599,18 +599,18 @@ def _fit(

def predict(
self,
X: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
) -> bpd.DataFrame:
if not self._bqml_model:
raise RuntimeError("A model must be fitted before predict")
(X,) = utils.convert_to_dataframe(X)
(X,) = utils.convert_to_dataframe(X, session=self._bqml_model.session)

return self._bqml_model.predict(X)

def score(
self,
X: Union[bpd.DataFrame, bpd.Series],
y: Union[bpd.DataFrame, bpd.Series],
X: utils.ArrayType,
y: utils.ArrayType,
):
"""Calculate evaluation metrics of the model.

Expand All @@ -632,7 +632,7 @@ def score(
if not self._bqml_model:
raise RuntimeError("A model must be fitted before score")

X, y = utils.convert_to_dataframe(X, y)
X, y = utils.convert_to_dataframe(X, y, session=self._bqml_model.session)

input_data = (
X.join(y, how="outer") if (X is not None) and (y is not None) else None
Expand Down
Loading
Morty Proxy This is a proxified and sanitized view of the page, visit original site.