From dbc68b3d1f325f80d24a2da5f028b0f653fb0317 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 9 Dec 2020 15:59:52 -0600 Subject: [PATCH 01/15] docs: add GEOGRAPHY data type code samples (#428) * docs: add GEOGRAPHY data type code samples These are added to a separate directory in order to isolate the GeoJSON and WKT dependencies from the other code samples. * skip geography samples in snippets session --- noxfile.py | 8 +- samples/geography/__init__.py | 13 ++ samples/geography/conftest.py | 55 +++++ samples/geography/insert_geojson.py | 49 +++++ samples/geography/insert_geojson_test.py | 20 ++ samples/geography/insert_wkt.py | 49 +++++ samples/geography/insert_wkt_test.py | 20 ++ samples/geography/noxfile.py | 246 +++++++++++++++++++++++ samples/geography/noxfile_config.py | 35 ++++ samples/geography/requirements-test.txt | 2 + samples/geography/requirements.txt | 3 + tests/system.py | 9 +- 12 files changed, 502 insertions(+), 7 deletions(-) create mode 100644 samples/geography/__init__.py create mode 100644 samples/geography/conftest.py create mode 100644 samples/geography/insert_geojson.py create mode 100644 samples/geography/insert_geojson_test.py create mode 100644 samples/geography/insert_wkt.py create mode 100644 samples/geography/insert_wkt_test.py create mode 100644 samples/geography/noxfile.py create mode 100644 samples/geography/noxfile_config.py create mode 100644 samples/geography/requirements-test.txt create mode 100644 samples/geography/requirements.txt diff --git a/noxfile.py b/noxfile.py index 95818d3c8..8523eabb5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -147,7 +147,13 @@ def snippets(session): # Skip tests in samples/snippets, as those are run in a different session # using the nox config from that directory. session.run("py.test", os.path.join("docs", "snippets.py"), *session.posargs) - session.run("py.test", "samples", "--ignore=samples/snippets", *session.posargs) + session.run( + "py.test", + "samples", + "--ignore=samples/snippets", + "--ignore=samples/geography", + *session.posargs, + ) @nox.session(python="3.8") diff --git a/samples/geography/__init__.py b/samples/geography/__init__.py new file mode 100644 index 000000000..c6334245a --- /dev/null +++ b/samples/geography/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/geography/conftest.py b/samples/geography/conftest.py new file mode 100644 index 000000000..265900f5a --- /dev/null +++ b/samples/geography/conftest.py @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import uuid + +from google.cloud import bigquery +import pytest + + +def temp_suffix(): + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" + + +@pytest.fixture(scope="session") +def bigquery_client(): + bigquery_client = bigquery.Client() + return bigquery_client + + +@pytest.fixture(scope="session") +def project_id(bigquery_client): + return bigquery_client.project + + +@pytest.fixture +def dataset_id(bigquery_client): + dataset_id = f"geography_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture +def table_id(bigquery_client, project_id, dataset_id): + table_id = f"{project_id}.{dataset_id}.geography_{temp_suffix()}" + table = bigquery.Table(table_id) + table.schema = [ + bigquery.SchemaField("geo", bigquery.SqlTypeNames.GEOGRAPHY), + ] + bigquery_client.create_table(table) + yield table_id + bigquery_client.delete_table(table_id) diff --git a/samples/geography/insert_geojson.py b/samples/geography/insert_geojson.py new file mode 100644 index 000000000..23f249c15 --- /dev/null +++ b/samples/geography/insert_geojson.py @@ -0,0 +1,49 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def insert_geojson(override_values={}): + # [START bigquery_insert_geojson] + import geojson + from google.cloud import bigquery + + bigquery_client = bigquery.Client() + + # This example uses a table containing a column named "geo" with the + # GEOGRAPHY data type. + table_id = "my-project.my_dataset.my_table" + # [END bigquery_insert_geojson] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + table_id = override_values.get("table_id", table_id) + # [START bigquery_insert_geojson] + + # Use the python-geojson library to generate GeoJSON of a line from LAX to + # JFK airports. Alternatively, you may define GeoJSON data directly, but it + # must be converted to a string before loading it into BigQuery. + my_geography = geojson.LineString([(-118.4085, 33.9416), (-73.7781, 40.6413)]) + rows = [ + # Convert GeoJSON data into a string. + {"geo": geojson.dumps(my_geography)} + ] + + # table already exists and has a column + # named "geo" with data type GEOGRAPHY. + errors = bigquery_client.insert_rows_json(table_id, rows) + if errors: + raise RuntimeError(f"row insert failed: {errors}") + else: + print(f"wrote 1 row to {table_id}") + # [END bigquery_insert_geojson] + return errors diff --git a/samples/geography/insert_geojson_test.py b/samples/geography/insert_geojson_test.py new file mode 100644 index 000000000..5ef15ee13 --- /dev/null +++ b/samples/geography/insert_geojson_test.py @@ -0,0 +1,20 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import insert_geojson + + +def test_insert_geojson(table_id): + errors = insert_geojson.insert_geojson(override_values={"table_id": table_id}) + assert not errors diff --git a/samples/geography/insert_wkt.py b/samples/geography/insert_wkt.py new file mode 100644 index 000000000..1f3d57546 --- /dev/null +++ b/samples/geography/insert_wkt.py @@ -0,0 +1,49 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def insert_wkt(override_values={}): + # [START bigquery_insert_geography_wkt] + from google.cloud import bigquery + import shapely + import shapely.wkt + + bigquery_client = bigquery.Client() + + # This example uses a table containing a column named "geo" with the + # GEOGRAPHY data type. + table_id = "my-project.my_dataset.my_table" + # [END bigquery_insert_geography_wkt] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + table_id = override_values.get("table_id", table_id) + # [START bigquery_insert_geography_wkt] + + # Use the Shapely library to generate WKT of a line from LAX to + # JFK airports. Alternatively, you may define WKT data directly. + my_geography = shapely.LineString([(-118.4085, 33.9416), (-73.7781, 40.6413)]) + rows = [ + # Convert data into a WKT string. + {"geo": shapely.wkt.dumps(my_geography)}, + ] + + # table already exists and has a column + # named "geo" with data type GEOGRAPHY. + errors = bigquery_client.insert_rows_json(table_id, rows) + if errors: + raise RuntimeError(f"row insert failed: {errors}") + else: + print(f"wrote 1 row to {table_id}") + # [END bigquery_insert_geography_wkt] + return errors diff --git a/samples/geography/insert_wkt_test.py b/samples/geography/insert_wkt_test.py new file mode 100644 index 000000000..5ef15ee13 --- /dev/null +++ b/samples/geography/insert_wkt_test.py @@ -0,0 +1,20 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import insert_geojson + + +def test_insert_geojson(table_id): + errors = insert_geojson.insert_geojson(override_values={"table_id": table_id}) + assert not errors diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py new file mode 100644 index 000000000..ab2c49227 --- /dev/null +++ b/samples/geography/noxfile.py @@ -0,0 +1,246 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session): + session.install("black") + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/geography/noxfile_config.py b/samples/geography/noxfile_config.py new file mode 100644 index 000000000..7d2e02346 --- /dev/null +++ b/samples/geography/noxfile_config.py @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be inported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt new file mode 100644 index 000000000..676ff949e --- /dev/null +++ b/samples/geography/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==5.4.3 +mock==4.0.2 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt new file mode 100644 index 000000000..9bd6638d7 --- /dev/null +++ b/samples/geography/requirements.txt @@ -0,0 +1,3 @@ +geojson==2.5.0 +google-cloud-bigquery==2.6.0 +Shapely==1.7.1 diff --git a/tests/system.py b/tests/system.py index d481967d8..185722e83 100644 --- a/tests/system.py +++ b/tests/system.py @@ -2414,9 +2414,8 @@ def test_querying_data_w_timeout(self): query_job = Config.CLIENT.query( """ - SELECT name, SUM(number) AS total_people - FROM `bigquery-public-data.usa_names.usa_1910_current` - GROUP BY name + SELECT COUNT(*) + FROM UNNEST(GENERATE_ARRAY(1,1000000)), UNNEST(GENERATE_ARRAY(1, 10000)) """, location="US", job_config=job_config, @@ -2427,9 +2426,7 @@ def test_querying_data_w_timeout(self): with self.assertRaises(requests.exceptions.Timeout): query_job.done(timeout=0.1) - # Now wait for the result using a more realistic deadline. - query_job.result(timeout=30) - self.assertTrue(query_job.done(timeout=30)) + Config.CLIENT.cancel_job(query_job.job_id, location=query_job.location) @unittest.skipIf(pandas is None, "Requires `pandas`") def test_query_results_to_dataframe(self): From 96a1c5b3c72855ba6ae8c88dfd0cdb02d2faf909 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 9 Dec 2020 16:57:16 -0600 Subject: [PATCH 02/15] docs: fix Shapely import in GEOGRAPHY sample (#431) --- samples/geography/insert_wkt.py | 6 ++++-- samples/geography/insert_wkt_test.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/samples/geography/insert_wkt.py b/samples/geography/insert_wkt.py index 1f3d57546..d7d3accde 100644 --- a/samples/geography/insert_wkt.py +++ b/samples/geography/insert_wkt.py @@ -16,7 +16,7 @@ def insert_wkt(override_values={}): # [START bigquery_insert_geography_wkt] from google.cloud import bigquery - import shapely + import shapely.geometry import shapely.wkt bigquery_client = bigquery.Client() @@ -32,7 +32,9 @@ def insert_wkt(override_values={}): # Use the Shapely library to generate WKT of a line from LAX to # JFK airports. Alternatively, you may define WKT data directly. - my_geography = shapely.LineString([(-118.4085, 33.9416), (-73.7781, 40.6413)]) + my_geography = shapely.geometry.LineString( + [(-118.4085, 33.9416), (-73.7781, 40.6413)] + ) rows = [ # Convert data into a WKT string. {"geo": shapely.wkt.dumps(my_geography)}, diff --git a/samples/geography/insert_wkt_test.py b/samples/geography/insert_wkt_test.py index 5ef15ee13..8bcb62cec 100644 --- a/samples/geography/insert_wkt_test.py +++ b/samples/geography/insert_wkt_test.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import insert_geojson +from . import insert_wkt -def test_insert_geojson(table_id): - errors = insert_geojson.insert_geojson(override_values={"table_id": table_id}) +def test_insert_wkt(table_id): + errors = insert_wkt.insert_wkt(override_values={"table_id": table_id}) assert not errors From dab7af3463457052f75991fb7e532ea719da0129 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 11 Dec 2020 00:04:26 +0100 Subject: [PATCH 03/15] chore(deps): update dependency google-cloud-bigquery to v2.6.1 (#430) --- samples/geography/requirements.txt | 2 +- samples/snippets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 9bd6638d7..3ea0e6e06 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,3 +1,3 @@ geojson==2.5.0 -google-cloud-bigquery==2.6.0 +google-cloud-bigquery==2.6.1 Shapely==1.7.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 1d3cace2b..0b9b69487 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery==2.6.0 +google-cloud-bigquery==2.6.1 google-cloud-bigquery-storage==2.1.0 google-auth-oauthlib==0.4.2 grpcio==1.34.0 From 079b6a162f6929bf801366d92f8daeb3318426c4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 10 Dec 2020 17:05:08 -0600 Subject: [PATCH 04/15] docs: move and refresh view samples (#420) docs: restore old view snippets remove relative imports docs: fix missing space in comment sort imports --- samples/snippets/conftest.py | 27 ++++ samples/snippets/materialized_view.py | 2 +- samples/snippets/materialized_view_test.py | 14 +- samples/snippets/view.py | 164 +++++++++++++++++++++ samples/snippets/view_test.py | 117 +++++++++++++++ 5 files changed, 311 insertions(+), 13 deletions(-) create mode 100644 samples/snippets/conftest.py create mode 100644 samples/snippets/view.py create mode 100644 samples/snippets/view_test.py diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py new file mode 100644 index 000000000..d22a33318 --- /dev/null +++ b/samples/snippets/conftest.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + + +@pytest.fixture(scope="session") +def bigquery_client(): + bigquery_client = bigquery.Client() + return bigquery_client + + +@pytest.fixture(scope="session") +def project_id(bigquery_client): + return bigquery_client.project diff --git a/samples/snippets/materialized_view.py b/samples/snippets/materialized_view.py index d925ec230..429bd98b4 100644 --- a/samples/snippets/materialized_view.py +++ b/samples/snippets/materialized_view.py @@ -25,7 +25,7 @@ def create_materialized_view(override_values={}): # To facilitate testing, we replace values with alternatives # provided by the testing harness. view_id = override_values.get("view_id", view_id) - base_table_id = override_values.get("base_table_id", view_id) + base_table_id = override_values.get("base_table_id", base_table_id) # [START bigquery_create_materialized_view] view = bigquery.Table(view_id) view.mview_query = f""" diff --git a/samples/snippets/materialized_view_test.py b/samples/snippets/materialized_view_test.py index fc3db533c..75c6b2106 100644 --- a/samples/snippets/materialized_view_test.py +++ b/samples/snippets/materialized_view_test.py @@ -23,13 +23,8 @@ def temp_suffix(): - return str(uuid.uuid4()).replace("-", "_") - - -@pytest.fixture(scope="module") -def bigquery_client(): - bigquery_client = bigquery.Client() - return bigquery_client + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @pytest.fixture(autouse=True) @@ -37,11 +32,6 @@ def bigquery_client_patch(monkeypatch, bigquery_client): monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) -@pytest.fixture(scope="module") -def project_id(bigquery_client): - return bigquery_client.project - - @pytest.fixture(scope="module") def dataset_id(bigquery_client): dataset_id = f"mvdataset_{temp_suffix()}" diff --git a/samples/snippets/view.py b/samples/snippets/view.py new file mode 100644 index 000000000..ad3f11717 --- /dev/null +++ b/samples/snippets/view.py @@ -0,0 +1,164 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_view(override_values={}): + # [START bigquery_create_view] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + source_id = "my-project.my_dataset.my_table" + # [END bigquery_create_view] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + source_id = override_values.get("source_id", source_id) + # [START bigquery_create_view] + view = bigquery.Table(view_id) + + # The source table in this example is created from a CSV file in Google + # Cloud Storage located at + # `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains + # 50 US states, while the view returns only those states with names + # starting with the letter 'W'. + view.view_query = f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'W%'" + + # Make an API request to create the view. + view = client.create_table(view) + print(f"Created {view.table_type}: {str(view.reference)}") + # [END bigquery_create_view] + return view + + +def get_view(override_values={}): + # [START bigquery_get_view] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + # [END bigquery_get_view] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + # [START bigquery_get_view] + # Make an API request to get the table resource. + view = client.get_table(view_id) + + # Display view properties + print(f"Retrieved {view.table_type}: {str(view.reference)}") + print(f"View Query:\n{view.view_query}") + # [END bigquery_get_view] + return view + + +def update_view(override_values={}): + # [START bigquery_update_view_query] + from google.cloud import bigquery + + client = bigquery.Client() + + view_id = "my-project.my_dataset.my_view" + source_id = "my-project.my_dataset.my_table" + # [END bigquery_update_view_query] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_id = override_values.get("view_id", view_id) + source_id = override_values.get("source_id", source_id) + # [START bigquery_update_view_query] + view = bigquery.Table(view_id) + + # The source table in this example is created from a CSV file in Google + # Cloud Storage located at + # `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains + # 50 US states, while the view returns only those states with names + # starting with the letter 'M'. + view.view_query = f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'M%'" + + # Make an API request to update the query property of the view. + view = client.update_table(view, ["view_query"]) + print(f"Updated {view.table_type}: {str(view.reference)}") + # [END bigquery_update_view_query] + return view + + +def grant_access(override_values={}): + # [START bigquery_grant_view_access] + from google.cloud import bigquery + + client = bigquery.Client() + + # To use a view, the analyst requires ACLs to both the view and the source + # table. Create an authorized view to allow an analyst to use a view + # without direct access permissions to the source table. + view_dataset_id = "my-project.my_view_dataset" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_dataset_id = override_values.get("view_dataset_id", view_dataset_id) + # [START bigquery_grant_view_access] + # Make an API request to get the view dataset ACLs. + view_dataset = client.get_dataset(view_dataset_id) + + analyst_group_email = "data_analysts@example.com" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + analyst_group_email = override_values.get( + "analyst_group_email", analyst_group_email + ) + # [START bigquery_grant_view_access] + access_entries = view_dataset.access_entries + access_entries.append( + bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) + ) + view_dataset.access_entries = access_entries + + # Make an API request to update the ACLs property of the view dataset. + view_dataset = client.update_dataset(view_dataset, ["access_entries"]) + print(f"Access to view: {view_dataset.access_entries}") + + # Group members of "data_analysts@example.com" now have access to the view, + # but they require access to the source table to use it. To remove this + # restriction, authorize the view to access the source dataset. + source_dataset_id = "my-project.my_source_dataset" + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + source_dataset_id = override_values.get("source_dataset_id", source_dataset_id) + # [START bigquery_grant_view_access] + # Make an API request to set the source dataset ACLs. + source_dataset = client.get_dataset(source_dataset_id) + + view_reference = { + "projectId": "my-project", + "datasetId": "my_view_dataset", + "tableId": "my_authorized_view", + } + # [END bigquery_grant_view_access] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + view_reference = override_values.get("view_reference", view_reference) + # [START bigquery_grant_view_access] + access_entries = source_dataset.access_entries + access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) + source_dataset.access_entries = access_entries + + # Make an API request to update the ACLs property of the source dataset. + source_dataset = client.update_dataset(source_dataset, ["access_entries"]) + print(f"Access to source: {source_dataset.access_entries}") + # [END bigquery_grant_view_access] + return view_dataset, source_dataset diff --git a/samples/snippets/view_test.py b/samples/snippets/view_test.py new file mode 100644 index 000000000..77105b61a --- /dev/null +++ b/samples/snippets/view_test.py @@ -0,0 +1,117 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import uuid + +from google.cloud import bigquery +import pytest + +import view + + +def temp_suffix(): + now = datetime.datetime.now() + return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" + + +@pytest.fixture(autouse=True) +def bigquery_client_patch(monkeypatch, bigquery_client): + monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) + + +@pytest.fixture(scope="module") +def view_dataset_id(bigquery_client, project_id): + dataset_id = f"{project_id}.view_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture(scope="module") +def view_id(bigquery_client, view_dataset_id): + view_id = f"{view_dataset_id}.my_view" + yield view_id + bigquery_client.delete_table(view_id, not_found_ok=True) + + +@pytest.fixture(scope="module") +def source_dataset_id(bigquery_client, project_id): + dataset_id = f"{project_id}.view_{temp_suffix()}" + bigquery_client.create_dataset(dataset_id) + yield dataset_id + bigquery_client.delete_dataset(dataset_id, delete_contents=True) + + +@pytest.fixture(scope="module") +def source_table_id(bigquery_client, source_dataset_id): + source_table_id = f"{source_dataset_id}.us_states" + job_config = bigquery.LoadJobConfig( + schema=[ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ], + skip_leading_rows=1, + ) + load_job = bigquery_client.load_table_from_uri( + "gs://cloud-samples-data/bigquery/us-states/us-states.csv", + source_table_id, + job_config=job_config, + ) + load_job.result() + yield source_table_id + bigquery_client.delete_table(source_table_id, not_found_ok=True) + + +def test_view(capsys, view_id, view_dataset_id, source_table_id, source_dataset_id): + override_values = { + "view_id": view_id, + "source_id": source_table_id, + } + got = view.create_view(override_values) + assert source_table_id in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + + got = view.get_view(override_values) + assert source_table_id in got.view_query + assert "'W%'" in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + assert source_table_id in out + assert "'W%'" in out + + got = view.update_view(override_values) + assert source_table_id in got.view_query + assert "'M%'" in got.view_query + out, _ = capsys.readouterr() + assert view_id in out + + project_id, dataset_id, table_id = view_id.split(".") + override_values = { + "analyst_group_email": "cloud-dpes-bigquery@google.com", + "view_dataset_id": view_dataset_id, + "source_dataset_id": source_dataset_id, + "view_reference": { + "projectId": project_id, + "datasetId": dataset_id, + "tableId": table_id, + }, + } + view_dataset, source_dataset = view.grant_access(override_values) + assert len(view_dataset.access_entries) != 0 + assert len(source_dataset.access_entries) != 0 + out, _ = capsys.readouterr() + assert "cloud-dpes-bigquery@google.com" in out + assert table_id in out From aac33df45f86ddb54f8d68ecafea8184bb009652 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 22 Dec 2020 16:20:48 -0600 Subject: [PATCH 05/15] test: add session to test with nightly dependencies (#449) This should catch errors introduced in the next versions of dependency packages. --- .kokoro/presubmit/prerelease-deps-3.8.cfg | 7 +++++ noxfile.py | 32 +++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .kokoro/presubmit/prerelease-deps-3.8.cfg diff --git a/.kokoro/presubmit/prerelease-deps-3.8.cfg b/.kokoro/presubmit/prerelease-deps-3.8.cfg new file mode 100644 index 000000000..f06806baf --- /dev/null +++ b/.kokoro/presubmit/prerelease-deps-3.8.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index 8523eabb5..f3326d01b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -168,6 +168,38 @@ def cover(session): session.run("coverage", "erase") +@nox.session(python="3.8") +def prerelease_deps(session): + """Run all tests with prerelease versions of dependencies installed. + + https://github.com/googleapis/python-bigquery/issues/95 + """ + # PyArrow prerelease packages are published to an alternative PyPI host. + # https://arrow.apache.org/docs/python/install.html#installing-nightly-packages + session.install( + "--extra-index-url", "https://pypi.fury.io/arrow-nightlies/", "--pre", "pyarrow" + ) + session.install("--pre", "grpcio", "pandas") + session.install( + "mock", + "pytest", + "google-cloud-testutils", + "pytest-cov", + "freezegun", + "IPython", + ) + session.install("-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run("python", "-c", "import pandas; print(pandas.__version__)") + session.run("python", "-c", "import pyarrow; print(pyarrow.__version__)") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run("py.test", "tests") + session.run("py.test", "samples/tests") + + @nox.session(python="3.8") def lint(session): """Run linters. From b1f2f48af8d460c80eda377518df968c300bd893 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 5 Jan 2021 15:37:44 -0700 Subject: [PATCH 06/15] chore: add constraints file (#456) * chore: add constraints file * chore: add constraints file * chore: add constraints file * chore: add constraints file --- testing/constraints-3.10.txt | 0 testing/constraints-3.11.txt | 0 testing/constraints-3.6.txt | 33 ++++++++++++++++++++++++--------- testing/constraints-3.9.txt | 0 4 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 testing/constraints-3.10.txt create mode 100644 testing/constraints-3.11.txt create mode 100644 testing/constraints-3.9.txt diff --git a/testing/constraints-3.10.txt b/testing/constraints-3.10.txt new file mode 100644 index 000000000..e69de29bb diff --git a/testing/constraints-3.11.txt b/testing/constraints-3.11.txt new file mode 100644 index 000000000..e69de29bb diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 91a507a5c..fe2bcfda7 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -1,16 +1,31 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List *all* library dependencies and extras in this file. +# Pin the version to the lower bound. +# +# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", +# Then this file should have foo==1.14.0 google-api-core==1.23.0 -google-cloud-bigquery-storage==2.0.0 +proto-plus==1.10.0 google-cloud-core==1.4.1 google-resumable-media==0.6.0 +six==1.13.0 +protobuf==3.12.0 +google-cloud-bigquery-storage==2.0.0 grpcio==1.32.0 -ipython==5.5 -libcst==0.2.5 -llvmlite==0.34.0 -# pandas 0.23.0 is the first version to work with pyarrow to_pandas. +pyarrow==1.0.0 pandas==0.23.0 -protobuf == 3.12.0 -proto-plus==1.10.0 pyarrow==1.0.0 -python-snappy==0.5.4 -six==1.13.0 tqdm==4.7.4 +opentelemetry-api==0.11b0 +opentelemetry-sdk==0.11b0 +opentelemetry-instrumentation==0.11b0 +google-cloud-bigquery-storage==2.0.0 +grpcio==1.32.0 +pyarrow==1.0.0 +opentelemetry-api==0.11b0 +opentelemetry-sdk==0.11b0 +opentelemetry-instrumentation==0.11b0 +pandas==0.23.0 +pyarrow==1.0.0 +tqdm==4.7.4 \ No newline at end of file diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt new file mode 100644 index 000000000..e69de29bb From c2e70603d2946eea244c371b5bf6758c2da8d6b3 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 7 Jan 2021 00:06:40 +0100 Subject: [PATCH 07/15] chore(deps): update dependency pytz to v2020.5 (#452) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 0b9b69487..5cda34214 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -7,4 +7,4 @@ ipython==7.17.0; python_version >= '3.7' matplotlib==3.3.3 pandas==1.1.5 pyarrow==2.0.0 -pytz==2020.4 +pytz==2020.5 From 0337ea0bde966c3ccb94960493a6fa6f2bee49b4 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 7 Jan 2021 00:26:11 +0100 Subject: [PATCH 08/15] chore(deps): update dependency pandas to v1.2.0 (#454) [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [pandas](https://pandas.pydata.org) ([source](https://togithub.com/pandas-dev/pandas)) | minor | `==1.1.5` -> `==1.2.0` | --- ### Release Notes
pandas-dev/pandas ### [`v1.2.0`](https://togithub.com/pandas-dev/pandas/releases/v1.2.0) [Compare Source](https://togithub.com/pandas-dev/pandas/compare/v1.1.5...v1.2.0) This release includes some new features, bug fixes, and performance improvements. We recommend that all users upgrade to this version. See the [full whatsnew](https://pandas.pydata.org/pandas-docs/version/1.2.0/whatsnew/v1.2.0.html) for a list of all the changes. The release will be available on the defaults and conda-forge channels: conda install -c conda-forge pandas Or via PyPI: python3 -m pip install --upgrade pandas Please report any issues with the release on the [pandas issue tracker](https://togithub.com/pandas-dev/pandas/issues).
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-bigquery). --- samples/snippets/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 5cda34214..208eb4526 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -5,6 +5,7 @@ grpcio==1.34.0 ipython==7.16.1; python_version < '3.7' ipython==7.17.0; python_version >= '3.7' matplotlib==3.3.3 -pandas==1.1.5 +pandas==1.1.5; python_version < '3.7' +pandas==1.2.0; python_version >= '3.7' pyarrow==2.0.0 pytz==2020.5 From d01d199839a13157e8bb290248c4a2e86916cc0c Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Fri, 8 Jan 2021 13:29:22 -0700 Subject: [PATCH 09/15] ci: use python3 instead of python3.6 in build.sh (#425) * ci: skip docfx in main 'Kokoro' presubmit * fix: specify default sessions in noxfile * fix: use python3 instead of 3.6 * fix: add NOX_SESSION to pass down envvars * fix: remove quotes arround sessions Co-authored-by: Tim Swast --- .kokoro/build.sh | 10 +++++----- .trampolinerc | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index cb81a05f8..058f363e1 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -34,16 +34,16 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") # Remove old nox -python3.6 -m pip uninstall --yes --quiet nox-automation +python3 -m pip uninstall --yes --quiet nox-automation # Install nox -python3.6 -m pip install --upgrade --quiet nox -python3.6 -m nox --version +python3 -m pip install --upgrade --quiet nox +python3 -m nox --version # If NOX_SESSION is set, it only runs the specified session, # otherwise run all the sessions. if [[ -n "${NOX_SESSION:-}" ]]; then - python3.6 -m nox -s "${NOX_SESSION:-}" + python3 -m nox -s ${NOX_SESSION:-} else - python3.6 -m nox + python3 -m nox fi diff --git a/.trampolinerc b/.trampolinerc index 995ee2911..c7d663ae9 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -18,12 +18,14 @@ required_envvars+=( "STAGING_BUCKET" "V2_STAGING_BUCKET" + "NOX_SESSION" ) # Add env vars which are passed down into the container here. pass_down_envvars+=( "STAGING_BUCKET" "V2_STAGING_BUCKET" + "NOX_SESSION" ) # Prevent unintentional override on the default image. From 0023d193d36e473095ecaf8a9b9456fb731d583b Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 8 Jan 2021 22:20:21 +0100 Subject: [PATCH 10/15] chore: remove six dependency (#461) * chore: remove six dependency * Remove now-redundant self argument --- google/cloud/bigquery/_helpers.py | 3 +- google/cloud/bigquery/_pandas_helpers.py | 5 +- google/cloud/bigquery/client.py | 5 +- google/cloud/bigquery/dataset.py | 15 ++-- google/cloud/bigquery/dbapi/_helpers.py | 10 ++- google/cloud/bigquery/dbapi/cursor.py | 4 +- google/cloud/bigquery/enums.py | 4 +- google/cloud/bigquery/job/base.py | 40 +++++------ google/cloud/bigquery/job/query.py | 5 +- google/cloud/bigquery/magics/magics.py | 12 ++-- google/cloud/bigquery/model.py | 5 +- google/cloud/bigquery/routine.py | 3 +- google/cloud/bigquery/schema.py | 4 +- google/cloud/bigquery/table.py | 18 +++-- samples/load_table_uri_truncate_avro.py | 4 +- samples/load_table_uri_truncate_csv.py | 4 +- samples/load_table_uri_truncate_json.py | 4 +- samples/load_table_uri_truncate_orc.py | 4 +- samples/load_table_uri_truncate_parquet.py | 4 +- .../tests/test_copy_table_multiple_source.py | 4 +- setup.py | 1 - tests/system.py | 26 ++++--- tests/unit/job/test_base.py | 6 +- tests/unit/job/test_query.py | 6 +- tests/unit/test__helpers.py | 5 +- tests/unit/test__http.py | 12 ++-- tests/unit/test_client.py | 72 +++++++++---------- tests/unit/test_dbapi__helpers.py | 8 +-- tests/unit/test_dbapi_connection.py | 5 +- tests/unit/test_dbapi_cursor.py | 7 +- tests/unit/test_opentelemetry_tracing.py | 6 +- tests/unit/test_table.py | 27 ++----- 32 files changed, 150 insertions(+), 188 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 100136108..6b66a3020 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -18,7 +18,6 @@ import datetime import decimal import re -import six from google.cloud._helpers import UTC from google.cloud._helpers import _date_from_iso8601_date @@ -451,7 +450,7 @@ def _record_field_to_json(fields, row_value): for field_name in not_processed: value = row_value[field_name] if value is not None: - record[field_name] = six.text_type(value) + record[field_name] = str(value) return record diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index 7774ce26b..162c58b4b 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -17,10 +17,9 @@ import concurrent.futures import functools import logging +import queue import warnings -import six -from six.moves import queue try: import pandas @@ -738,7 +737,7 @@ def download_dataframe_bqstorage( def dataframe_to_json_generator(dataframe): for row in dataframe.itertuples(index=False, name=None): output = {} - for column, value in six.moves.zip(dataframe.columns, row): + for column, value in zip(dataframe.columns, row): # Omit NaN values. if value != value: continue diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 28cac64ad..19693c9ff 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -34,7 +34,6 @@ import pyarrow except ImportError: # pragma: NO COVER pyarrow = None -import six from google import resumable_media from google.resumable_media.requests import MultipartUpload @@ -2017,7 +2016,7 @@ def load_table_from_uri( job_ref = job._JobReference(job_id, project=project, location=location) - if isinstance(source_uris, six.string_types): + if isinstance(source_uris, str): source_uris = [source_uris] destination = _table_arg_to_table_ref(destination, default_project=self.project) @@ -2779,7 +2778,7 @@ def extract_table( ) ) - if isinstance(destination_uris, six.string_types): + if isinstance(destination_uris, str): destination_uris = [destination_uris] if job_config: diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index ce07c8048..2d3a4755f 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -16,7 +16,6 @@ from __future__ import absolute_import -import six import copy import google.cloud._helpers @@ -260,9 +259,9 @@ class DatasetReference(object): """ def __init__(self, project, dataset_id): - if not isinstance(project, six.string_types): + if not isinstance(project, str): raise ValueError("Pass a string for project") - if not isinstance(dataset_id, six.string_types): + if not isinstance(dataset_id, str): raise ValueError("Pass a string for dataset_id") self._project = project self._dataset_id = dataset_id @@ -407,7 +406,7 @@ class Dataset(object): } def __init__(self, dataset_ref): - if isinstance(dataset_ref, six.string_types): + if isinstance(dataset_ref, str): dataset_ref = DatasetReference.from_string(dataset_ref) self._properties = {"datasetReference": dataset_ref.to_api_repr(), "labels": {}} @@ -544,7 +543,7 @@ def default_table_expiration_ms(self): @default_table_expiration_ms.setter def default_table_expiration_ms(self, value): - if not isinstance(value, six.integer_types) and value is not None: + if not isinstance(value, int) and value is not None: raise ValueError("Pass an integer, or None") self._properties["defaultTableExpirationMs"] = _helpers._str_or_none(value) @@ -560,7 +559,7 @@ def description(self): @description.setter def description(self, value): - if not isinstance(value, six.string_types) and value is not None: + if not isinstance(value, str) and value is not None: raise ValueError("Pass a string, or None") self._properties["description"] = value @@ -576,7 +575,7 @@ def friendly_name(self): @friendly_name.setter def friendly_name(self, value): - if not isinstance(value, six.string_types) and value is not None: + if not isinstance(value, str) and value is not None: raise ValueError("Pass a string, or None") self._properties["friendlyName"] = value @@ -592,7 +591,7 @@ def location(self): @location.setter def location(self, value): - if not isinstance(value, six.string_types) and value is not None: + if not isinstance(value, str) and value is not None: raise ValueError("Pass a string, or None") self._properties["location"] = value diff --git a/google/cloud/bigquery/dbapi/_helpers.py b/google/cloud/bigquery/dbapi/_helpers.py index fdf4e17c3..95b5869e5 100644 --- a/google/cloud/bigquery/dbapi/_helpers.py +++ b/google/cloud/bigquery/dbapi/_helpers.py @@ -19,8 +19,6 @@ import functools import numbers -import six - from google.cloud import bigquery from google.cloud.bigquery import table from google.cloud.bigquery.dbapi import exceptions @@ -132,7 +130,7 @@ def to_query_parameters_dict(parameters): """ result = [] - for name, value in six.iteritems(parameters): + for name, value in parameters.items(): if isinstance(value, collections_abc.Mapping): raise NotImplementedError( "STRUCT-like parameter values are not supported " @@ -187,9 +185,9 @@ def bigquery_scalar_type(value): return "FLOAT64" elif isinstance(value, decimal.Decimal): return "NUMERIC" - elif isinstance(value, six.text_type): + elif isinstance(value, str): return "STRING" - elif isinstance(value, six.binary_type): + elif isinstance(value, bytes): return "BYTES" elif isinstance(value, datetime.datetime): return "DATETIME" if value.tzinfo is None else "TIMESTAMP" @@ -215,7 +213,7 @@ def array_like(value): bool: ``True`` if the value is considered array-like, ``False`` otherwise. """ return isinstance(value, collections_abc.Sequence) and not isinstance( - value, (six.text_type, six.binary_type, bytearray) + value, (str, bytes, bytearray) ) diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index f48b47c12..e90bcc2c0 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -19,8 +19,6 @@ import copy import logging -import six - from google.cloud.bigquery import job from google.cloud.bigquery.dbapi import _helpers from google.cloud.bigquery.dbapi import exceptions @@ -289,7 +287,7 @@ def fetchone(self): """ self._try_fetch() try: - return six.next(self._query_data) + return next(self._query_data) except StopIteration: return None diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 3f72333af..2268808fd 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -15,7 +15,7 @@ import re import enum -import six +import itertools from google.cloud.bigquery_v2 import types as gapic_types @@ -178,7 +178,7 @@ def _make_sql_scalars_enum(): ) new_doc = "\n".join( - six.moves.filterfalse(skip_pattern.search, orig_doc.splitlines()) + itertools.filterfalse(skip_pattern.search, orig_doc.splitlines()) ) new_enum.__doc__ = "An Enum of scalar SQL types.\n" + new_doc diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 2f4ae1460..3c601f072 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -15,11 +15,11 @@ """Base classes and helpers for job classes.""" import copy +import http import threading from google.api_core import exceptions import google.api_core.future.polling -from six.moves import http_client from google.cloud.bigquery import _helpers from google.cloud.bigquery.retry import DEFAULT_RETRY @@ -28,24 +28,24 @@ _DONE_STATE = "DONE" _STOPPED_REASON = "stopped" _ERROR_REASON_TO_EXCEPTION = { - "accessDenied": http_client.FORBIDDEN, - "backendError": http_client.INTERNAL_SERVER_ERROR, - "billingNotEnabled": http_client.FORBIDDEN, - "billingTierLimitExceeded": http_client.BAD_REQUEST, - "blocked": http_client.FORBIDDEN, - "duplicate": http_client.CONFLICT, - "internalError": http_client.INTERNAL_SERVER_ERROR, - "invalid": http_client.BAD_REQUEST, - "invalidQuery": http_client.BAD_REQUEST, - "notFound": http_client.NOT_FOUND, - "notImplemented": http_client.NOT_IMPLEMENTED, - "quotaExceeded": http_client.FORBIDDEN, - "rateLimitExceeded": http_client.FORBIDDEN, - "resourceInUse": http_client.BAD_REQUEST, - "resourcesExceeded": http_client.BAD_REQUEST, - "responseTooLarge": http_client.FORBIDDEN, - "stopped": http_client.OK, - "tableUnavailable": http_client.BAD_REQUEST, + "accessDenied": http.client.FORBIDDEN, + "backendError": http.client.INTERNAL_SERVER_ERROR, + "billingNotEnabled": http.client.FORBIDDEN, + "billingTierLimitExceeded": http.client.BAD_REQUEST, + "blocked": http.client.FORBIDDEN, + "duplicate": http.client.CONFLICT, + "internalError": http.client.INTERNAL_SERVER_ERROR, + "invalid": http.client.BAD_REQUEST, + "invalidQuery": http.client.BAD_REQUEST, + "notFound": http.client.NOT_FOUND, + "notImplemented": http.client.NOT_IMPLEMENTED, + "quotaExceeded": http.client.FORBIDDEN, + "rateLimitExceeded": http.client.FORBIDDEN, + "resourceInUse": http.client.BAD_REQUEST, + "resourcesExceeded": http.client.BAD_REQUEST, + "responseTooLarge": http.client.FORBIDDEN, + "stopped": http.client.OK, + "tableUnavailable": http.client.BAD_REQUEST, } @@ -66,7 +66,7 @@ def _error_result_to_exception(error_result): """ reason = error_result.get("reason") status_code = _ERROR_REASON_TO_EXCEPTION.get( - reason, http_client.INTERNAL_SERVER_ERROR + reason, http.client.INTERNAL_SERVER_ERROR ) return exceptions.from_http_status( status_code, error_result.get("message", ""), errors=[error_result] diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 9e8908613..d87f87f52 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -20,7 +20,6 @@ from google.api_core import exceptions import requests -import six from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetListItem @@ -192,7 +191,7 @@ def default_dataset(self, value): self._set_sub_prop("defaultDataset", None) return - if isinstance(value, six.string_types): + if isinstance(value, str): value = DatasetReference.from_string(value) if isinstance(value, (Dataset, DatasetListItem)): @@ -1168,7 +1167,7 @@ def result( exc.query_job = self raise except requests.exceptions.Timeout as exc: - six.raise_from(concurrent.futures.TimeoutError, exc) + raise concurrent.futures.TimeoutError from exc # If the query job is complete but there are no query results, this was # special job, such as a DDL query. Return an empty result set to diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index f04a6364a..8f343ddcc 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -153,8 +153,6 @@ except ImportError: # pragma: NO COVER raise ImportError("This module can only be loaded in IPython.") -import six - from google.api_core import client_info from google.api_core import client_options from google.api_core.exceptions import NotFound @@ -577,16 +575,16 @@ def _cell_magic(line, query): "--params is not a correctly formatted JSON string or a JSON " "serializable dictionary" ) - six.raise_from(rebranded_error, exc) + raise rebranded_error from exc except lap.exceptions.DuplicateQueryParamsError as exc: rebranded_error = ValueError("Duplicate --params option.") - six.raise_from(rebranded_error, exc) + raise rebranded_error from exc except lap.exceptions.ParseError as exc: rebranded_error = ValueError( "Unrecognized input, are option values correct? " "Error details: {}".format(exc.args[0]) ) - six.raise_from(rebranded_error, exc) + raise rebranded_error from exc args = magic_arguments.parse_argstring(_cell_magic, rest_of_args) @@ -768,7 +766,7 @@ def _make_bqstorage_client(use_bqstorage_api, credentials, client_options): "to use it. Alternatively, use the classic REST API by specifying " "the --use_rest_api magic option." ) - six.raise_from(customized_error, err) + raise customized_error from err try: from google.api_core.gapic_v1 import client_info as gapic_client_info @@ -776,7 +774,7 @@ def _make_bqstorage_client(use_bqstorage_api, credentials, client_options): customized_error = ImportError( "Install the grpcio package to use the BigQuery Storage API." ) - six.raise_from(customized_error, err) + raise customized_error from err return bigquery_storage.BigQueryReadClient( credentials=credentials, diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 0f5d8f83b..55846bd1a 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -19,7 +19,6 @@ import copy from google.protobuf import json_format -import six import google.cloud._helpers from google.api_core import datetime_helpers @@ -63,7 +62,7 @@ def __init__(self, model_ref): # buffer classes do not. self._properties = {} - if isinstance(model_ref, six.string_types): + if isinstance(model_ref, str): model_ref = ModelReference.from_string(model_ref) if model_ref: @@ -455,7 +454,7 @@ def _model_arg_to_model_ref(value, default_project=None): This function keeps ModelReference and other kinds of objects unchanged. """ - if isinstance(value, six.string_types): + if isinstance(value, str): return ModelReference.from_string(value, default_project=default_project) if isinstance(value, Model): return value.reference diff --git a/google/cloud/bigquery/routine.py b/google/cloud/bigquery/routine.py index 6a0ed9fb0..f26f20886 100644 --- a/google/cloud/bigquery/routine.py +++ b/google/cloud/bigquery/routine.py @@ -17,7 +17,6 @@ """Define resources for the BigQuery Routines API.""" from google.protobuf import json_format -import six import google.cloud._helpers from google.cloud.bigquery import _helpers @@ -54,7 +53,7 @@ class Routine(object): } def __init__(self, routine_ref, **kwargs): - if isinstance(routine_ref, six.string_types): + if isinstance(routine_ref, str): routine_ref = RoutineReference.from_string(routine_ref) self._properties = {"routineReference": routine_ref.to_api_repr()} diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 8ae0a3a85..c76aded02 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -14,7 +14,7 @@ """Schemas for BigQuery tables / queries.""" -from six.moves import collections_abc +import collections from google.cloud.bigquery_v2 import types @@ -318,7 +318,7 @@ def _to_schema_fields(schema): instance or a compatible mapping representation of the field. """ for field in schema: - if not isinstance(field, (SchemaField, collections_abc.Mapping)): + if not isinstance(field, (SchemaField, collections.abc.Mapping)): raise ValueError( "Schema items must either be fields or compatible " "mapping representations." diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 6daccf518..a2366b806 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -24,8 +24,6 @@ import pytz import warnings -import six - try: import pandas except ImportError: # pragma: NO COVER @@ -657,7 +655,7 @@ def description(self): @description.setter def description(self, value): - if not isinstance(value, six.string_types) and value is not None: + if not isinstance(value, str) and value is not None: raise ValueError("Pass a string, or None") self._properties["description"] = value @@ -694,7 +692,7 @@ def friendly_name(self): @friendly_name.setter def friendly_name(self, value): - if not isinstance(value, six.string_types) and value is not None: + if not isinstance(value, str) and value is not None: raise ValueError("Pass a string, or None") self._properties["friendlyName"] = value @@ -721,7 +719,7 @@ def view_query(self): @view_query.setter def view_query(self, value): - if not isinstance(value, six.string_types): + if not isinstance(value, str): raise ValueError("Pass a string") _helpers._set_sub_prop(self._properties, ["view", "query"], value) view = self._properties["view"] @@ -1244,7 +1242,7 @@ def keys(self): >>> list(Row(('a', 'b'), {'x': 0, 'y': 1}).keys()) ['x', 'y'] """ - return six.iterkeys(self._xxx_field_to_index) + return self._xxx_field_to_index.keys() def items(self): """Return items as ``(key, value)`` pairs. @@ -1258,7 +1256,7 @@ def items(self): >>> list(Row(('a', 'b'), {'x': 0, 'y': 1}).items()) [('x', 'a'), ('y', 'b')] """ - for key, index in six.iteritems(self._xxx_field_to_index): + for key, index in self._xxx_field_to_index.items(): yield (key, copy.deepcopy(self._xxx_values[index])) def get(self, key, default=None): @@ -1308,7 +1306,7 @@ def __len__(self): return len(self._xxx_values) def __getitem__(self, key): - if isinstance(key, six.string_types): + if isinstance(key, str): value = self._xxx_field_to_index.get(key) if value is None: raise KeyError("no row field {!r}".format(key)) @@ -2293,7 +2291,7 @@ def _table_arg_to_table_ref(value, default_project=None): This function keeps TableReference and other kinds of objects unchanged. """ - if isinstance(value, six.string_types): + if isinstance(value, str): value = TableReference.from_string(value, default_project=default_project) if isinstance(value, (Table, TableListItem)): value = value.reference @@ -2305,7 +2303,7 @@ def _table_arg_to_table(value, default_project=None): This function keeps Table and other kinds of objects unchanged. """ - if isinstance(value, six.string_types): + if isinstance(value, str): value = TableReference.from_string(value, default_project=default_project) if isinstance(value, TableReference): value = Table(value) diff --git a/samples/load_table_uri_truncate_avro.py b/samples/load_table_uri_truncate_avro.py index 98a791477..1aa0aa49c 100644 --- a/samples/load_table_uri_truncate_avro.py +++ b/samples/load_table_uri_truncate_avro.py @@ -16,7 +16,7 @@ def load_table_uri_truncate_avro(table_id): # [START bigquery_load_table_gcs_avro_truncate] - import six + import io from google.cloud import bigquery @@ -33,7 +33,7 @@ def load_table_uri_truncate_avro(table_id): ], ) - body = six.BytesIO(b"Washington,WA") + body = io.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_id, job_config=job_config).result() previous_rows = client.get_table(table_id).num_rows assert previous_rows > 0 diff --git a/samples/load_table_uri_truncate_csv.py b/samples/load_table_uri_truncate_csv.py index 73de7a8c1..198cdc281 100644 --- a/samples/load_table_uri_truncate_csv.py +++ b/samples/load_table_uri_truncate_csv.py @@ -16,7 +16,7 @@ def load_table_uri_truncate_csv(table_id): # [START bigquery_load_table_gcs_csv_truncate] - import six + import io from google.cloud import bigquery @@ -33,7 +33,7 @@ def load_table_uri_truncate_csv(table_id): ], ) - body = six.BytesIO(b"Washington,WA") + body = io.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_id, job_config=job_config).result() previous_rows = client.get_table(table_id).num_rows assert previous_rows > 0 diff --git a/samples/load_table_uri_truncate_json.py b/samples/load_table_uri_truncate_json.py index a30fae736..d67d93e7b 100644 --- a/samples/load_table_uri_truncate_json.py +++ b/samples/load_table_uri_truncate_json.py @@ -16,7 +16,7 @@ def load_table_uri_truncate_json(table_id): # [START bigquery_load_table_gcs_json_truncate] - import six + import io from google.cloud import bigquery @@ -33,7 +33,7 @@ def load_table_uri_truncate_json(table_id): ], ) - body = six.BytesIO(b"Washington,WA") + body = io.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_id, job_config=job_config).result() previous_rows = client.get_table(table_id).num_rows assert previous_rows > 0 diff --git a/samples/load_table_uri_truncate_orc.py b/samples/load_table_uri_truncate_orc.py index 18f963be2..90543b791 100644 --- a/samples/load_table_uri_truncate_orc.py +++ b/samples/load_table_uri_truncate_orc.py @@ -16,7 +16,7 @@ def load_table_uri_truncate_orc(table_id): # [START bigquery_load_table_gcs_orc_truncate] - import six + import io from google.cloud import bigquery @@ -33,7 +33,7 @@ def load_table_uri_truncate_orc(table_id): ], ) - body = six.BytesIO(b"Washington,WA") + body = io.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_id, job_config=job_config).result() previous_rows = client.get_table(table_id).num_rows assert previous_rows > 0 diff --git a/samples/load_table_uri_truncate_parquet.py b/samples/load_table_uri_truncate_parquet.py index 28692d840..e036fc180 100644 --- a/samples/load_table_uri_truncate_parquet.py +++ b/samples/load_table_uri_truncate_parquet.py @@ -16,7 +16,7 @@ def load_table_uri_truncate_parquet(table_id): # [START bigquery_load_table_gcs_parquet_truncate] - import six + import io from google.cloud import bigquery @@ -33,7 +33,7 @@ def load_table_uri_truncate_parquet(table_id): ], ) - body = six.BytesIO(b"Washington,WA") + body = io.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_id, job_config=job_config).result() previous_rows = client.get_table(table_id).num_rows assert previous_rows > 0 diff --git a/samples/tests/test_copy_table_multiple_source.py b/samples/tests/test_copy_table_multiple_source.py index 45c6d34f5..5bc4668b0 100644 --- a/samples/tests/test_copy_table_multiple_source.py +++ b/samples/tests/test_copy_table_multiple_source.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six +import io from google.cloud import bigquery from .. import copy_table_multiple_source @@ -32,7 +32,7 @@ def test_copy_table_multiple_source(capsys, random_table_id, random_dataset_id, bigquery.SchemaField("post_abbr", "STRING"), ] ) - body = six.BytesIO(data) + body = io.BytesIO(data) client.load_table_from_file( body, table_ref, location="US", job_config=job_config ).result() diff --git a/setup.py b/setup.py index 5f4e506eb..fcafddbd2 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ "proto-plus >= 1.10.0", "google-cloud-core >= 1.4.1, < 2.0dev", "google-resumable-media >= 0.6.0, < 2.0dev", - "six >=1.13.0,< 2.0.0dev", "protobuf >= 3.12.0", ] extras = { diff --git a/tests/system.py b/tests/system.py index 185722e83..bfe54b7df 100644 --- a/tests/system.py +++ b/tests/system.py @@ -18,6 +18,7 @@ import csv import datetime import decimal +import io import json import operator import os @@ -27,7 +28,6 @@ import re import requests -import six import psutil import pytest import pytz @@ -54,7 +54,7 @@ pyarrow = None try: import IPython - from IPython.utils import io + from IPython.utils import io as ipython_io from IPython.testing import tools from IPython.terminal import interactiveshell except ImportError: # pragma: NO COVER @@ -219,7 +219,7 @@ def test_get_service_account_email(self): got = client.get_service_account_email() - self.assertIsInstance(got, six.text_type) + self.assertIsInstance(got, str) self.assertIn("@", got) def _create_bucket(self, bucket_name, location=None): @@ -598,7 +598,7 @@ def test_update_table_schema(self): @staticmethod def _fetch_single_page(table, selected_fields=None): iterator = Config.CLIENT.list_rows(table, selected_fields=selected_fields) - page = six.next(iterator.pages) + page = next(iterator.pages) return list(page) def _create_table_many_columns(self, rowcount): @@ -1415,7 +1415,7 @@ def test_load_table_from_file_w_explicit_location(self): self._create_bucket(bucket_name, location="eu") # Create a temporary dataset & table in the EU. - table_bytes = six.BytesIO(b"a,3\nb,2\nc,1\n") + table_bytes = io.BytesIO(b"a,3\nb,2\nc,1\n") client = Config.CLIENT dataset = self.temp_dataset(_make_dataset_id("eu_load_file"), location="EU") table_ref = dataset.table("letters") @@ -2444,7 +2444,7 @@ def test_query_results_to_dataframe(self): self.assertEqual(list(df), column_names) # verify the column names exp_datatypes = { "id": int, - "author": six.text_type, + "author": str, "time_ts": pandas.Timestamp, "dead": bool, } @@ -2477,7 +2477,7 @@ def test_query_results_to_dataframe_w_bqstorage(self): self.assertEqual(list(df), column_names) exp_datatypes = { "id": int, - "author": six.text_type, + "author": str, "time_ts": pandas.Timestamp, "dead": bool, } @@ -2572,9 +2572,7 @@ def test_insert_rows_from_dataframe(self): assert len(row_tuples) == len(expected) for row, expected_row in zip(row_tuples, expected): - six.assertCountEqual( - self, row, expected_row - ) # column order does not matter + self.assertCountEqual(row, expected_row) # column order does not matter def test_insert_rows_nested_nested(self): # See #2951 @@ -2780,7 +2778,7 @@ def test_nested_table_to_arrow(self): {"string_col": "Some value", "record_col": record, "float_col": 3.14} ] rows = [json.dumps(row) for row in to_insert] - body = six.BytesIO("{}\n".format("\n".join(rows)).encode("ascii")) + body = io.BytesIO("{}\n".format("\n".join(rows)).encode("ascii")) table_id = "test_table" dataset = self.temp_dataset(_make_dataset_id("nested_df")) table = dataset.table(table_id) @@ -2858,7 +2856,7 @@ def test_nested_table_to_dataframe(self): } ] rows = [json.dumps(row) for row in to_insert] - body = six.BytesIO("{}\n".format("\n".join(rows)).encode("ascii")) + body = io.BytesIO("{}\n".format("\n".join(rows)).encode("ascii")) table_id = "test_table" dataset = self.temp_dataset(_make_dataset_id("nested_df")) table = dataset.table(table_id) @@ -2923,7 +2921,7 @@ def test_list_rows_page_size(self): schema = [SF("string_col", "STRING", mode="NULLABLE")] to_insert = [{"string_col": "item%d" % i} for i in range(num_items)] rows = [json.dumps(row) for row in to_insert] - body = six.BytesIO("{}\n".format("\n".join(rows)).encode("ascii")) + body = io.BytesIO("{}\n".format("\n".join(rows)).encode("ascii")) table_id = "test_table" dataset = self.temp_dataset(_make_dataset_id("nested_df")) @@ -2997,7 +2995,7 @@ def test_bigquery_magic(): ORDER BY view_count DESC LIMIT 10 """ - with io.capture_output() as captured: + with ipython_io.capture_output() as captured: result = ip.run_cell_magic("bigquery", "--use_rest_api", sql) conn_count_end = len(current_process.connections()) diff --git a/tests/unit/job/test_base.py b/tests/unit/job/test_base.py index 12e2d4b8b..478e30e6f 100644 --- a/tests/unit/job/test_base.py +++ b/tests/unit/job/test_base.py @@ -13,12 +13,12 @@ # limitations under the License. import copy +import http import unittest from google.api_core import exceptions import google.api_core.retry import mock -from six.moves import http_client from .helpers import _make_client from .helpers import _make_connection @@ -35,14 +35,14 @@ def _call_fut(self, *args, **kwargs): def test_simple(self): error_result = {"reason": "invalid", "message": "bad request"} exception = self._call_fut(error_result) - self.assertEqual(exception.code, http_client.BAD_REQUEST) + self.assertEqual(exception.code, http.client.BAD_REQUEST) self.assertTrue(exception.message.startswith("bad request")) self.assertIn(error_result, exception.errors) def test_missing_reason(self): error_result = {} exception = self._call_fut(error_result) - self.assertEqual(exception.code, http_client.INTERNAL_SERVER_ERROR) + self.assertEqual(exception.code, http.client.INTERNAL_SERVER_ERROR) class Test_JobReference(unittest.TestCase): diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 0567b59cd..579a841d1 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -14,6 +14,7 @@ import concurrent import copy +import http import textwrap import freezegun @@ -21,7 +22,6 @@ import google.api_core.retry import mock import requests -from six.moves import http_client from google.cloud.bigquery.client import _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS import google.cloud.bigquery.query @@ -1210,7 +1210,7 @@ def test_result_error(self): job.result() self.assertIsInstance(exc_info.exception, exceptions.GoogleCloudError) - self.assertEqual(exc_info.exception.code, http_client.BAD_REQUEST) + self.assertEqual(exc_info.exception.code, http.client.BAD_REQUEST) exc_job_instance = getattr(exc_info.exception, "query_job", None) self.assertIs(exc_job_instance, job) @@ -1265,7 +1265,7 @@ def test__begin_error(self): job.result() self.assertIsInstance(exc_info.exception, exceptions.GoogleCloudError) - self.assertEqual(exc_info.exception.code, http_client.BAD_REQUEST) + self.assertEqual(exc_info.exception.code, http.client.BAD_REQUEST) exc_job_instance = getattr(exc_info.exception, "query_job", None) self.assertIs(exc_job_instance, job) diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index 5907a3678..8948d4152 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -18,7 +18,6 @@ import unittest import mock -import six class Test_not_null(unittest.TestCase): @@ -894,7 +893,7 @@ def test_w_list_missing_fields(self): ] original = [42] - with six.assertRaisesRegex(self, ValueError, r".*not match schema length.*"): + with self.assertRaisesRegex(ValueError, r".*not match schema length.*"): self._call_fut(fields, original) def test_w_list_too_many_fields(self): @@ -904,7 +903,7 @@ def test_w_list_too_many_fields(self): ] original = [42, "two", "three"] - with six.assertRaisesRegex(self, ValueError, r".*not match schema length.*"): + with self.assertRaisesRegex(ValueError, r".*not match schema length.*"): self._call_fut(fields, original) def test_w_non_empty_dict(self): diff --git a/tests/unit/test__http.py b/tests/unit/test__http.py index 691c4c802..78e59cb30 100644 --- a/tests/unit/test__http.py +++ b/tests/unit/test__http.py @@ -35,8 +35,8 @@ def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) def test_build_api_url_no_extra_query_params(self): - from six.moves.urllib.parse import parse_qsl - from six.moves.urllib.parse import urlsplit + from urllib.parse import parse_qsl + from urllib.parse import urlsplit conn = self._make_one(object()) uri = conn.build_api_url("/foo") @@ -49,8 +49,8 @@ def test_build_api_url_no_extra_query_params(self): self.assertEqual(parms, {}) def test_build_api_url_w_custom_endpoint(self): - from six.moves.urllib.parse import parse_qsl - from six.moves.urllib.parse import urlsplit + from urllib.parse import parse_qsl + from urllib.parse import urlsplit custom_endpoint = "https://foo-bigquery.googleapis.com" conn = self._make_one(object(), api_endpoint=custom_endpoint) @@ -64,8 +64,8 @@ def test_build_api_url_w_custom_endpoint(self): self.assertEqual(parms, {}) def test_build_api_url_w_extra_query_params(self): - from six.moves.urllib.parse import parse_qsl - from six.moves.urllib.parse import urlsplit + from urllib.parse import parse_qsl + from urllib.parse import urlsplit conn = self._make_one(object()) uri = conn.build_api_url("/foo", {"bar": "baz"}) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index e5ead0ccc..98dec00f9 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -18,7 +18,9 @@ import decimal import email import gzip +import http.client import io +import itertools import json import operator import unittest @@ -26,8 +28,6 @@ import mock import requests -import six -from six.moves import http_client import pytest import pytz import pkg_resources @@ -474,7 +474,7 @@ def test_list_projects_defaults(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/projects"}, client, None) projects = list(page) @@ -508,7 +508,7 @@ def test_list_projects_w_timeout(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - six.next(iterator.pages) + next(iterator.pages) final_attributes.assert_called_once_with({"path": "/projects"}, client, None) @@ -528,7 +528,7 @@ def test_list_projects_explicit_response_missing_projects_key(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/projects"}, client, None) projects = list(page) @@ -582,7 +582,7 @@ def test_list_datasets_defaults(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) datasets = list(page) @@ -635,7 +635,7 @@ def test_list_datasets_explicit_response_missing_datasets_key(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) datasets = list(page) @@ -2919,7 +2919,7 @@ def test_list_tables_empty_w_timeout(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": path}, client, None) tables = list(page) @@ -2942,7 +2942,7 @@ def test_list_models_empty_w_timeout(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": path}, client, None) models = list(page) @@ -2991,7 +2991,7 @@ def test_list_models_defaults(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) models = list(page) @@ -3022,7 +3022,7 @@ def test_list_routines_empty_w_timeout(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with( {"path": "/projects/test-routines/datasets/test_routines/routines"}, @@ -3080,7 +3080,7 @@ def test_list_routines_defaults(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": path}, client, None) routines = list(page) @@ -3149,7 +3149,7 @@ def test_list_tables_defaults(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) tables = list(page) @@ -3213,7 +3213,7 @@ def test_list_tables_explicit(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) tables = list(page) @@ -4040,7 +4040,7 @@ def test_list_jobs_defaults(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) jobs = list(page) @@ -4090,7 +4090,7 @@ def test_list_jobs_load_job_wo_sourceUris(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) jobs = list(page) @@ -4124,7 +4124,7 @@ def test_list_jobs_explicit_missing(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - page = six.next(iterator.pages) + page = next(iterator.pages) final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) jobs = list(page) @@ -4412,7 +4412,7 @@ def _initiate_resumable_upload_helper(self, num_retries=None): # Create mocks to be checked for doing transport. resumable_url = "http://test.invalid?upload_id=hey-you" response_headers = {"location": resumable_url} - fake_transport = self._mock_transport(http_client.OK, response_headers) + fake_transport = self._mock_transport(http.client.OK, response_headers) client = self._make_one(project=self.PROJECT, _http=fake_transport) conn = client._connection = make_connection() @@ -4479,7 +4479,7 @@ def _do_multipart_upload_success_helper(self, get_boundary, num_retries=None): from google.cloud.bigquery.job import LoadJobConfig from google.cloud.bigquery.job import SourceFormat - fake_transport = self._mock_transport(http_client.OK, {}) + fake_transport = self._mock_transport(http.client.OK, {}) client = self._make_one(project=self.PROJECT, _http=fake_transport) conn = client._connection = make_connection() @@ -5022,7 +5022,7 @@ def test_extract_table_generated_job_id(self): _, req = conn.api_request.call_args self.assertEqual(req["method"], "POST") self.assertEqual(req["path"], "/projects/PROJECT/jobs") - self.assertIsInstance(req["data"]["jobReference"]["jobId"], six.string_types) + self.assertIsInstance(req["data"]["jobReference"]["jobId"], str) self.assertIsNone(req["timeout"]) # Check the job resource. @@ -5227,7 +5227,7 @@ def test_query_defaults(self): job = client.query(QUERY) self.assertIsInstance(job, QueryJob) - self.assertIsInstance(job.job_id, six.string_types) + self.assertIsInstance(job.job_id, str) self.assertIs(job._client, client) self.assertEqual(job.query, QUERY) self.assertEqual(job.udf_resources, []) @@ -5240,7 +5240,7 @@ def test_query_defaults(self): self.assertEqual(req["path"], "/projects/PROJECT/jobs") self.assertIsNone(req["timeout"]) sent = req["data"] - self.assertIsInstance(sent["jobReference"]["jobId"], six.string_types) + self.assertIsInstance(sent["jobReference"]["jobId"], str) sent_config = sent["configuration"]["query"] self.assertEqual(sent_config["query"], QUERY) self.assertFalse(sent_config["useLegacySql"]) @@ -5687,7 +5687,7 @@ def test_query_w_udf_resources(self): self.assertEqual(req["path"], "/projects/PROJECT/jobs") self.assertIsNone(req["timeout"]) sent = req["data"] - self.assertIsInstance(sent["jobReference"]["jobId"], six.string_types) + self.assertIsInstance(sent["jobReference"]["jobId"], str) sent_config = sent["configuration"]["query"] self.assertEqual(sent_config["query"], QUERY) self.assertTrue(sent_config["useLegacySql"]) @@ -6398,7 +6398,7 @@ def test_insert_rows_from_dataframe(self): actual_calls = conn.api_request.call_args_list - for call, expected_data in six.moves.zip_longest( + for call, expected_data in itertools.zip_longest( actual_calls, EXPECTED_SENT_DATA ): expected_call = mock.call( @@ -6466,7 +6466,7 @@ def test_insert_rows_from_dataframe_nan(self): actual_calls = conn.api_request.call_args_list - for call, expected_data in six.moves.zip_longest( + for call, expected_data in itertools.zip_longest( actual_calls, EXPECTED_SENT_DATA ): expected_call = mock.call( @@ -6776,7 +6776,7 @@ def test_list_rows(self): # Check that initial total_rows is populated from the table. self.assertEqual(iterator.total_rows, 7) - page = six.next(iterator.pages) + page = next(iterator.pages) rows = list(page) # Check that total_rows is updated based on API response. @@ -6831,14 +6831,14 @@ def test_list_rows_w_start_index_w_page_size(self): table = Table(self.TABLE_REF, schema=[full_name]) iterator = client.list_rows(table, max_results=4, page_size=2, start_index=1) pages = iterator.pages - rows = list(six.next(pages)) + rows = list(next(pages)) extra_params = iterator.extra_params f2i = {"full_name": 0} self.assertEqual(len(rows), 2) self.assertEqual(rows[0], Row(("Phred Phlyntstone",), f2i)) self.assertEqual(rows[1], Row(("Bharney Rhubble",), f2i)) - rows = list(six.next(pages)) + rows = list(next(pages)) self.assertEqual(len(rows), 2) self.assertEqual(rows[0], Row(("Wylma Phlyntstone",), f2i)) @@ -6915,7 +6915,7 @@ def test_list_rows_query_params(self): conn = client._connection = make_connection(*len(tests) * [{}]) for i, test in enumerate(tests): iterator = client.list_rows(table, **test[0]) - six.next(iterator.pages) + next(iterator.pages) req = conn.api_request.call_args_list[i] test[1]["formatOptions.useInt64Timestamp"] = True self.assertEqual(req[1]["query_params"], test[1], "for kwargs %s" % test[0]) @@ -7000,7 +7000,7 @@ def test_list_rows_repeated_fields(self): struct = SchemaField("struct", "RECORD", mode="REPEATED", fields=[index, score]) iterator = client.list_rows(self.TABLE_REF, selected_fields=[color, struct]) - page = six.next(iterator.pages) + page = next(iterator.pages) rows = list(page) total_rows = iterator.total_rows page_token = iterator.next_page_token @@ -7065,7 +7065,7 @@ def test_list_rows_w_record_schema(self): table = Table(self.TABLE_REF, schema=[full_name, phone]) iterator = client.list_rows(table) - page = six.next(iterator.pages) + page = next(iterator.pages) rows = list(page) total_rows = iterator.total_rows page_token = iterator.next_page_token @@ -7241,7 +7241,7 @@ def _make_do_upload_patch(cls, client, method, resource={}, side_effect=None): if side_effect is None: side_effect = [ cls._make_response( - http_client.OK, + http.client.OK, json.dumps(resource), {"Content-Type": "application/json"}, ) @@ -7522,7 +7522,7 @@ def test_load_table_from_file_failure(self): file_obj = self._make_file_obj() response = self._make_response( - content="Someone is already in this spot.", status_code=http_client.CONFLICT + content="Someone is already in this spot.", status_code=http.client.CONFLICT ) do_upload_patch = self._make_do_upload_patch( @@ -8584,7 +8584,7 @@ def _make_resumable_upload_responses(cls, size): resumable_url = "http://test.invalid?upload_id=and-then-there-was-1" initial_response = cls._make_response( - http_client.OK, "", {"location": resumable_url} + http.client.OK, "", {"location": resumable_url} ) data_response = cls._make_response( resumable_media.PERMANENT_REDIRECT, @@ -8592,7 +8592,7 @@ def _make_resumable_upload_responses(cls, size): {"range": "bytes=0-{:d}".format(size - 1)}, ) final_response = cls._make_response( - http_client.OK, + http.client.OK, json.dumps({"size": size}), {"Content-Type": "application/json"}, ) @@ -8634,7 +8634,7 @@ def test__do_resumable_upload(self): ) def test__do_multipart_upload(self): - transport = self._make_transport([self._make_response(http_client.OK)]) + transport = self._make_transport([self._make_response(http.client.OK)]) client = self._make_client(transport) file_obj = self._make_file_obj() file_obj_len = len(file_obj.getvalue()) diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py index 08dd6dcfa..fffa46aa8 100644 --- a/tests/unit/test_dbapi__helpers.py +++ b/tests/unit/test_dbapi__helpers.py @@ -23,8 +23,6 @@ except ImportError: # pragma: NO COVER pyarrow = None -import six - import google.cloud._helpers from google.cloud.bigquery import table from google.cloud.bigquery.dbapi import _helpers @@ -293,7 +291,7 @@ def test_public_instance_methods_on_closed_instance(self): instance = decorated_class() instance._closed = True - with six.assertRaisesRegex(self, exceptions.ProgrammingError, "I'm closed!"): + with self.assertRaisesRegex(exceptions.ProgrammingError, "I'm closed!"): instance.instance_method() def test_methods_wo_public_instance_methods_on_closed_instance(self): @@ -316,7 +314,7 @@ def test_custom_class_closed_attribute(self): instance._closed = False instance._really_closed = True - with six.assertRaisesRegex(self, exceptions.ProgrammingError, "I'm closed!"): + with self.assertRaisesRegex(exceptions.ProgrammingError, "I'm closed!"): instance.instance_method() def test_custom_on_closed_error_type(self): @@ -327,5 +325,5 @@ def test_custom_on_closed_error_type(self): instance = decorated_class() instance._closed = True - with six.assertRaisesRegex(self, RuntimeError, "I'm closed!"): + with self.assertRaisesRegex(RuntimeError, "I'm closed!"): instance.instance_method() diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py index 30fb1292e..edec559b2 100644 --- a/tests/unit/test_dbapi_connection.py +++ b/tests/unit/test_dbapi_connection.py @@ -16,7 +16,6 @@ import unittest import mock -import six try: from google.cloud import bigquery_storage @@ -124,8 +123,8 @@ def test_raises_error_if_closed(self): connection.close() for method in ("close", "commit", "cursor"): - with six.assertRaisesRegex( - self, ProgrammingError, r"Operating on a closed connection\." + with self.assertRaisesRegex( + ProgrammingError, r"Operating on a closed connection\." ): getattr(connection, method)() diff --git a/tests/unit/test_dbapi_cursor.py b/tests/unit/test_dbapi_cursor.py index f55b3fd3f..cbd6f6909 100644 --- a/tests/unit/test_dbapi_cursor.py +++ b/tests/unit/test_dbapi_cursor.py @@ -16,7 +16,6 @@ import unittest import mock -import six try: import pyarrow @@ -181,8 +180,8 @@ def test_raises_error_if_closed(self): ) for method in method_names: - with six.assertRaisesRegex( - self, ProgrammingError, r"Operating on a closed cursor\." + with self.assertRaisesRegex( + ProgrammingError, r"Operating on a closed cursor\." ): getattr(cursor, method)() @@ -375,7 +374,7 @@ def test_fetchall_w_bqstorage_client_fetch_error_no_fallback(self): cursor = connection.cursor() cursor.execute("SELECT foo, bar FROM some_table") - with six.assertRaisesRegex(self, exceptions.Forbidden, "invalid credentials"): + with self.assertRaisesRegex(exceptions.Forbidden, "invalid credentials"): cursor.fetchall() # the default client was not used diff --git a/tests/unit/test_opentelemetry_tracing.py b/tests/unit/test_opentelemetry_tracing.py index 09afa7531..5d0cf2053 100644 --- a/tests/unit/test_opentelemetry_tracing.py +++ b/tests/unit/test_opentelemetry_tracing.py @@ -13,6 +13,7 @@ # limitations under the License. import datetime +import importlib import sys import mock @@ -28,7 +29,6 @@ except ImportError: # pragma: NO COVER opentelemetry = None import pytest -from six.moves import reload_module from google.cloud.bigquery import opentelemetry_tracing @@ -39,7 +39,7 @@ @pytest.mark.skipif(opentelemetry is None, reason="Require `opentelemetry`") @pytest.fixture def setup(): - reload_module(opentelemetry_tracing) + importlib.reload(opentelemetry_tracing) tracer_provider = TracerProvider() memory_exporter = InMemorySpanExporter() span_processor = SimpleExportSpanProcessor(memory_exporter) @@ -51,7 +51,7 @@ def setup(): @pytest.mark.skipif(opentelemetry is None, reason="Require `opentelemetry`") def test_opentelemetry_not_installed(setup, monkeypatch): monkeypatch.setitem(sys.modules, "opentelemetry", None) - reload_module(opentelemetry_tracing) + importlib.reload(opentelemetry_tracing) with opentelemetry_tracing.create_span("No-op for opentelemetry") as span: assert span is None diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 0e7b0bb4d..3373528e0 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -22,7 +22,6 @@ import pkg_resources import pytest import pytz -import six import google.api_core.exceptions @@ -1674,16 +1673,16 @@ def test_iterate(self): rows_iter = iter(row_iterator) - val1 = six.next(rows_iter) + val1 = next(rows_iter) self.assertEqual(val1.name, "Phred Phlyntstone") self.assertEqual(row_iterator.num_results, 1) - val2 = six.next(rows_iter) + val2 = next(rows_iter) self.assertEqual(val2.name, "Bharney Rhubble") self.assertEqual(row_iterator.num_results, 2) with self.assertRaises(StopIteration): - six.next(rows_iter) + next(rows_iter) api_request.assert_called_once_with(method="GET", path=path, query_params={}) @@ -2437,13 +2436,6 @@ def test_to_dataframe(self): self.assertEqual(df.name.dtype.name, "object") self.assertEqual(df.age.dtype.name, "int64") - @pytest.mark.xfail( - six.PY2, - reason=( - "Requires pyarrow>-1.0 to work, but the latter is not compatible " - "with Python 2 anymore." - ), - ) @unittest.skipIf(pandas is None, "Requires `pandas`") @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_to_dataframe_timestamp_out_of_pyarrow_bounds(self): @@ -2475,13 +2467,6 @@ def test_to_dataframe_timestamp_out_of_pyarrow_bounds(self): ], ) - @pytest.mark.xfail( - six.PY2, - reason=( - "Requires pyarrow>-1.0 to work, but the latter is not compatible " - "with Python 2 anymore." - ), - ) @unittest.skipIf(pandas is None, "Requires `pandas`") @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_to_dataframe_datetime_out_of_pyarrow_bounds(self): @@ -2697,7 +2682,7 @@ def test_to_dataframe_w_various_types_nullable(self): else: self.assertIsInstance(row.start_timestamp, pandas.Timestamp) self.assertIsInstance(row.seconds, float) - self.assertIsInstance(row.payment_type, six.string_types) + self.assertIsInstance(row.payment_type, str) self.assertIsInstance(row.complete, bool) self.assertIsInstance(row.date, datetime.date) @@ -3542,7 +3527,7 @@ def test__eq___type_mismatch(self): def test_unhashable_object(self): object_under_test1 = self._make_one(start=1, end=10, interval=2) - with six.assertRaisesRegex(self, TypeError, r".*unhashable type.*"): + with self.assertRaisesRegex(TypeError, r".*unhashable type.*"): hash(object_under_test1) def test_repr(self): @@ -3642,7 +3627,7 @@ def test_unhashable_object(self): object_under_test1 = self._make_one( range_=PartitionRange(start=1, end=10, interval=2), field="integer_col" ) - with six.assertRaisesRegex(self, TypeError, r".*unhashable type.*"): + with self.assertRaisesRegex(TypeError, r".*unhashable type.*"): hash(object_under_test1) def test_repr(self): From 015a73e1839e3427408ef6e0f879717d9ddbdb61 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 8 Jan 2021 16:56:18 -0600 Subject: [PATCH 11/15] fix: add minimum timeout to getQueryResults API requests (#444) * fix: add minimum timeout to getQueryResults API requests Since successful responses can still take a long time to download, have a minimum timeout which should accomodate 99.9%+ of responses. I figure it's more important that *any* timeout is set if desired than it is that the specific timeout is used. This is especially true in cases where a short timeout is requested for the purposes of a progress bar. Making forward progress is more important than the progress bar update frequency. * docs: document minimum timeout value * test: remove redundant query timeout test * test: change assertion for done method * chore: remove unused import --- google/cloud/bigquery/client.py | 22 ++++++++++++++++++++-- tests/system.py | 33 +++++++++++---------------------- tests/unit/job/test_query.py | 6 ++++++ tests/unit/test_client.py | 29 +++++++++++++++++++++++++++-- 4 files changed, 64 insertions(+), 26 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 19693c9ff..3541726b8 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -93,6 +93,14 @@ ) _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS = "jobReference,totalRows,pageToken,rows" +# In microbenchmarks, it's been shown that even in ideal conditions (query +# finished, local data), requests to getQueryResults can take 10+ seconds. +# In less-than-ideal situations, the response can take even longer, as it must +# be able to download a full 100+ MB row in that time. Don't let the +# connection timeout before data can be downloaded. +# https://github.com/googleapis/python-bigquery/issues/438 +_MIN_GET_QUERY_RESULTS_TIMEOUT = 120 + class Project(object): """Wrapper for resource describing a BigQuery project. @@ -1570,7 +1578,9 @@ def _get_query_results( location (Optional[str]): Location of the query job. timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. If set, this connection timeout may be + increased to a minimum value. This prevents retries on what + would otherwise be a successful response. Returns: google.cloud.bigquery.query._QueryResults: @@ -1579,6 +1589,9 @@ def _get_query_results( extra_params = {"maxResults": 0} + if timeout is not None: + timeout = max(timeout, _MIN_GET_QUERY_RESULTS_TIMEOUT) + if project is None: project = self.project @@ -3293,7 +3306,9 @@ def _list_rows_from_query_results( How to retry the RPC. timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. If set, this connection timeout may be + increased to a minimum value. This prevents retries on what + would otherwise be a successful response. If multiple requests are made under the hood, ``timeout`` applies to each individual request. Returns: @@ -3306,6 +3321,9 @@ def _list_rows_from_query_results( "location": location, } + if timeout is not None: + timeout = max(timeout, _MIN_GET_QUERY_RESULTS_TIMEOUT) + if start_index is not None: params["startIndex"] = start_index diff --git a/tests/system.py b/tests/system.py index bfe54b7df..102c8f78d 100644 --- a/tests/system.py +++ b/tests/system.py @@ -27,7 +27,6 @@ import uuid import re -import requests import psutil import pytest import pytz @@ -1798,15 +1797,25 @@ def test_query_w_wrong_config(self): Config.CLIENT.query(good_query, job_config=bad_config).result() def test_query_w_timeout(self): + job_config = bigquery.QueryJobConfig() + job_config.use_query_cache = False + query_job = Config.CLIENT.query( "SELECT * FROM `bigquery-public-data.github_repos.commits`;", job_id_prefix="test_query_w_timeout_", + location="US", + job_config=job_config, ) with self.assertRaises(concurrent.futures.TimeoutError): - # 1 second is much too short for this query. query_job.result(timeout=1) + # Even though the query takes >1 second, the call to getQueryResults + # should succeed. + self.assertFalse(query_job.done(timeout=1)) + + Config.CLIENT.cancel_job(query_job.job_id, location=query_job.location) + def test_query_w_page_size(self): page_size = 45 query_job = Config.CLIENT.query( @@ -2408,26 +2417,6 @@ def test_query_iter(self): row_tuples = [r.values() for r in query_job] self.assertEqual(row_tuples, [(1,)]) - def test_querying_data_w_timeout(self): - job_config = bigquery.QueryJobConfig() - job_config.use_query_cache = False - - query_job = Config.CLIENT.query( - """ - SELECT COUNT(*) - FROM UNNEST(GENERATE_ARRAY(1,1000000)), UNNEST(GENERATE_ARRAY(1, 10000)) - """, - location="US", - job_config=job_config, - ) - - # Specify a very tight deadline to demonstrate that the timeout - # actually has effect. - with self.assertRaises(requests.exceptions.Timeout): - query_job.done(timeout=0.1) - - Config.CLIENT.cancel_job(query_job.job_id, location=query_job.location) - @unittest.skipIf(pandas is None, "Requires `pandas`") def test_query_results_to_dataframe(self): QUERY = """ diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 579a841d1..a4ab11ab6 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -1046,6 +1046,8 @@ def test_result_invokes_begins(self): self.assertEqual(reload_request[1]["method"], "GET") def test_result_w_timeout(self): + import google.cloud.bigquery.client + begun_resource = self._make_resource() query_resource = { "jobComplete": True, @@ -1072,6 +1074,10 @@ def test_result_w_timeout(self): "/projects/{}/queries/{}".format(self.PROJECT, self.JOB_ID), ) self.assertEqual(query_request[1]["query_params"]["timeoutMs"], 900) + self.assertEqual( + query_request[1]["timeout"], + google.cloud.bigquery.client._MIN_GET_QUERY_RESULTS_TIMEOUT, + ) self.assertEqual(reload_request[1]["method"], "GET") def test_result_w_page_size(self): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 98dec00f9..bf183b5a4 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -311,7 +311,7 @@ def test__get_query_results_miss_w_explicit_project_and_timeout(self): project="other-project", location=self.LOCATION, timeout_ms=500, - timeout=42, + timeout=420, ) final_attributes.assert_called_once_with({"path": path}, client, None) @@ -320,7 +320,32 @@ def test__get_query_results_miss_w_explicit_project_and_timeout(self): method="GET", path=path, query_params={"maxResults": 0, "timeoutMs": 500, "location": self.LOCATION}, - timeout=42, + timeout=420, + ) + + def test__get_query_results_miss_w_short_timeout(self): + import google.cloud.bigquery.client + from google.cloud.exceptions import NotFound + + creds = _make_credentials() + client = self._make_one(self.PROJECT, creds) + conn = client._connection = make_connection() + path = "/projects/other-project/queries/nothere" + with self.assertRaises(NotFound): + client._get_query_results( + "nothere", + None, + project="other-project", + location=self.LOCATION, + timeout_ms=500, + timeout=1, + ) + + conn.api_request.assert_called_once_with( + method="GET", + path=path, + query_params={"maxResults": 0, "timeoutMs": 500, "location": self.LOCATION}, + timeout=google.cloud.bigquery.client._MIN_GET_QUERY_RESULTS_TIMEOUT, ) def test__get_query_results_miss_w_client_location(self): From 99ef1d20faede0d3b949c6f0cdb3c38f738d630b Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 11 Jan 2021 10:18:08 +0100 Subject: [PATCH 12/15] chore: Bound maximum supported Python version (#465) * chore: bound maximum supported Python version * Bound supported Python versions claim in README --- README.rst | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c7d50d729..61192b625 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ dependencies. Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Python >= 3.6 +Python >= 3.6, < 3.9 Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/setup.py b/setup.py index fcafddbd2..0ea6ccca2 100644 --- a/setup.py +++ b/setup.py @@ -120,7 +120,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6", + python_requires=">=3.6, <3.9", include_package_data=True, zip_safe=False, ) From fb3ad7682dc19189c75f8ab4345794a613ac0ca8 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 11 Jan 2021 10:20:06 +0100 Subject: [PATCH 13/15] refactor: simplify AutoStrEnum definition (#458) With now only Python 3.6 supported, we can use the _generate_next_value() hook instead of metaclass magic. --- .../bigquery/magics/line_arg_parser/lexer.py | 41 ++++--------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/google/cloud/bigquery/magics/line_arg_parser/lexer.py b/google/cloud/bigquery/magics/line_arg_parser/lexer.py index 17e1ffdae..0cb63292c 100644 --- a/google/cloud/bigquery/magics/line_arg_parser/lexer.py +++ b/google/cloud/bigquery/magics/line_arg_parser/lexer.py @@ -136,40 +136,17 @@ ) -# The _generate_next_value_() enum hook is only available in Python 3.6+, thus we -# need to do some acrobatics to implement an "auto str enum" base class. Implementation -# based on the recipe provided by the very author of the Enum library: -# https://stackoverflow.com/a/32313954/5040035 -class StrEnumMeta(enum.EnumMeta): - @classmethod - def __prepare__(metacls, name, bases, **kwargs): - # Having deterministic enum members definition order is nice. - return OrderedDict() +class AutoStrEnum(str, enum.Enum): + """Base enum class for for name=value str enums.""" - def __new__(metacls, name, bases, oldclassdict): - # Scan through the declared enum members and convert any value that is a plain - # empty tuple into a `str` of the name instead. - newclassdict = enum._EnumDict() - for key, val in oldclassdict.items(): - if val == (): - val = key - newclassdict[key] = val - return super(StrEnumMeta, metacls).__new__(metacls, name, bases, newclassdict) + def _generate_next_value_(name, start, count, last_values): + return name -# The @six.add_metaclass decorator does not work, Enum complains about _sunder_ names, -# and we cannot use class syntax directly, because the Python 3 version would cause -# a syntax error under Python 2. -AutoStrEnum = StrEnumMeta( - "AutoStrEnum", - (str, enum.Enum), - {"__doc__": "Base enum class for for name=value str enums."}, -) - TokenType = AutoStrEnum( "TokenType", [ - (name, name) + (name, enum.auto()) for name in itertools.chain.from_iterable(token_types.values()) if not name.startswith("GOTO_") ], @@ -177,10 +154,10 @@ def __new__(metacls, name, bases, oldclassdict): class LexerState(AutoStrEnum): - PARSE_POS_ARGS = () # parsing positional arguments - PARSE_NON_PARAMS_OPTIONS = () # parsing options other than "--params" - PARSE_PARAMS_OPTION = () # parsing the "--params" option - STATE_END = () + PARSE_POS_ARGS = enum.auto() # parsing positional arguments + PARSE_NON_PARAMS_OPTIONS = enum.auto() # parsing options other than "--params" + PARSE_PARAMS_OPTION = enum.auto() # parsing the "--params" option + STATE_END = enum.auto() class Lexer(object): From 7ea6b7c2469d2415192cfdacc379e38e49d24775 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 11 Jan 2021 11:07:35 -0600 Subject: [PATCH 14/15] fix: use debug logging level for OpenTelemetry message (#442) * fix: use debug logging level for OpenTelemetry message * only warn at span creation time * add unit test for skipping warning * refactor: rename _warned_telemetry to indicate private and mutable --- .../cloud/bigquery/opentelemetry_tracing.py | 19 ++++++++++++------- tests/unit/test_opentelemetry_tracing.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/google/cloud/bigquery/opentelemetry_tracing.py b/google/cloud/bigquery/opentelemetry_tracing.py index b9d18efad..57f258ac4 100644 --- a/google/cloud/bigquery/opentelemetry_tracing.py +++ b/google/cloud/bigquery/opentelemetry_tracing.py @@ -23,16 +23,11 @@ from opentelemetry.trace.status import Status HAS_OPENTELEMETRY = True + _warned_telemetry = True except ImportError: - logger.info( - "This service is instrumented using OpenTelemetry. " - "OpenTelemetry could not be imported; please " - "add opentelemetry-api and opentelemetry-instrumentation " - "packages in order to get BigQuery Tracing data." - ) - HAS_OPENTELEMETRY = False + _warned_telemetry = False _default_attributes = { "db.system": "BigQuery" @@ -64,8 +59,18 @@ def create_span(name, attributes=None, client=None, job_ref=None): Raised if a span could not be yielded or issue with call to OpenTelemetry. """ + global _warned_telemetry final_attributes = _get_final_span_attributes(attributes, client, job_ref) if not HAS_OPENTELEMETRY: + if not _warned_telemetry: + logger.debug( + "This service is instrumented using OpenTelemetry. " + "OpenTelemetry could not be imported; please " + "add opentelemetry-api and opentelemetry-instrumentation " + "packages in order to get BigQuery Tracing data." + ) + _warned_telemetry = True + yield None return tracer = trace.get_tracer(__name__) diff --git a/tests/unit/test_opentelemetry_tracing.py b/tests/unit/test_opentelemetry_tracing.py index 5d0cf2053..726e3cf6f 100644 --- a/tests/unit/test_opentelemetry_tracing.py +++ b/tests/unit/test_opentelemetry_tracing.py @@ -52,8 +52,20 @@ def setup(): def test_opentelemetry_not_installed(setup, monkeypatch): monkeypatch.setitem(sys.modules, "opentelemetry", None) importlib.reload(opentelemetry_tracing) + assert not opentelemetry_tracing._warned_telemetry with opentelemetry_tracing.create_span("No-op for opentelemetry") as span: assert span is None + assert opentelemetry_tracing._warned_telemetry + + +@pytest.mark.skipif(opentelemetry is None, reason="Require `opentelemetry`") +def test_opentelemetry_not_installed_doesnt_warn(setup, monkeypatch): + monkeypatch.setitem(sys.modules, "opentelemetry", None) + importlib.reload(opentelemetry_tracing) + opentelemetry_tracing._warned_telemetry = True + with opentelemetry_tracing.create_span("No-op for opentelemetry") as span: + assert span is None + assert opentelemetry_tracing._warned_telemetry @pytest.mark.skipif(opentelemetry is None, reason="Require `opentelemetry`") From b0e074f7522710886be1da2f117ea22de411b408 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 11:40:09 -0600 Subject: [PATCH 15/15] chore: release 2.6.2 (#429) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 15 +++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01f62ff6..4d58072e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +### [2.6.2](https://www.github.com/googleapis/python-bigquery/compare/v2.6.1...v2.6.2) (2021-01-11) + + +### Bug Fixes + +* add minimum timeout to getQueryResults API requests ([#444](https://www.github.com/googleapis/python-bigquery/issues/444)) ([015a73e](https://www.github.com/googleapis/python-bigquery/commit/015a73e1839e3427408ef6e0f879717d9ddbdb61)) +* use debug logging level for OpenTelemetry message ([#442](https://www.github.com/googleapis/python-bigquery/issues/442)) ([7ea6b7c](https://www.github.com/googleapis/python-bigquery/commit/7ea6b7c2469d2415192cfdacc379e38e49d24775)) + + +### Documentation + +* add GEOGRAPHY data type code samples ([#428](https://www.github.com/googleapis/python-bigquery/issues/428)) ([dbc68b3](https://www.github.com/googleapis/python-bigquery/commit/dbc68b3d1f325f80d24a2da5f028b0f653fb0317)) +* fix Shapely import in GEOGRAPHY sample ([#431](https://www.github.com/googleapis/python-bigquery/issues/431)) ([96a1c5b](https://www.github.com/googleapis/python-bigquery/commit/96a1c5b3c72855ba6ae8c88dfd0cdb02d2faf909)) +* move and refresh view samples ([#420](https://www.github.com/googleapis/python-bigquery/issues/420)) ([079b6a1](https://www.github.com/googleapis/python-bigquery/commit/079b6a162f6929bf801366d92f8daeb3318426c4)) + ### [2.6.1](https://www.github.com/googleapis/python-bigquery/compare/v2.6.0...v2.6.1) (2020-12-09) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 410cd066e..9aaeb8bc4 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.6.1" +__version__ = "2.6.2"