From 811c2bf0f8f13979b1fa6a3d6c43d7ea599e335c Mon Sep 17 00:00:00 2001 From: Sangram Date: Sat, 30 Jun 2018 22:25:01 +0530 Subject: [PATCH 01/14] Add tableadmin sample tableadmin sample for gcloud python samples. --- .project | 11 ++ bigtable/.DS_Store | Bin 0 -> 6148 bytes bigtable/tableadmin/.DS_Store | Bin 0 -> 6148 bytes bigtable/tableadmin/requirements.txt | 1 + bigtable/tableadmin/tableadmin.py | 272 +++++++++++++++++++++++++++ 5 files changed, 284 insertions(+) create mode 100644 .project create mode 100644 bigtable/.DS_Store create mode 100644 bigtable/tableadmin/.DS_Store create mode 100755 bigtable/tableadmin/requirements.txt create mode 100644 bigtable/tableadmin/tableadmin.py diff --git a/.project b/.project new file mode 100644 index 00000000000..7f04fb8c28d --- /dev/null +++ b/.project @@ -0,0 +1,11 @@ + + + python-docs-samples + + + + + + + + diff --git a/bigtable/.DS_Store b/bigtable/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..66e7eaa2255ecff4f2d42cc4fab862658cedc7b3 GIT binary patch literal 6148 zcmeHK%}N773{G^BqL&^$&Lg;Qu#^=AFJdnq)vi=we@fl;yszj%d>y|`B6R5%dJ?e- z%zQJMBs2L+XEsE{v#v;qrbIMB6J!~Th^R-Gj?DN3$T7w84`?D{*UJEc m@!%r>6S+su=h-7=%YK(KLjEn-Z=xmz`!?Z>p(03 literal 0 HcmV?d00001 diff --git a/bigtable/tableadmin/.DS_Store b/bigtable/tableadmin/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Sat, 30 Jun 2018 22:46:29 +0530 Subject: [PATCH 02/14] Revert "Add tableadmin sample" This reverts commit 811c2bf0f8f13979b1fa6a3d6c43d7ea599e335c. --- .project | 11 -- bigtable/.DS_Store | Bin 6148 -> 0 bytes bigtable/tableadmin/.DS_Store | Bin 6148 -> 0 bytes bigtable/tableadmin/requirements.txt | 1 - bigtable/tableadmin/tableadmin.py | 272 --------------------------- 5 files changed, 284 deletions(-) delete mode 100644 .project delete mode 100644 bigtable/.DS_Store delete mode 100644 bigtable/tableadmin/.DS_Store delete mode 100755 bigtable/tableadmin/requirements.txt delete mode 100644 bigtable/tableadmin/tableadmin.py diff --git a/.project b/.project deleted file mode 100644 index 7f04fb8c28d..00000000000 --- a/.project +++ /dev/null @@ -1,11 +0,0 @@ - - - python-docs-samples - - - - - - - - diff --git a/bigtable/.DS_Store b/bigtable/.DS_Store deleted file mode 100644 index 66e7eaa2255ecff4f2d42cc4fab862658cedc7b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}N773{G^BqL&^$&Lg;Qu#^=AFJdnq)vi=we@fl;yszj%d>y|`B6R5%dJ?e- z%zQJMBs2L+XEsE{v#v;qrbIMB6J!~Th^R-Gj?DN3$T7w84`?D{*UJEc m@!%r>6S+su=h-7=%YK(KLjEn-Z=xmz`!?Z>p(03 diff --git a/bigtable/tableadmin/.DS_Store b/bigtable/tableadmin/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Sun, 1 Jul 2018 19:28:57 +0530 Subject: [PATCH 03/14] Add tableadmin sample Create tableadmin sample for python. --- bigtable/tableadmin/README.rst.in | 23 +++ bigtable/tableadmin/requirements.txt | 1 + bigtable/tableadmin/tableadmin.py | 276 +++++++++++++++++++++++++ bigtable/tableadmin/tableadmin_test.py | 77 +++++++ 4 files changed, 377 insertions(+) create mode 100644 bigtable/tableadmin/README.rst.in create mode 100755 bigtable/tableadmin/requirements.txt create mode 100644 bigtable/tableadmin/tableadmin.py create mode 100755 bigtable/tableadmin/tableadmin_test.py diff --git a/bigtable/tableadmin/README.rst.in b/bigtable/tableadmin/README.rst.in new file mode 100644 index 00000000000..7fd37641969 --- /dev/null +++ b/bigtable/tableadmin/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable and run some basic operations. + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +setup: +- auth +- install_deps + +samples: +- name: Basic example with Bigtable Column family and GC rules. + file: tableadmin.py + show_help: true + +cloud_client_library: true + +folder: bigtable/tableadmin \ No newline at end of file diff --git a/bigtable/tableadmin/requirements.txt b/bigtable/tableadmin/requirements.txt new file mode 100755 index 00000000000..ff62d05fda0 --- /dev/null +++ b/bigtable/tableadmin/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable==0.29.0 diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py new file mode 100644 index 00000000000..093ee621993 --- /dev/null +++ b/bigtable/tableadmin/tableadmin.py @@ -0,0 +1,276 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. +# http://www.apache.org/licenses/LICENSE-2.0 +Prerequisites: +- Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials +""" + +import datetime +import argparse +from google.cloud import bigtable + +def run_table_operations(project_id, instance_id, table_id): + ''' Create bigtable and perform different operations on it. + :project_id: Project id of the client + :instance_id: Instance of the client + :table_id: Table id to create table. + ''' + + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + table = instance.table(table_id) + + # Check whether table exists in an instance. + # Create table if it does not exists. + print('Checking if table {} exists...'.format(table_id)) + if exists(instance, table_id): + print('Table {} already exists.'.format(table_id)) + else: + print('Creating the {} table.'.format(table_id)) + try: + table.create() + except : + print('Error creating table {}.'.format(table_id)) + else: + print('Created table {}.'.format(table_id)) + + try: + tables = instance.list_tables() + except: + print('Error listing tables.') + + # List existing tables in the instance. + print('Listing tables in current project...') + if tables != []: + for t in tables: + print t.name + else: + print('No table exists in current project...') + + # Display name of the table. + print('Printing table metadata...') + try: + print(table.name) + except : + print('Error retrieving table metadata.') + + + print('Creating column family cf1 with with MaxAge GC Rule...') + # [START bigtable_create_family_gc_max_age] + # Create a column family with GC policy : maximum age + # where age = current time minus cell timestamp + column_family_id1 = 'cf1' + try: + # Define the GC rule to retain data with max age of 5 days + max_age = datetime.timedelta(days=5) + max_age_rule = table.max_age_gc_rule(max_age) + print('Created MaxAge GC rule.') + except: + print('Error creating MaxAge GC Rule.') + + try: + cf1 = table.column_family(column_family_id1, max_age_rule) + cf1.create() + print('Created column family cf1 with MaxAge GC Rule.') + except: + print('Error creating column family with MaxAge GC Rule.') + # [End bigtable_create_family_gc_max_age] + + + print('Creating column family cf2 with max versions GC rule...') + # [START bigtable_create_family_gc_max_versions] + # Create a column family with GC policy : most recent N versions + # where 1 = most recent version + column_family_id2 = 'cf2' + try: + # Define the GC policy to retain only the most recent 2 versions + max_versions = 2 + max_versions_rule = table.max_versions_gc_rule(max_versions) + print('Created Max Versions GC Rule.') + except: + print('Error creating Max Versions GC Rule.') + + try: + cf2 = table.column_family(column_family_id2, max_versions_rule) + cf2 .create() + print('Created column family cf2 with Max Versions GC Rule.') + except: + print('Error creating column family with Max Versions GC Rule.') + # [END bigtable_create_family_gc_max_versions] + + + print('Creating column family cf3 with union GC rule...') + # [START bigtable_create_family_gc_union] + # Create a column family with GC policy to drop data that matches at least one condition. + column_family_id3 = 'cf3' + try: + # GC rule : Drop max age rule OR the most recent version rule. + union = [max_age_rule, max_versions_rule] + union_rule = table.gc_rule_union(union) + print('Created Union GC Rule.') + except: + print('Error creating Union GC Rule.') + + try: + cf3 = table.column_family(column_family_id3, union_rule) + cf3.create() + print('Created column family cf3 with Union GC rule') + except: + print('Error creating column family with Union GC rule') + # [END bigtable_create_family_gc_union] + + + + print('Creating column family cf4 with Intersection GC rule...') + # [START bigtable_create_family_gc_intersection] + # Create a column family with GC policy to drop data that matches all conditions + column_family_id4 = 'cf4' + + try: + # GC rule: Drop cells older than 5 days AND older than the most recent 2 versions + intersection = [max_age_rule, max_versions_rule] + intersection_rule = table.gc_rule_intersection(union) + print('Created Intersection GC Rule.') + except: + print('Error creating Intersection GC Rule.') + + try: + cf4 = table.column_family(column_family_id4, intersection_rule) + cf4.create() + print('Created column family cf4 with Intersection GC rule.') + except: + print('Error creating column family with Intersection GC rule.') + # [END bigtable_create_family_gc_intersection] + + + print('Creating column family cf5 with a Nested GC rule...') + # [START bigtable_create_family_gc_nested] + # Create a column family with nested GC policys. + + # Create a nested GC rule: + # Drop cells that are either older than the 10 recent versions + # OR + # Drop cells that are older than a month AND older than the 2 recent versions + column_family_id5 = 'cf5' + try: + # Drop cells that are either older than the 10 recent versions + max_versions_rule1 = table.max_versions_gc_rule(10) + + # Drop cells that are older than a month AND older than the 2 recent versions + max_age = datetime.timedelta(days=30) + max_age_rule = table.max_age_gc_rule(max_age) + max_versions_rule2 = table.max_versions_gc_rule(2) + intersection = [max_age_rule, max_versions_rule2] + intersection_rule = table.gc_rule_intersection(intersection) + + # This nesting is done with union rule since it is OR between the selected rules. + nest = [max_versions_rule1, intersection_rule] + nested_rule = table.gc_rule_union(nest) + print('Created Nested GC Rule.') + except: + print('Error creating Nested GC Rule.') + + try: + cf5 = table.column_family(column_family_id5, nested_rule) + cf5.create() + print('Created column family cf5 with a Nested GC rule.') + except: + print('Error creating column family with a Nested GC rule.') + # [END bigtable_create_family_gc_nested] + + + print('Printing Column Family and GC Rule for all column families...') + try: + cf = table.list_column_families() + column_families = [key for key,val in cf.items()] + for cf, gc_rule in sorted(cf.items()): + print "Column Family:", cf + print "GC Rule:" + print gc_rule.to_pb() + except: + print('Error Column Family and GC Rule.') + +def exists(instance_obj, table_id): + """ Check whether table exists or not. + + Note: This is temporary function as code for this is under development. + Once complete, this function would be removed. + + :instance_ojt: Instance object + Returns bool + """ + for table in instance_obj.list_tables(): + if table_id == table.table_id: + return True + return False + + +def delete_table(project_id, instance_id, table_id): + ''' Delete bigtable. + :project_id: Project id of the client + :instance_id: Instance of the client + :table_id: Table id to create table. + ''' + + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + table = instance.table(table_id) + + print('Checking if table {} exists...'.format(table_id)) + + if exists(instance, table_id): + print('Table {} exists.'.format(table_id)) + try: + print('Deleting {} table.'.format(table_id)) + table.delete() + except: + print('Error deleting {} table.'.format(table_id)) + else: + print('Deleted {} table.'.format(table_id)) + else: + print('Table {} does not exists.'.format(table_id)) + + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('command', + help='run or delete. Operation to perform on table.') + parser.add_argument( + '--table', + help='Cloud Bigtable Table name.', + default='Hello-Bigtable') + + parser.add_argument('project_id', help='Your Cloud Platform project ID.') + parser.add_argument( + 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') + + args = parser.parse_args() + + if args.command.lower() == 'run': + run_table_operations(args.project_id, args.instance_id, args.table) + elif args.command.lower() == 'delete': + delete_table(args.project_id, args.instance_id, args.table) + else: + print ('Command should be either run or delete.\n Use argument -h, --help to show help and exit.') + + diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py new file mode 100755 index 00000000000..7d3b2f49e77 --- /dev/null +++ b/bigtable/tableadmin/tableadmin_test.py @@ -0,0 +1,77 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random + + +from tableadmin import run_table_operations +from tableadmin import delete_table +from _pytest.capture import capsys + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] +TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}' +TABLE_NAME_RANGE = 10000 + + +def test_run_table_operations(capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + + run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) + out, _ = capsys.readouterr() + assert 'Creating the {} table.'.format(table_name) in out + assert not 'Table {} already exists.'.format(table_name) in out + assert 'Listing tables in current project.' in out + assert not 'Error creating table {}.'.format(table_name) in out + assert 'Creating column family cf1 with with MaxAge GC Rule' in out + assert 'Created MaxAge GC rule.' in out + assert not 'Error creating MaxAge GC Rule.' in out + assert 'Created column family cf1 with MaxAge GC Rule.' in out + assert not 'Error creating column family with MaxAge GC Rule.' in out + assert 'Created Max Versions GC Rule.' in out + assert not 'Error creating Max Versions GC Rule.' in out + assert 'Created column family cf2 with Max Versions GC Rule.' in out + assert not 'Error creating column family with Max Versions GC Rule.' in out + assert 'Created Union GC Rule.' in out + assert not 'Error creating Union GC Rule.' in out + assert 'Created column family cf3 with Union GC rule' in out + assert not 'Error creating column family with Union GC rule' in out + assert 'Created Intersection GC Rule.' in out + assert not 'Error creating Intersection GC Rule.' in out + assert 'Created column family cf4 with Intersection GC rule.' in out + assert not 'Error creating column family with Intersection GC rule.' in out + assert 'Created Nested GC Rule.' in out + assert not 'Error creating Nested GC Rule.' in out + assert 'Created column family cf5 with a Nested GC rule.' in out + assert not 'Error creating column family with a Nested GC rule.' in out + assert 'Printing Column Family and GC Rule for all column families.' in out + assert not 'Error Column Family and GC Rule.' in out + + delete_table(PROJECT, BIGTABLE_CLUSTER, table_name) + out, _ = capsys.readouterr() + assert 'Table {} exists.'.format(table_name) in out + assert 'Deleting {} table.'.format(table_name) in out + assert 'Deleted {} table.'.format(table_name) in out + assert not 'Table {} does not exists.'.format(table_name) in out + + + + + + + + + \ No newline at end of file From 5b24797eed49276b18a88394afddc73e9a4bbe52 Mon Sep 17 00:00:00 2001 From: Sangram Date: Tue, 3 Jul 2018 22:27:20 +0530 Subject: [PATCH 04/14] Updated files for fixes Fix lint issues. Fix Testcases. --- bigtable/tableadmin/README.rst | 115 +++++++++ bigtable/tableadmin/tableadmin.py | 323 ++++++++++++------------- bigtable/tableadmin/tableadmin_test.py | 34 +-- 3 files changed, 272 insertions(+), 200 deletions(-) create mode 100644 bigtable/tableadmin/README.rst diff --git a/bigtable/tableadmin/README.rst b/bigtable/tableadmin/README.rst new file mode 100644 index 00000000000..f7f83d6d2a1 --- /dev/null +++ b/bigtable/tableadmin/README.rst @@ -0,0 +1,115 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Basic example ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/tableadmin.py,bigtable/hello/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python tableadmin.py + + usage: tableadmin.py [-h] [run] [delete] [--table TABLE] project_id instance_id + + Demonstrates how to connect to Cloud Bigtable and run some basic operations. + Prerequisites: - Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google + Application Default Credentials. + https://developers.google.com/identity/protocols/application-default- + credentials + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + + optional arguments: + -h, --help show this help message and exit + --table TABLE Table to create and destroy. (default: Hello-Bigtable) + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 093ee621993..4b8fb935a60 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +20,14 @@ https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google Application Default Credentials. https://developers.google.com/identity/protocols/application-default-credentials + +Operations performed: +- Create bigtable. +- List bigtables existing in current project, if any. +- List metadata(Name) of created bigtable. +- Create Column Families with different GC rules. + - GC Rules like: MaxAge, MaxVersions, Union, Intersection and Nested. +- Delete bigtable. """ import datetime @@ -26,192 +36,159 @@ def run_table_operations(project_id, instance_id, table_id): ''' Create bigtable and perform different operations on it. - :project_id: Project id of the client - :instance_id: Instance of the client - :table_id: Table id to create table. + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type table_id: str + :param table_id: Table id to create table. ''' - + client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) table = instance.table(table_id) - + # Check whether table exists in an instance. # Create table if it does not exists. - print('Checking if table {} exists...'.format(table_id)) + print 'Checking if table {} exists...'.format(table_id) if exists(instance, table_id): - print('Table {} already exists.'.format(table_id)) + print 'Table {} already exists.'.format(table_id) else: - print('Creating the {} table.'.format(table_id)) - try: - table.create() - except : - print('Error creating table {}.'.format(table_id)) - else: - print('Created table {}.'.format(table_id)) - - try: - tables = instance.list_tables() - except: - print('Error listing tables.') - - # List existing tables in the instance. - print('Listing tables in current project...') + print 'Creating the {} table.'.format(table_id) + table.create() + print 'Created table {}.'.format(table_id) + + tables = instance.list_tables() + + # [START List existing tables in the instance.] + print 'Listing tables in current project...' if tables != []: - for t in tables: - print t.name + for tbl in tables: + print tbl.name else: - print('No table exists in current project...') - + print 'No table exists in current project...' + # [END List existing tables in the instance.] + # Display name of the table. - print('Printing table metadata...') - try: - print(table.name) - except : - print('Error retrieving table metadata.') - - - print('Creating column family cf1 with with MaxAge GC Rule...') + print 'Printing table metadata...' + print table.name + + # [START bigtable_create_family_gc_max_age] + print 'Creating column family cf1 with with MaxAge GC Rule...' # Create a column family with GC policy : maximum age # where age = current time minus cell timestamp column_family_id1 = 'cf1' - try: - # Define the GC rule to retain data with max age of 5 days - max_age = datetime.timedelta(days=5) - max_age_rule = table.max_age_gc_rule(max_age) - print('Created MaxAge GC rule.') - except: - print('Error creating MaxAge GC Rule.') - - try: - cf1 = table.column_family(column_family_id1, max_age_rule) - cf1.create() - print('Created column family cf1 with MaxAge GC Rule.') - except: - print('Error creating column family with MaxAge GC Rule.') - # [End bigtable_create_family_gc_max_age] - - - print('Creating column family cf2 with max versions GC rule...') + # Define the GC rule to retain data with max age of 5 days + max_age = datetime.timedelta(days=5) + max_age_rule = table.max_age_gc_rule(max_age) + print 'Created MaxAge GC rule.' + + cf1 = table.column_family(column_family_id1, max_age_rule) + cf1.create() + print 'Created column family cf1 with MaxAge GC Rule.' + # [END bigtable_create_family_gc_max_age] + + # [START bigtable_create_family_gc_max_versions] + print 'Creating column family cf2 with max versions GC rule...' # Create a column family with GC policy : most recent N versions # where 1 = most recent version column_family_id2 = 'cf2' - try: - # Define the GC policy to retain only the most recent 2 versions - max_versions = 2 - max_versions_rule = table.max_versions_gc_rule(max_versions) - print('Created Max Versions GC Rule.') - except: - print('Error creating Max Versions GC Rule.') - - try: - cf2 = table.column_family(column_family_id2, max_versions_rule) - cf2 .create() - print('Created column family cf2 with Max Versions GC Rule.') - except: - print('Error creating column family with Max Versions GC Rule.') - # [END bigtable_create_family_gc_max_versions] - - - print('Creating column family cf3 with union GC rule...') + # Define the GC policy to retain only the most recent 2 versions + max_versions = 2 + max_versions_rule = table.max_versions_gc_rule(max_versions) + print 'Created Max Versions GC Rule.' + + cf2 = table.column_family(column_family_id2, max_versions_rule) + cf2 .create() + print 'Created column family cf2 with Max Versions GC Rule.' + # [END bigtable_create_family_gc_max_versions] + + # [START bigtable_create_family_gc_union] + print 'Creating column family cf3 with union GC rule...' # Create a column family with GC policy to drop data that matches at least one condition. column_family_id3 = 'cf3' - try: - # GC rule : Drop max age rule OR the most recent version rule. - union = [max_age_rule, max_versions_rule] - union_rule = table.gc_rule_union(union) - print('Created Union GC Rule.') - except: - print('Error creating Union GC Rule.') - - try: - cf3 = table.column_family(column_family_id3, union_rule) - cf3.create() - print('Created column family cf3 with Union GC rule') - except: - print('Error creating column family with Union GC rule') + # GC rule : Drop max age rule OR the most recent version rule. + union = [max_age_rule, max_versions_rule] + union_rule = table.gc_rule_union(union) + print 'Created Union GC Rule.' + + cf3 = table.column_family(column_family_id3, union_rule) + cf3.create() + print 'Created column family cf3 with Union GC rule' # [END bigtable_create_family_gc_union] - - - - print('Creating column family cf4 with Intersection GC rule...') + + # [START bigtable_create_family_gc_intersection] + print 'Creating column family cf4 with Intersection GC rule...' # Create a column family with GC policy to drop data that matches all conditions column_family_id4 = 'cf4' - - try: - # GC rule: Drop cells older than 5 days AND older than the most recent 2 versions - intersection = [max_age_rule, max_versions_rule] - intersection_rule = table.gc_rule_intersection(union) - print('Created Intersection GC Rule.') - except: - print('Error creating Intersection GC Rule.') - - try: - cf4 = table.column_family(column_family_id4, intersection_rule) - cf4.create() - print('Created column family cf4 with Intersection GC rule.') - except: - print('Error creating column family with Intersection GC rule.') + # GC rule: Drop cells older than 5 days AND older than the most recent 2 versions + intersection = [max_age_rule, max_versions_rule] + intersection_rule = table.gc_rule_intersection(union) + print 'Created Intersection GC Rule.' + + cf4 = table.column_family(column_family_id4, intersection_rule) + cf4.create() + print 'Created column family cf4 with Intersection GC rule.' # [END bigtable_create_family_gc_intersection] - - - print('Creating column family cf5 with a Nested GC rule...') + + # [START bigtable_create_family_gc_nested] + print 'Creating column family cf5 with a Nested GC rule...' # Create a column family with nested GC policys. - # Create a nested GC rule: # Drop cells that are either older than the 10 recent versions # OR # Drop cells that are older than a month AND older than the 2 recent versions column_family_id5 = 'cf5' - try: - # Drop cells that are either older than the 10 recent versions - max_versions_rule1 = table.max_versions_gc_rule(10) - - # Drop cells that are older than a month AND older than the 2 recent versions - max_age = datetime.timedelta(days=30) - max_age_rule = table.max_age_gc_rule(max_age) - max_versions_rule2 = table.max_versions_gc_rule(2) - intersection = [max_age_rule, max_versions_rule2] - intersection_rule = table.gc_rule_intersection(intersection) - - # This nesting is done with union rule since it is OR between the selected rules. - nest = [max_versions_rule1, intersection_rule] - nested_rule = table.gc_rule_union(nest) - print('Created Nested GC Rule.') - except: - print('Error creating Nested GC Rule.') - - try: - cf5 = table.column_family(column_family_id5, nested_rule) - cf5.create() - print('Created column family cf5 with a Nested GC rule.') - except: - print('Error creating column family with a Nested GC rule.') + # Drop cells that are either older than the 10 recent versions + max_versions_rule1 = table.max_versions_gc_rule(10) + + # Drop cells that are older than a month AND older than the 2 recent versions + max_age = datetime.timedelta(days=30) + max_age_rule = table.max_age_gc_rule(max_age) + max_versions_rule2 = table.max_versions_gc_rule(2) + intersection = [max_age_rule, max_versions_rule2] + intersection_rule = table.gc_rule_intersection(intersection) + + # This nesting is done with union rule since it is OR between the selected rules. + nest = [max_versions_rule1, intersection_rule] + nested_rule = table.gc_rule_union(nest) + print 'Created Nested GC Rule.' + + cf5 = table.column_family(column_family_id5, nested_rule) + cf5.create() + print 'Created column family cf5 with a Nested GC rule.' # [END bigtable_create_family_gc_nested] - - - print('Printing Column Family and GC Rule for all column families...') - try: - cf = table.list_column_families() - column_families = [key for key,val in cf.items()] - for cf, gc_rule in sorted(cf.items()): - print "Column Family:", cf - print "GC Rule:" - print gc_rule.to_pb() - except: - print('Error Column Family and GC Rule.') + + + # [START Printing Column Family and GC Rule for all column families.] + print 'Printing Column Family and GC Rule for all column families...' + column_families = table.list_column_families() + for column_family, gc_rule in sorted(column_families.items()): + print "Column Family:", column_family + print "GC Rule:" + print gc_rule.to_pb() + # [END Printing Column Family and GC Rule for all column families.] + def exists(instance_obj, table_id): """ Check whether table exists or not. - + Note: This is temporary function as code for this is under development. Once complete, this function would be removed. - - :instance_ojt: Instance object + + :type instance_obj: Instance Class. + :param instance_obj: Instance object. + + :type table_id: str + :param table_id: Table id to create table. Returns bool """ for table in instance_obj.list_tables(): @@ -219,58 +196,56 @@ def exists(instance_obj, table_id): return True return False - + def delete_table(project_id, instance_id, table_id): ''' Delete bigtable. - :project_id: Project id of the client - :instance_id: Instance of the client - :table_id: Table id to create table. + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type table_id: str + :param table_id: Table id to create table. ''' - + client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) table = instance.table(table_id) - - print('Checking if table {} exists...'.format(table_id)) - + + print 'Checking if table {} exists...'.format(table_id) if exists(instance, table_id): - print('Table {} exists.'.format(table_id)) - try: - print('Deleting {} table.'.format(table_id)) - table.delete() - except: - print('Error deleting {} table.'.format(table_id)) - else: - print('Deleted {} table.'.format(table_id)) + print 'Table {} exists.'.format(table_id) + print 'Deleting {} table.'.format(table_id) + table.delete() + print 'Deleted {} table.'.format(table_id) else: - print('Table {} does not exists.'.format(table_id)) - - - + print 'Table {} does not exists.'.format(table_id) + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('command', + + parser.add_argument('command', help='run or delete. Operation to perform on table.') parser.add_argument( '--table', help='Cloud Bigtable Table name.', default='Hello-Bigtable') - + parser.add_argument('project_id', help='Your Cloud Platform project ID.') parser.add_argument( 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') - + args = parser.parse_args() - + if args.command.lower() == 'run': run_table_operations(args.project_id, args.instance_id, args.table) elif args.command.lower() == 'delete': delete_table(args.project_id, args.instance_id, args.table) else: - print ('Command should be either run or delete.\n Use argument -h, --help to show help and exit.') - - + print 'Command should be either run or delete.\n Use argument -h, \ + --help to show help and exit.' diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index 7d3b2f49e77..dcea85a0e22 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -15,10 +15,9 @@ import os import random - +from _pytest.capture import capsys from tableadmin import run_table_operations from tableadmin import delete_table -from _pytest.capture import capsys PROJECT = os.environ['GCLOUD_PROJECT'] BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] @@ -33,45 +32,28 @@ def test_run_table_operations(capsys): run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() assert 'Creating the {} table.'.format(table_name) in out - assert not 'Table {} already exists.'.format(table_name) in out assert 'Listing tables in current project.' in out - assert not 'Error creating table {}.'.format(table_name) in out assert 'Creating column family cf1 with with MaxAge GC Rule' in out assert 'Created MaxAge GC rule.' in out - assert not 'Error creating MaxAge GC Rule.' in out assert 'Created column family cf1 with MaxAge GC Rule.' in out - assert not 'Error creating column family with MaxAge GC Rule.' in out assert 'Created Max Versions GC Rule.' in out - assert not 'Error creating Max Versions GC Rule.' in out assert 'Created column family cf2 with Max Versions GC Rule.' in out - assert not 'Error creating column family with Max Versions GC Rule.' in out assert 'Created Union GC Rule.' in out - assert not 'Error creating Union GC Rule.' in out assert 'Created column family cf3 with Union GC rule' in out - assert not 'Error creating column family with Union GC rule' in out assert 'Created Intersection GC Rule.' in out - assert not 'Error creating Intersection GC Rule.' in out assert 'Created column family cf4 with Intersection GC rule.' in out - assert not 'Error creating column family with Intersection GC rule.' in out assert 'Created Nested GC Rule.' in out - assert not 'Error creating Nested GC Rule.' in out assert 'Created column family cf5 with a Nested GC rule.' in out - assert not 'Error creating column family with a Nested GC rule.' in out assert 'Printing Column Family and GC Rule for all column families.' in out - assert not 'Error Column Family and GC Rule.' in out - + + +def test_delete_table(capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + delete_table(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() + assert 'Table {} exists.'.format(table_name) in out assert 'Deleting {} table.'.format(table_name) in out assert 'Deleted {} table.'.format(table_name) in out - assert not 'Table {} does not exists.'.format(table_name) in out - - - - - - - - - \ No newline at end of file From 4264123022a8e3b131990f25dd3b1061886880f1 Mon Sep 17 00:00:00 2001 From: Sangram Date: Thu, 5 Jul 2018 21:51:02 +0530 Subject: [PATCH 05/14] Fix CI failures Fixed newline and import errors in tableadmin. FIxed import and unsed errors from tableadmin_test. --- bigtable/tableadmin/tableadmin.py | 28 ++++++++++++++------------ bigtable/tableadmin/tableadmin_test.py | 3 +-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 4b8fb935a60..ccde2b99ef5 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -30,10 +30,12 @@ - Delete bigtable. """ -import datetime import argparse +import datetime + from google.cloud import bigtable + def run_table_operations(project_id, instance_id, table_id): ''' Create bigtable and perform different operations on it. @@ -76,7 +78,6 @@ def run_table_operations(project_id, instance_id, table_id): print 'Printing table metadata...' print table.name - # [START bigtable_create_family_gc_max_age] print 'Creating column family cf1 with with MaxAge GC Rule...' # Create a column family with GC policy : maximum age @@ -92,7 +93,6 @@ def run_table_operations(project_id, instance_id, table_id): print 'Created column family cf1 with MaxAge GC Rule.' # [END bigtable_create_family_gc_max_age] - # [START bigtable_create_family_gc_max_versions] print 'Creating column family cf2 with max versions GC rule...' # Create a column family with GC policy : most recent N versions @@ -108,10 +108,10 @@ def run_table_operations(project_id, instance_id, table_id): print 'Created column family cf2 with Max Versions GC Rule.' # [END bigtable_create_family_gc_max_versions] - # [START bigtable_create_family_gc_union] print 'Creating column family cf3 with union GC rule...' - # Create a column family with GC policy to drop data that matches at least one condition. + # Create a column family with GC policy to drop data that matches + # at least one condition. column_family_id3 = 'cf3' # GC rule : Drop max age rule OR the most recent version rule. union = [max_age_rule, max_versions_rule] @@ -123,12 +123,13 @@ def run_table_operations(project_id, instance_id, table_id): print 'Created column family cf3 with Union GC rule' # [END bigtable_create_family_gc_union] - # [START bigtable_create_family_gc_intersection] print 'Creating column family cf4 with Intersection GC rule...' - # Create a column family with GC policy to drop data that matches all conditions + # Create a column family with GC policy to drop data that matches + # all conditions column_family_id4 = 'cf4' - # GC rule: Drop cells older than 5 days AND older than the most recent 2 versions + # GC rule: Drop cells older than 5 days AND older than the most + # recent 2 versions intersection = [max_age_rule, max_versions_rule] intersection_rule = table.gc_rule_intersection(union) print 'Created Intersection GC Rule.' @@ -138,26 +139,28 @@ def run_table_operations(project_id, instance_id, table_id): print 'Created column family cf4 with Intersection GC rule.' # [END bigtable_create_family_gc_intersection] - # [START bigtable_create_family_gc_nested] print 'Creating column family cf5 with a Nested GC rule...' # Create a column family with nested GC policys. # Create a nested GC rule: # Drop cells that are either older than the 10 recent versions # OR - # Drop cells that are older than a month AND older than the 2 recent versions + # Drop cells that are older than a month AND older than the + # 2 recent versions column_family_id5 = 'cf5' # Drop cells that are either older than the 10 recent versions max_versions_rule1 = table.max_versions_gc_rule(10) - # Drop cells that are older than a month AND older than the 2 recent versions + # Drop cells that are older than a month AND older than + # the 2 recent versions max_age = datetime.timedelta(days=30) max_age_rule = table.max_age_gc_rule(max_age) max_versions_rule2 = table.max_versions_gc_rule(2) intersection = [max_age_rule, max_versions_rule2] intersection_rule = table.gc_rule_intersection(intersection) - # This nesting is done with union rule since it is OR between the selected rules. + # This nesting is done with union rule since it is OR between + # the selected rules. nest = [max_versions_rule1, intersection_rule] nested_rule = table.gc_rule_union(nest) print 'Created Nested GC Rule.' @@ -167,7 +170,6 @@ def run_table_operations(project_id, instance_id, table_id): print 'Created column family cf5 with a Nested GC rule.' # [END bigtable_create_family_gc_nested] - # [START Printing Column Family and GC Rule for all column families.] print 'Printing Column Family and GC Rule for all column families...' column_families = table.list_column_families() diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index dcea85a0e22..fb235d1c3a2 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -15,9 +15,8 @@ import os import random -from _pytest.capture import capsys -from tableadmin import run_table_operations from tableadmin import delete_table +from tableadmin import run_table_operations PROJECT = os.environ['GCLOUD_PROJECT'] BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] From 1b0a4f7b928641930cf1c10cce8c887ff2c11e0b Mon Sep 17 00:00:00 2001 From: Sangram Date: Sat, 7 Jul 2018 23:09:14 +0530 Subject: [PATCH 06/14] Updated the review comments and additional code from internal review. Implemented all review comments. - Done Print only table_id instead of full path of table - Done Please comment proper start tag for bigtable_list_column_families - Done Create `update-gc-rule` example. - Done Create `delete-column-family` example. - Done Show GC rule sample in comments. - Done --- bigtable/tableadmin/tableadmin.py | 73 +++++++++++++++++++++----- bigtable/tableadmin/tableadmin_test.py | 14 +++-- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index ccde2b99ef5..27a642f38bf 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -# Copyright 2016 Google Inc. -# +# Copyright 2018, Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -22,12 +22,12 @@ https://developers.google.com/identity/protocols/application-default-credentials Operations performed: -- Create bigtable. -- List bigtables existing in current project, if any. -- List metadata(Name) of created bigtable. +- Create a Cloud Bigtable table. +- List tables for a Cloud Bigtable instance. +- Print metadata of the newly created table. - Create Column Families with different GC rules. - GC Rules like: MaxAge, MaxVersions, Union, Intersection and Nested. -- Delete bigtable. +- Delete a Bigtable table. """ import argparse @@ -38,6 +38,7 @@ def run_table_operations(project_id, instance_id, table_id): ''' Create bigtable and perform different operations on it. + ''' Create a Bigtable table and perform basic table operations. :type project_id: str :param project_id: Project id of the client. @@ -63,20 +64,19 @@ def run_table_operations(project_id, instance_id, table_id): table.create() print 'Created table {}.'.format(table_id) - tables = instance.list_tables() - # [START List existing tables in the instance.] + tables = instance.list_tables() print 'Listing tables in current project...' if tables != []: for tbl in tables: - print tbl.name + print tbl.table_id else: print 'No table exists in current project...' # [END List existing tables in the instance.] # Display name of the table. print 'Printing table metadata...' - print table.name + print table.table_id # [START bigtable_create_family_gc_max_age] print 'Creating column family cf1 with with MaxAge GC Rule...' @@ -170,21 +170,66 @@ def run_table_operations(project_id, instance_id, table_id): print 'Created column family cf5 with a Nested GC rule.' # [END bigtable_create_family_gc_nested] - # [START Printing Column Family and GC Rule for all column families.] + # [START bigtable_list_column_families] print 'Printing Column Family and GC Rule for all column families...' column_families = table.list_column_families() for column_family, gc_rule in sorted(column_families.items()): print "Column Family:", column_family print "GC Rule:" print gc_rule.to_pb() - # [END Printing Column Family and GC Rule for all column families.] + # Sample output: + # Column Family: cf4 + # GC Rule: + # gc_rule { + # intersection { + # rules { + # max_age { + # seconds: 432000 + # } + # } + # rules { + # max_num_versions: 2 + # } + # } + # } + # [END bigtable_list_column_families] + + # [START bigtable_update_gc_rule] + print 'Updating column family cf1 GC rule...' + # Update the column family cf1 to update the GC rule + max_versions = 1 + max_versions_rule = table.max_versions_gc_rule(max_versions) + # Create a reference to the column family with GC rule + cf1 = table.column_family(column_family_id1, max_versions_rule) + cf1.update() + print 'Updated column family cf1 GC rule' + # [END bigtable_update_gc_rule] + + # [START bigtable_family_get_gc_rule] + print 'Print updated column family cf1 GC rule...' + print 'Column Family:', column_family_id1 + print 'GC Rule:' + print cf1.to_pb() + # [END bigtable_family_get_gc_rule] + + # [START bigtable_delete_family] + print 'Delete a column family cf2...' + # Delete a column family + cf2.delete() + print 'Column family cf2 deleted successfully.' + # [END bigtable_delete_family] + + print 'execute command python tableadmin.py delete [project_id] \ + [instance_id] --table [tableName] to delete the table.' def exists(instance_obj, table_id): """ Check whether table exists or not. - Note: This is temporary function as code for this is under development. - Once complete, this function would be removed. + .. note:: + + This is temporary function as code for this is under development. + Once complete, this function would be removed. :type instance_obj: Instance Class. :param instance_obj: Instance object. diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index fb235d1c3a2..b0a764decc0 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -1,10 +1,11 @@ -# Copyright 2016 Google Inc. -# +#!/usr/bin/env python + +# Copyright 2018, Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -44,6 +45,13 @@ def test_run_table_operations(capsys): assert 'Created Nested GC Rule.' in out assert 'Created column family cf5 with a Nested GC rule.' in out assert 'Printing Column Family and GC Rule for all column families.' in out + assert 'Updating column family cf1 GC rule...' in out + assert 'Updated column family cf1 GC rule' in out + assert 'Print updated column family cf1 GC rule...' in out + assert 'Column Family: cf1' in out + assert 'max_num_versions: 1' in out + assert 'Delete a column family cf2...' in out + assert 'Column family cf2 deleted successfully.' in out def test_delete_table(capsys): From c01df0ac4903144dd87edb9d4e481512837d3a05 Mon Sep 17 00:00:00 2001 From: Sangram Date: Sat, 7 Jul 2018 23:23:57 +0530 Subject: [PATCH 07/14] Fix extra comments Remove extra comments which got added just before commit. --- bigtable/tableadmin/tableadmin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 27a642f38bf..889f134ceb0 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -37,8 +37,7 @@ def run_table_operations(project_id, instance_id, table_id): - ''' Create bigtable and perform different operations on it. - ''' Create a Bigtable table and perform basic table operations. + ''' Create a Bigtable table and perform basic table operations :type project_id: str :param project_id: Project id of the client. From b12a8be1b29833b87db004f853979877fa08eddf Mon Sep 17 00:00:00 2001 From: Sangram Date: Wed, 11 Jul 2018 23:18:20 +0530 Subject: [PATCH 08/14] Fixing unit test failures and add metadata function Fix unit test failures for python3.6 . Add metadata function to get metadata of table. --- bigtable/tableadmin/tableadmin.py | 33 +++++++++++++++++++------- bigtable/tableadmin/tableadmin_test.py | 9 +++---- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 889f134ceb0..8b18173fec2 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -63,19 +63,15 @@ def run_table_operations(project_id, instance_id, table_id): table.create() print 'Created table {}.'.format(table_id) - # [START List existing tables in the instance.] + # [START bigtable_list_tables] tables = instance.list_tables() print 'Listing tables in current project...' if tables != []: for tbl in tables: - print tbl.table_id + print get_metadata(tbl), "\n" else: print 'No table exists in current project...' - # [END List existing tables in the instance.] - - # Display name of the table. - print 'Printing table metadata...' - print table.table_id + # [END bigtable_list_tables] # [START bigtable_create_family_gc_max_age] print 'Creating column family cf1 with with MaxAge GC Rule...' @@ -221,6 +217,23 @@ def run_table_operations(project_id, instance_id, table_id): print 'execute command python tableadmin.py delete [project_id] \ [instance_id] --table [tableName] to delete the table.' +def get_metadata(table): + """ Get table metadata. + + .. note:: + + This is temporary function as code for this is planned for\ + development. Once complete, this function would be removed. + + :type table: Table Class. + :param table: Table object. + Returns result dictionary of table metadata + """ + result = {("Table ID ", table.table_id): {}} + column_families = table.list_column_families() + for column_family, gc_rule in sorted(column_families.items()): + result[("Table ID ", table.table_id)][("Column Family ", column_family)] = gc_rule.to_pb() + return result def exists(instance_obj, table_id): """ Check whether table exists or not. @@ -234,7 +247,7 @@ def exists(instance_obj, table_id): :param instance_obj: Instance object. :type table_id: str - :param table_id: Table id to create table. + :param table_id: Table id to identify table. Returns bool """ for table in instance_obj.list_tables(): @@ -259,7 +272,8 @@ def delete_table(project_id, instance_id, table_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) table = instance.table(table_id) - + # [START bigtable_delete_table] + # Delete the entire table print 'Checking if table {} exists...'.format(table_id) if exists(instance, table_id): print 'Table {} exists.'.format(table_id) @@ -268,6 +282,7 @@ def delete_table(project_id, instance_id, table_id): print 'Deleted {} table.'.format(table_id) else: print 'Table {} does not exists.'.format(table_id) + # [END bigtable_delete_table] if __name__ == '__main__': diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index b0a764decc0..b324c5a342b 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -31,7 +31,8 @@ def test_run_table_operations(capsys): run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - assert 'Creating the {} table.'.format(table_name) in out + + assert 'Creating the ' + table_name + ' table.' in out assert 'Listing tables in current project.' in out assert 'Creating column family cf1 with with MaxAge GC Rule' in out assert 'Created MaxAge GC rule.' in out @@ -61,6 +62,6 @@ def test_delete_table(capsys): delete_table(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - assert 'Table {} exists.'.format(table_name) in out - assert 'Deleting {} table.'.format(table_name) in out - assert 'Deleted {} table.'.format(table_name) in out + assert 'Table ' + table_name + ' exists.' in out + assert 'Deleting ' + table_name + ' table.' in out + assert 'Deleted ' + table_name + ' table.' in out From 81c5d4a8558e07a4c1240d7f70a1a60337d3c6b0 Mon Sep 17 00:00:00 2001 From: Sangram Date: Thu, 12 Jul 2018 02:21:12 +0530 Subject: [PATCH 09/14] Remove too many variables and prints Remove too many variables. Remove too many prints. --- bigtable/tableadmin/tableadmin.py | 107 +++++++++---------------- bigtable/tableadmin/tableadmin_test.py | 9 +-- 2 files changed, 38 insertions(+), 78 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 8b18173fec2..6ba19401474 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -68,7 +68,7 @@ def run_table_operations(project_id, instance_id, table_id): print 'Listing tables in current project...' if tables != []: for tbl in tables: - print get_metadata(tbl), "\n" + print tbl.table_id else: print 'No table exists in current project...' # [END bigtable_list_tables] @@ -77,13 +77,11 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf1 with with MaxAge GC Rule...' # Create a column family with GC policy : maximum age # where age = current time minus cell timestamp - column_family_id1 = 'cf1' + # Define the GC rule to retain data with max age of 5 days - max_age = datetime.timedelta(days=5) - max_age_rule = table.max_age_gc_rule(max_age) - print 'Created MaxAge GC rule.' + max_age_rule = table.max_age_gc_rule(datetime.timedelta(days=5)) - cf1 = table.column_family(column_family_id1, max_age_rule) + cf1 = table.column_family('cf1', max_age_rule) cf1.create() print 'Created column family cf1 with MaxAge GC Rule.' # [END bigtable_create_family_gc_max_age] @@ -92,13 +90,11 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf2 with max versions GC rule...' # Create a column family with GC policy : most recent N versions # where 1 = most recent version - column_family_id2 = 'cf2' + # Define the GC policy to retain only the most recent 2 versions - max_versions = 2 - max_versions_rule = table.max_versions_gc_rule(max_versions) - print 'Created Max Versions GC Rule.' + max_versions_rule = table.max_versions_gc_rule(2) - cf2 = table.column_family(column_family_id2, max_versions_rule) + cf2 = table.column_family('cf2', max_versions_rule) cf2 .create() print 'Created column family cf2 with Max Versions GC Rule.' # [END bigtable_create_family_gc_max_versions] @@ -107,13 +103,13 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf3 with union GC rule...' # Create a column family with GC policy to drop data that matches # at least one condition. - column_family_id3 = 'cf3' - # GC rule : Drop max age rule OR the most recent version rule. - union = [max_age_rule, max_versions_rule] - union_rule = table.gc_rule_union(union) - print 'Created Union GC Rule.' + # Define a GC rule to drop cells older than 5 days or not the \ + # most recent version + union_rule = table.gc_rule_union([ \ + table.max_age_gc_rule(datetime.timedelta(days=5)),\ + table.max_versions_gc_rule(2)]) - cf3 = table.column_family(column_family_id3, union_rule) + cf3 = table.column_family('cf3', union_rule) cf3.create() print 'Created column family cf3 with Union GC rule' # [END bigtable_create_family_gc_union] @@ -122,45 +118,33 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf4 with Intersection GC rule...' # Create a column family with GC policy to drop data that matches # all conditions - column_family_id4 = 'cf4' # GC rule: Drop cells older than 5 days AND older than the most # recent 2 versions - intersection = [max_age_rule, max_versions_rule] - intersection_rule = table.gc_rule_intersection(union) - print 'Created Intersection GC Rule.' + intersection_rule = table.gc_rule_intersection([ \ + table.max_age_gc_rule(datetime.timedelta(days=5)),\ + table.max_versions_gc_rule(2)]) - cf4 = table.column_family(column_family_id4, intersection_rule) + cf4 = table.column_family('cf4', intersection_rule) cf4.create() print 'Created column family cf4 with Intersection GC rule.' # [END bigtable_create_family_gc_intersection] # [START bigtable_create_family_gc_nested] print 'Creating column family cf5 with a Nested GC rule...' - # Create a column family with nested GC policys. + # Create a column family with nested GC policies. # Create a nested GC rule: # Drop cells that are either older than the 10 recent versions # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - column_family_id5 = 'cf5' - # Drop cells that are either older than the 10 recent versions - max_versions_rule1 = table.max_versions_gc_rule(10) - - # Drop cells that are older than a month AND older than - # the 2 recent versions - max_age = datetime.timedelta(days=30) - max_age_rule = table.max_age_gc_rule(max_age) - max_versions_rule2 = table.max_versions_gc_rule(2) - intersection = [max_age_rule, max_versions_rule2] - intersection_rule = table.gc_rule_intersection(intersection) - - # This nesting is done with union rule since it is OR between - # the selected rules. - nest = [max_versions_rule1, intersection_rule] - nested_rule = table.gc_rule_union(nest) - print 'Created Nested GC Rule.' - - cf5 = table.column_family(column_family_id5, nested_rule) + # Drop cells that are older than a month AND older than the 2 recent versions + rule1 = table.max_versions_gc_rule(10) + rule2 = table.gc_rule_intersection([ \ + table.max_age_gc_rule(datetime.timedelta(days=30)), \ + table.max_versions_gc_rule(2) \ + ]) + + nested_rule = table.gc_rule_union([rule1, rule2]) + + cf5 = table.column_family('cf5', nested_rule) cf5.create() print 'Created column family cf5 with a Nested GC rule.' # [END bigtable_create_family_gc_nested] @@ -189,23 +173,21 @@ def run_table_operations(project_id, instance_id, table_id): # } # [END bigtable_list_column_families] + print 'Print column family cf1 GC rule before update...' + print "Column Family: cf1" + print cf1.to_pb() + # [START bigtable_update_gc_rule] print 'Updating column family cf1 GC rule...' # Update the column family cf1 to update the GC rule - max_versions = 1 - max_versions_rule = table.max_versions_gc_rule(max_versions) - # Create a reference to the column family with GC rule - cf1 = table.column_family(column_family_id1, max_versions_rule) + cf1 = table.column_family('cf1', table.max_versions_gc_rule(1)) cf1.update() - print 'Updated column family cf1 GC rule' + print 'Updated column family cf1 GC rule\n' # [END bigtable_update_gc_rule] - # [START bigtable_family_get_gc_rule] - print 'Print updated column family cf1 GC rule...' - print 'Column Family:', column_family_id1 - print 'GC Rule:' + print 'Print column family cf1 GC rule after update...' + print "Column Family: cf1" print cf1.to_pb() - # [END bigtable_family_get_gc_rule] # [START bigtable_delete_family] print 'Delete a column family cf2...' @@ -217,23 +199,6 @@ def run_table_operations(project_id, instance_id, table_id): print 'execute command python tableadmin.py delete [project_id] \ [instance_id] --table [tableName] to delete the table.' -def get_metadata(table): - """ Get table metadata. - - .. note:: - - This is temporary function as code for this is planned for\ - development. Once complete, this function would be removed. - - :type table: Table Class. - :param table: Table object. - Returns result dictionary of table metadata - """ - result = {("Table ID ", table.table_id): {}} - column_families = table.list_column_families() - for column_family, gc_rule in sorted(column_families.items()): - result[("Table ID ", table.table_id)][("Column Family ", column_family)] = gc_rule.to_pb() - return result def exists(instance_obj, table_id): """ Check whether table exists or not. diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index b324c5a342b..df384a18b80 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -31,24 +31,19 @@ def test_run_table_operations(capsys): run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - + assert 'Creating the ' + table_name + ' table.' in out assert 'Listing tables in current project.' in out assert 'Creating column family cf1 with with MaxAge GC Rule' in out - assert 'Created MaxAge GC rule.' in out assert 'Created column family cf1 with MaxAge GC Rule.' in out - assert 'Created Max Versions GC Rule.' in out assert 'Created column family cf2 with Max Versions GC Rule.' in out - assert 'Created Union GC Rule.' in out assert 'Created column family cf3 with Union GC rule' in out - assert 'Created Intersection GC Rule.' in out assert 'Created column family cf4 with Intersection GC rule.' in out - assert 'Created Nested GC Rule.' in out assert 'Created column family cf5 with a Nested GC rule.' in out assert 'Printing Column Family and GC Rule for all column families.' in out assert 'Updating column family cf1 GC rule...' in out assert 'Updated column family cf1 GC rule' in out - assert 'Print updated column family cf1 GC rule...' in out + assert 'Print column family cf1 GC rule after update...' in out assert 'Column Family: cf1' in out assert 'max_num_versions: 1' in out assert 'Delete a column family cf2...' in out From e6aba49f5664f94b18f692aef7c497096c092a32 Mon Sep 17 00:00:00 2001 From: Sangram Date: Fri, 13 Jul 2018 22:51:54 +0530 Subject: [PATCH 10/14] Revert "Remove too many variables and prints" This reverts commit 81c5d4a8558e07a4c1240d7f70a1a60337d3c6b0. --- bigtable/tableadmin/tableadmin.py | 107 ++++++++++++++++--------- bigtable/tableadmin/tableadmin_test.py | 9 ++- 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 6ba19401474..8b18173fec2 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -68,7 +68,7 @@ def run_table_operations(project_id, instance_id, table_id): print 'Listing tables in current project...' if tables != []: for tbl in tables: - print tbl.table_id + print get_metadata(tbl), "\n" else: print 'No table exists in current project...' # [END bigtable_list_tables] @@ -77,11 +77,13 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf1 with with MaxAge GC Rule...' # Create a column family with GC policy : maximum age # where age = current time minus cell timestamp - + column_family_id1 = 'cf1' # Define the GC rule to retain data with max age of 5 days - max_age_rule = table.max_age_gc_rule(datetime.timedelta(days=5)) + max_age = datetime.timedelta(days=5) + max_age_rule = table.max_age_gc_rule(max_age) + print 'Created MaxAge GC rule.' - cf1 = table.column_family('cf1', max_age_rule) + cf1 = table.column_family(column_family_id1, max_age_rule) cf1.create() print 'Created column family cf1 with MaxAge GC Rule.' # [END bigtable_create_family_gc_max_age] @@ -90,11 +92,13 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf2 with max versions GC rule...' # Create a column family with GC policy : most recent N versions # where 1 = most recent version - + column_family_id2 = 'cf2' # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = table.max_versions_gc_rule(2) + max_versions = 2 + max_versions_rule = table.max_versions_gc_rule(max_versions) + print 'Created Max Versions GC Rule.' - cf2 = table.column_family('cf2', max_versions_rule) + cf2 = table.column_family(column_family_id2, max_versions_rule) cf2 .create() print 'Created column family cf2 with Max Versions GC Rule.' # [END bigtable_create_family_gc_max_versions] @@ -103,13 +107,13 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf3 with union GC rule...' # Create a column family with GC policy to drop data that matches # at least one condition. - # Define a GC rule to drop cells older than 5 days or not the \ - # most recent version - union_rule = table.gc_rule_union([ \ - table.max_age_gc_rule(datetime.timedelta(days=5)),\ - table.max_versions_gc_rule(2)]) + column_family_id3 = 'cf3' + # GC rule : Drop max age rule OR the most recent version rule. + union = [max_age_rule, max_versions_rule] + union_rule = table.gc_rule_union(union) + print 'Created Union GC Rule.' - cf3 = table.column_family('cf3', union_rule) + cf3 = table.column_family(column_family_id3, union_rule) cf3.create() print 'Created column family cf3 with Union GC rule' # [END bigtable_create_family_gc_union] @@ -118,33 +122,45 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf4 with Intersection GC rule...' # Create a column family with GC policy to drop data that matches # all conditions + column_family_id4 = 'cf4' # GC rule: Drop cells older than 5 days AND older than the most # recent 2 versions - intersection_rule = table.gc_rule_intersection([ \ - table.max_age_gc_rule(datetime.timedelta(days=5)),\ - table.max_versions_gc_rule(2)]) + intersection = [max_age_rule, max_versions_rule] + intersection_rule = table.gc_rule_intersection(union) + print 'Created Intersection GC Rule.' - cf4 = table.column_family('cf4', intersection_rule) + cf4 = table.column_family(column_family_id4, intersection_rule) cf4.create() print 'Created column family cf4 with Intersection GC rule.' # [END bigtable_create_family_gc_intersection] # [START bigtable_create_family_gc_nested] print 'Creating column family cf5 with a Nested GC rule...' - # Create a column family with nested GC policies. + # Create a column family with nested GC policys. # Create a nested GC rule: # Drop cells that are either older than the 10 recent versions # OR - # Drop cells that are older than a month AND older than the 2 recent versions - rule1 = table.max_versions_gc_rule(10) - rule2 = table.gc_rule_intersection([ \ - table.max_age_gc_rule(datetime.timedelta(days=30)), \ - table.max_versions_gc_rule(2) \ - ]) - - nested_rule = table.gc_rule_union([rule1, rule2]) - - cf5 = table.column_family('cf5', nested_rule) + # Drop cells that are older than a month AND older than the + # 2 recent versions + column_family_id5 = 'cf5' + # Drop cells that are either older than the 10 recent versions + max_versions_rule1 = table.max_versions_gc_rule(10) + + # Drop cells that are older than a month AND older than + # the 2 recent versions + max_age = datetime.timedelta(days=30) + max_age_rule = table.max_age_gc_rule(max_age) + max_versions_rule2 = table.max_versions_gc_rule(2) + intersection = [max_age_rule, max_versions_rule2] + intersection_rule = table.gc_rule_intersection(intersection) + + # This nesting is done with union rule since it is OR between + # the selected rules. + nest = [max_versions_rule1, intersection_rule] + nested_rule = table.gc_rule_union(nest) + print 'Created Nested GC Rule.' + + cf5 = table.column_family(column_family_id5, nested_rule) cf5.create() print 'Created column family cf5 with a Nested GC rule.' # [END bigtable_create_family_gc_nested] @@ -173,21 +189,23 @@ def run_table_operations(project_id, instance_id, table_id): # } # [END bigtable_list_column_families] - print 'Print column family cf1 GC rule before update...' - print "Column Family: cf1" - print cf1.to_pb() - # [START bigtable_update_gc_rule] print 'Updating column family cf1 GC rule...' # Update the column family cf1 to update the GC rule - cf1 = table.column_family('cf1', table.max_versions_gc_rule(1)) + max_versions = 1 + max_versions_rule = table.max_versions_gc_rule(max_versions) + # Create a reference to the column family with GC rule + cf1 = table.column_family(column_family_id1, max_versions_rule) cf1.update() - print 'Updated column family cf1 GC rule\n' + print 'Updated column family cf1 GC rule' # [END bigtable_update_gc_rule] - print 'Print column family cf1 GC rule after update...' - print "Column Family: cf1" + # [START bigtable_family_get_gc_rule] + print 'Print updated column family cf1 GC rule...' + print 'Column Family:', column_family_id1 + print 'GC Rule:' print cf1.to_pb() + # [END bigtable_family_get_gc_rule] # [START bigtable_delete_family] print 'Delete a column family cf2...' @@ -199,6 +217,23 @@ def run_table_operations(project_id, instance_id, table_id): print 'execute command python tableadmin.py delete [project_id] \ [instance_id] --table [tableName] to delete the table.' +def get_metadata(table): + """ Get table metadata. + + .. note:: + + This is temporary function as code for this is planned for\ + development. Once complete, this function would be removed. + + :type table: Table Class. + :param table: Table object. + Returns result dictionary of table metadata + """ + result = {("Table ID ", table.table_id): {}} + column_families = table.list_column_families() + for column_family, gc_rule in sorted(column_families.items()): + result[("Table ID ", table.table_id)][("Column Family ", column_family)] = gc_rule.to_pb() + return result def exists(instance_obj, table_id): """ Check whether table exists or not. diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index df384a18b80..b324c5a342b 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -31,19 +31,24 @@ def test_run_table_operations(capsys): run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - + assert 'Creating the ' + table_name + ' table.' in out assert 'Listing tables in current project.' in out assert 'Creating column family cf1 with with MaxAge GC Rule' in out + assert 'Created MaxAge GC rule.' in out assert 'Created column family cf1 with MaxAge GC Rule.' in out + assert 'Created Max Versions GC Rule.' in out assert 'Created column family cf2 with Max Versions GC Rule.' in out + assert 'Created Union GC Rule.' in out assert 'Created column family cf3 with Union GC rule' in out + assert 'Created Intersection GC Rule.' in out assert 'Created column family cf4 with Intersection GC rule.' in out + assert 'Created Nested GC Rule.' in out assert 'Created column family cf5 with a Nested GC rule.' in out assert 'Printing Column Family and GC Rule for all column families.' in out assert 'Updating column family cf1 GC rule...' in out assert 'Updated column family cf1 GC rule' in out - assert 'Print column family cf1 GC rule after update...' in out + assert 'Print updated column family cf1 GC rule...' in out assert 'Column Family: cf1' in out assert 'max_num_versions: 1' in out assert 'Delete a column family cf2...' in out From d0d845bd21ad864ef32a5bcaf30bdcc34949043d Mon Sep 17 00:00:00 2001 From: Sangram Date: Fri, 13 Jul 2018 22:51:59 +0530 Subject: [PATCH 11/14] Revert "Fixing unit test failures and add metadata function" This reverts commit b12a8be1b29833b87db004f853979877fa08eddf. --- bigtable/tableadmin/tableadmin.py | 33 +++++++------------------- bigtable/tableadmin/tableadmin_test.py | 9 ++++--- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 8b18173fec2..889f134ceb0 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -63,15 +63,19 @@ def run_table_operations(project_id, instance_id, table_id): table.create() print 'Created table {}.'.format(table_id) - # [START bigtable_list_tables] + # [START List existing tables in the instance.] tables = instance.list_tables() print 'Listing tables in current project...' if tables != []: for tbl in tables: - print get_metadata(tbl), "\n" + print tbl.table_id else: print 'No table exists in current project...' - # [END bigtable_list_tables] + # [END List existing tables in the instance.] + + # Display name of the table. + print 'Printing table metadata...' + print table.table_id # [START bigtable_create_family_gc_max_age] print 'Creating column family cf1 with with MaxAge GC Rule...' @@ -217,23 +221,6 @@ def run_table_operations(project_id, instance_id, table_id): print 'execute command python tableadmin.py delete [project_id] \ [instance_id] --table [tableName] to delete the table.' -def get_metadata(table): - """ Get table metadata. - - .. note:: - - This is temporary function as code for this is planned for\ - development. Once complete, this function would be removed. - - :type table: Table Class. - :param table: Table object. - Returns result dictionary of table metadata - """ - result = {("Table ID ", table.table_id): {}} - column_families = table.list_column_families() - for column_family, gc_rule in sorted(column_families.items()): - result[("Table ID ", table.table_id)][("Column Family ", column_family)] = gc_rule.to_pb() - return result def exists(instance_obj, table_id): """ Check whether table exists or not. @@ -247,7 +234,7 @@ def exists(instance_obj, table_id): :param instance_obj: Instance object. :type table_id: str - :param table_id: Table id to identify table. + :param table_id: Table id to create table. Returns bool """ for table in instance_obj.list_tables(): @@ -272,8 +259,7 @@ def delete_table(project_id, instance_id, table_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) table = instance.table(table_id) - # [START bigtable_delete_table] - # Delete the entire table + print 'Checking if table {} exists...'.format(table_id) if exists(instance, table_id): print 'Table {} exists.'.format(table_id) @@ -282,7 +268,6 @@ def delete_table(project_id, instance_id, table_id): print 'Deleted {} table.'.format(table_id) else: print 'Table {} does not exists.'.format(table_id) - # [END bigtable_delete_table] if __name__ == '__main__': diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index b324c5a342b..b0a764decc0 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -31,8 +31,7 @@ def test_run_table_operations(capsys): run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - - assert 'Creating the ' + table_name + ' table.' in out + assert 'Creating the {} table.'.format(table_name) in out assert 'Listing tables in current project.' in out assert 'Creating column family cf1 with with MaxAge GC Rule' in out assert 'Created MaxAge GC rule.' in out @@ -62,6 +61,6 @@ def test_delete_table(capsys): delete_table(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - assert 'Table ' + table_name + ' exists.' in out - assert 'Deleting ' + table_name + ' table.' in out - assert 'Deleted ' + table_name + ' table.' in out + assert 'Table {} exists.'.format(table_name) in out + assert 'Deleting {} table.'.format(table_name) in out + assert 'Deleted {} table.'.format(table_name) in out From 4132b9701d06519cabc130fda425945726304016 Mon Sep 17 00:00:00 2001 From: Sangram Date: Fri, 13 Jul 2018 23:44:38 +0530 Subject: [PATCH 12/14] Remove extra prints and variables Remove extra prints and variables. Fix tests accordingly. --- bigtable/tableadmin/tableadmin.py | 106 ++++++++++--------------- bigtable/tableadmin/tableadmin_test.py | 16 ++-- 2 files changed, 49 insertions(+), 73 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 889f134ceb0..e3d0f735693 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -63,7 +63,7 @@ def run_table_operations(project_id, instance_id, table_id): table.create() print 'Created table {}.'.format(table_id) - # [START List existing tables in the instance.] + # [START bigtable_list_tables] tables = instance.list_tables() print 'Listing tables in current project...' if tables != []: @@ -71,23 +71,17 @@ def run_table_operations(project_id, instance_id, table_id): print tbl.table_id else: print 'No table exists in current project...' - # [END List existing tables in the instance.] - - # Display name of the table. - print 'Printing table metadata...' - print table.table_id + # [END bigtable_list_tables] # [START bigtable_create_family_gc_max_age] print 'Creating column family cf1 with with MaxAge GC Rule...' # Create a column family with GC policy : maximum age # where age = current time minus cell timestamp - column_family_id1 = 'cf1' + # Define the GC rule to retain data with max age of 5 days - max_age = datetime.timedelta(days=5) - max_age_rule = table.max_age_gc_rule(max_age) - print 'Created MaxAge GC rule.' + max_age_rule = table.max_age_gc_rule(datetime.timedelta(days=5)) - cf1 = table.column_family(column_family_id1, max_age_rule) + cf1 = table.column_family('cf1', max_age_rule) cf1.create() print 'Created column family cf1 with MaxAge GC Rule.' # [END bigtable_create_family_gc_max_age] @@ -96,13 +90,11 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf2 with max versions GC rule...' # Create a column family with GC policy : most recent N versions # where 1 = most recent version - column_family_id2 = 'cf2' + # Define the GC policy to retain only the most recent 2 versions - max_versions = 2 - max_versions_rule = table.max_versions_gc_rule(max_versions) - print 'Created Max Versions GC Rule.' + max_versions_rule = table.max_versions_gc_rule(2) - cf2 = table.column_family(column_family_id2, max_versions_rule) + cf2 = table.column_family('cf2', max_versions_rule) cf2 .create() print 'Created column family cf2 with Max Versions GC Rule.' # [END bigtable_create_family_gc_max_versions] @@ -111,13 +103,13 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf3 with union GC rule...' # Create a column family with GC policy to drop data that matches # at least one condition. - column_family_id3 = 'cf3' - # GC rule : Drop max age rule OR the most recent version rule. - union = [max_age_rule, max_versions_rule] - union_rule = table.gc_rule_union(union) - print 'Created Union GC Rule.' + # Define a GC rule to drop cells older than 5 days or not the \ + # most recent version + union_rule = table.gc_rule_union([ \ + table.max_age_gc_rule(datetime.timedelta(days=5)),\ + table.max_versions_gc_rule(2)]) - cf3 = table.column_family(column_family_id3, union_rule) + cf3 = table.column_family('cf3', union_rule) cf3.create() print 'Created column family cf3 with Union GC rule' # [END bigtable_create_family_gc_union] @@ -126,45 +118,33 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf4 with Intersection GC rule...' # Create a column family with GC policy to drop data that matches # all conditions - column_family_id4 = 'cf4' # GC rule: Drop cells older than 5 days AND older than the most # recent 2 versions - intersection = [max_age_rule, max_versions_rule] - intersection_rule = table.gc_rule_intersection(union) - print 'Created Intersection GC Rule.' + intersection_rule = table.gc_rule_intersection([ \ + table.max_age_gc_rule(datetime.timedelta(days=5)),\ + table.max_versions_gc_rule(2)]) - cf4 = table.column_family(column_family_id4, intersection_rule) + cf4 = table.column_family('cf4', intersection_rule) cf4.create() print 'Created column family cf4 with Intersection GC rule.' # [END bigtable_create_family_gc_intersection] # [START bigtable_create_family_gc_nested] print 'Creating column family cf5 with a Nested GC rule...' - # Create a column family with nested GC policys. + # Create a column family with nested GC policies. # Create a nested GC rule: # Drop cells that are either older than the 10 recent versions # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - column_family_id5 = 'cf5' - # Drop cells that are either older than the 10 recent versions - max_versions_rule1 = table.max_versions_gc_rule(10) - - # Drop cells that are older than a month AND older than - # the 2 recent versions - max_age = datetime.timedelta(days=30) - max_age_rule = table.max_age_gc_rule(max_age) - max_versions_rule2 = table.max_versions_gc_rule(2) - intersection = [max_age_rule, max_versions_rule2] - intersection_rule = table.gc_rule_intersection(intersection) - - # This nesting is done with union rule since it is OR between - # the selected rules. - nest = [max_versions_rule1, intersection_rule] - nested_rule = table.gc_rule_union(nest) - print 'Created Nested GC Rule.' - - cf5 = table.column_family(column_family_id5, nested_rule) + # Drop cells that are older than a month AND older than the 2 recent versions + rule1 = table.max_versions_gc_rule(10) + rule2 = table.gc_rule_intersection([ \ + table.max_age_gc_rule(datetime.timedelta(days=30)), \ + table.max_versions_gc_rule(2) \ + ]) + + nested_rule = table.gc_rule_union([rule1, rule2]) + + cf5 = table.column_family('cf5', nested_rule) cf5.create() print 'Created column family cf5 with a Nested GC rule.' # [END bigtable_create_family_gc_nested] @@ -193,23 +173,21 @@ def run_table_operations(project_id, instance_id, table_id): # } # [END bigtable_list_column_families] + print 'Print column family cf1 GC rule before update...' + print "Column Family: cf1" + print cf1.to_pb() + # [START bigtable_update_gc_rule] print 'Updating column family cf1 GC rule...' # Update the column family cf1 to update the GC rule - max_versions = 1 - max_versions_rule = table.max_versions_gc_rule(max_versions) - # Create a reference to the column family with GC rule - cf1 = table.column_family(column_family_id1, max_versions_rule) + cf1 = table.column_family('cf1', table.max_versions_gc_rule(1)) cf1.update() - print 'Updated column family cf1 GC rule' + print 'Updated column family cf1 GC rule\n' # [END bigtable_update_gc_rule] - # [START bigtable_family_get_gc_rule] - print 'Print updated column family cf1 GC rule...' - print 'Column Family:', column_family_id1 - print 'GC Rule:' + print 'Print column family cf1 GC rule after update...' + print "Column Family: cf1" print cf1.to_pb() - # [END bigtable_family_get_gc_rule] # [START bigtable_delete_family] print 'Delete a column family cf2...' @@ -218,8 +196,8 @@ def run_table_operations(project_id, instance_id, table_id): print 'Column family cf2 deleted successfully.' # [END bigtable_delete_family] - print 'execute command python tableadmin.py delete [project_id] \ - [instance_id] --table [tableName] to delete the table.' + print 'execute command "python tableadmin.py delete [project_id] \ + [instance_id] --table [tableName]" to delete the table.' def exists(instance_obj, table_id): @@ -234,7 +212,7 @@ def exists(instance_obj, table_id): :param instance_obj: Instance object. :type table_id: str - :param table_id: Table id to create table. + :param table_id: Table id to identify table. Returns bool """ for table in instance_obj.list_tables(): @@ -259,7 +237,8 @@ def delete_table(project_id, instance_id, table_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) table = instance.table(table_id) - + # [START bigtable_delete_table] + # Delete the entire table print 'Checking if table {} exists...'.format(table_id) if exists(instance, table_id): print 'Table {} exists.'.format(table_id) @@ -268,6 +247,7 @@ def delete_table(project_id, instance_id, table_id): print 'Deleted {} table.'.format(table_id) else: print 'Table {} does not exists.'.format(table_id) + # [END bigtable_delete_table] if __name__ == '__main__': diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py index b0a764decc0..df384a18b80 100755 --- a/bigtable/tableadmin/tableadmin_test.py +++ b/bigtable/tableadmin/tableadmin_test.py @@ -31,23 +31,19 @@ def test_run_table_operations(capsys): run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - assert 'Creating the {} table.'.format(table_name) in out + + assert 'Creating the ' + table_name + ' table.' in out assert 'Listing tables in current project.' in out assert 'Creating column family cf1 with with MaxAge GC Rule' in out - assert 'Created MaxAge GC rule.' in out assert 'Created column family cf1 with MaxAge GC Rule.' in out - assert 'Created Max Versions GC Rule.' in out assert 'Created column family cf2 with Max Versions GC Rule.' in out - assert 'Created Union GC Rule.' in out assert 'Created column family cf3 with Union GC rule' in out - assert 'Created Intersection GC Rule.' in out assert 'Created column family cf4 with Intersection GC rule.' in out - assert 'Created Nested GC Rule.' in out assert 'Created column family cf5 with a Nested GC rule.' in out assert 'Printing Column Family and GC Rule for all column families.' in out assert 'Updating column family cf1 GC rule...' in out assert 'Updated column family cf1 GC rule' in out - assert 'Print updated column family cf1 GC rule...' in out + assert 'Print column family cf1 GC rule after update...' in out assert 'Column Family: cf1' in out assert 'max_num_versions: 1' in out assert 'Delete a column family cf2...' in out @@ -61,6 +57,6 @@ def test_delete_table(capsys): delete_table(PROJECT, BIGTABLE_CLUSTER, table_name) out, _ = capsys.readouterr() - assert 'Table {} exists.'.format(table_name) in out - assert 'Deleting {} table.'.format(table_name) in out - assert 'Deleted {} table.'.format(table_name) in out + assert 'Table ' + table_name + ' exists.' in out + assert 'Deleting ' + table_name + ' table.' in out + assert 'Deleted ' + table_name + ' table.' in out From fb8fd3745f34173a05aabdaff059898cdab1802f Mon Sep 17 00:00:00 2001 From: Sangram Date: Sat, 14 Jul 2018 00:09:15 +0530 Subject: [PATCH 13/14] Fix build failures. --- bigtable/tableadmin/tableadmin.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index e3d0f735693..ba9aae45db4 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -103,11 +103,11 @@ def run_table_operations(project_id, instance_id, table_id): print 'Creating column family cf3 with union GC rule...' # Create a column family with GC policy to drop data that matches # at least one condition. - # Define a GC rule to drop cells older than 5 days or not the \ + # Define a GC rule to drop cells older than 5 days or not the # most recent version - union_rule = table.gc_rule_union([ \ - table.max_age_gc_rule(datetime.timedelta(days=5)),\ - table.max_versions_gc_rule(2)]) + union_rule = table.gc_rule_union([ + table.max_age_gc_rule(datetime.timedelta(days=5)), + table.max_versions_gc_rule(2)]) cf3 = table.column_family('cf3', union_rule) cf3.create() @@ -120,9 +120,9 @@ def run_table_operations(project_id, instance_id, table_id): # all conditions # GC rule: Drop cells older than 5 days AND older than the most # recent 2 versions - intersection_rule = table.gc_rule_intersection([ \ - table.max_age_gc_rule(datetime.timedelta(days=5)),\ - table.max_versions_gc_rule(2)]) + intersection_rule = table.gc_rule_intersection([ + table.max_age_gc_rule(datetime.timedelta(days=5)), + table.max_versions_gc_rule(2)]) cf4 = table.column_family('cf4', intersection_rule) cf4.create() @@ -135,12 +135,12 @@ def run_table_operations(project_id, instance_id, table_id): # Create a nested GC rule: # Drop cells that are either older than the 10 recent versions # OR - # Drop cells that are older than a month AND older than the 2 recent versions + # Drop cells that are older than a month AND older than the + # 2 recent versions rule1 = table.max_versions_gc_rule(10) - rule2 = table.gc_rule_intersection([ \ - table.max_age_gc_rule(datetime.timedelta(days=30)), \ - table.max_versions_gc_rule(2) \ - ]) + rule2 = table.gc_rule_intersection([ + table.max_age_gc_rule(datetime.timedelta(days=30)), + table.max_versions_gc_rule(2)]) nested_rule = table.gc_rule_union([rule1, rule2]) From f0bf0d8d96c1428e7b9df5f687dda58fd0fd03a7 Mon Sep 17 00:00:00 2001 From: Sangram Date: Sat, 14 Jul 2018 00:12:41 +0530 Subject: [PATCH 14/14] Fix white space --- bigtable/tableadmin/tableadmin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py index 41ee715f798..f3922d5d16c 100644 --- a/bigtable/tableadmin/tableadmin.py +++ b/bigtable/tableadmin/tableadmin.py @@ -63,7 +63,7 @@ def run_table_operations(project_id, instance_id, table_id): table.create() print 'Created table {}.'.format(table_id) - # [START bigtable_list_tables] + # [START bigtable_list_tables] tables = instance.list_tables() print 'Listing tables in current project...' if tables != []: