Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit bc0924a

Browse filesBrowse files
waprinwaprin
authored
Update Bigtable Programmatic Scaling Example (GoogleCloudPlatform#1003)
* Update Bigtable Programmatic Scaling Example * Rename "autoscaling" to "metricscaler" and the the term "programmatic scaling" * Remove `strategies.py` to simplify example * Fix wrong sleep length bug * Add maximum node count * hegemonic review
1 parent 57fbe30 commit bc0924a
Copy full SHA for bc0924a

File tree

Expand file treeCollapse file tree

7 files changed

+47
-112
lines changed
Filter options
Expand file treeCollapse file tree

7 files changed

+47
-112
lines changed

‎bigtable/autoscaler/strategies.py

Copy file name to clipboardExpand all lines: bigtable/autoscaler/strategies.py
-51Lines changed: 0 additions & 51 deletions
This file was deleted.

‎bigtable/autoscaler/strategies_test.py

Copy file name to clipboardExpand all lines: bigtable/autoscaler/strategies_test.py
-30Lines changed: 0 additions & 30 deletions
This file was deleted.

‎bigtable/autoscaler/README.rst renamed to ‎bigtable/metricscaler/README.rst

Copy file name to clipboardExpand all lines: bigtable/metricscaler/README.rst
+6-6Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ Install Dependencies
7474
Samples
7575
-------------------------------------------------------------------------------
7676

77-
Autoscaling example
77+
Metricscaling example
7878
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
7979

8080

@@ -83,12 +83,12 @@ To run this sample:
8383

8484
.. code-block:: bash
8585
86-
$ python autoscaler.py
86+
$ python metricscaler.py
8787
88-
usage: autoscaler.py [-h] [--high_cpu_threshold HIGH_CPU_THRESHOLD]
89-
[--low_cpu_threshold LOW_CPU_THRESHOLD]
90-
[--short_sleep SHORT_SLEEP] [--long_sleep LONG_SLEEP]
91-
bigtable_instance bigtable_cluster
88+
usage: metricscaler.py [-h] [--high_cpu_threshold HIGH_CPU_THRESHOLD]
89+
[--low_cpu_threshold LOW_CPU_THRESHOLD]
90+
[--short_sleep SHORT_SLEEP] [--long_sleep LONG_SLEEP]
91+
bigtable_instance bigtable_cluster
9292
9393
Scales Cloud Bigtable clusters based on CPU usage.
9494

‎bigtable/autoscaler/README.rst.in renamed to ‎bigtable/metricscaler/README.rst.in

Copy file name to clipboardExpand all lines: bigtable/metricscaler/README.rst.in
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ setup:
2020
- install_deps
2121

2222
samples:
23-
- name: Autoscaling example
24-
file: autoscaler.py
23+
- name: Metricscaling example
24+
file: metricscaler.py
2525
show_help: true
2626

2727
cloud_client_library: true

‎bigtable/autoscaler/autoscaler.py renamed to ‎bigtable/metricscaler/metricscaler.py

Copy file name to clipboardExpand all lines: bigtable/metricscaler/metricscaler.py
+31-15Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
from google.cloud import bigtable
2222
from google.cloud import monitoring
2323

24-
import strategies
2524

2625

2726
def get_cpu_load():
@@ -52,6 +51,23 @@ def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up):
5251
bigtable_cluster (str): Cloud Bigtable cluster ID to scale
5352
scale_up (bool): If true, scale up, otherwise scale down
5453
"""
54+
_MIN_NODE_COUNT = 3
55+
"""
56+
The minimum number of nodes to use. The default minimum is 3. If you have a
57+
lot of data, the rule of thumb is to not go below 2.5 TB per node for SSD
58+
clusters, and 8 TB for HDD. The bigtable.googleapis.com/disk/bytes_used
59+
metric is useful in figuring out the minimum number of nodes.
60+
"""
61+
62+
_MAX_NODE_COUNT = 30
63+
"""
64+
The maximum number of nodes to use. The default maximum is 30 nodes per zone.
65+
If you need more quota, you can request more by following the instructions
66+
<a href="https://cloud.google.com/bigtable/quota">here</a>.
67+
"""
68+
69+
_SIZE_CHANGE_STEP = 3
70+
"""The number of nodes to change the cluster by."""
5571
# [START bigtable_scale]
5672
bigtable_client = bigtable.Client(admin=True)
5773
instance = bigtable_client.instance(bigtable_instance)
@@ -62,21 +78,21 @@ def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up):
6278

6379
current_node_count = cluster.serve_nodes
6480

65-
if current_node_count <= 3 and not scale_up:
66-
# Can't downscale lower than 3 nodes
67-
return
68-
6981
if scale_up:
70-
strategies_dict = strategies.UPSCALE_STRATEGIES
82+
if current_node_count < _MAX_NODE_COUNT:
83+
new_node_count = min(current_node_count + 3, _MAX_NODE_COUNT)
84+
cluster.serve_nodes = new_node_count
85+
cluster.update()
86+
print('Scaled up from {} to {} nodes.'.format(
87+
current_node_count, new_node_count))
7188
else:
72-
strategies_dict = strategies.DOWNSCALE_STRATEGIES
73-
74-
strategy = strategies_dict['incremental']
75-
new_node_count = strategy(cluster.serve_nodes)
76-
cluster.serve_nodes = new_node_count
77-
cluster.update()
78-
print('Scaled from {} up to {} nodes.'.format(
79-
current_node_count, new_node_count))
89+
if current_node_count > _MIN_NODE_COUNT:
90+
new_node_count = max(
91+
current_node_count - _SIZE_CHANGE_STEP, _MIN_NODE_COUNT)
92+
cluster.serve_nodes = new_node_count
93+
cluster.update()
94+
print('Scaled down from {} to {} nodes.'.format(
95+
current_node_count, new_node_count))
8096
# [END bigtable_scale]
8197

8298

@@ -104,7 +120,7 @@ def main(
104120
time.sleep(long_sleep)
105121
elif cluster_cpu < low_cpu_threshold:
106122
scale_bigtable(bigtable_instance, bigtable_cluster, False)
107-
time.sleep(short_sleep)
123+
time.sleep(long_sleep)
108124
else:
109125
print('CPU within threshold, sleeping.')
110126
time.sleep(short_sleep)

‎bigtable/autoscaler/autoscaler_test.py renamed to ‎bigtable/metricscaler/metricscaler_test.py

Copy file name to clipboardExpand all lines: bigtable/metricscaler/metricscaler_test.py
+8-8Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,18 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
"""Unit and system tests for autoscaler.py"""
15+
"""Unit and system tests for metricscaler.py"""
1616

1717
import os
1818
import time
1919

2020
from google.cloud import bigtable
2121
from mock import patch
2222

23-
from autoscaler import get_cpu_load
24-
from autoscaler import main
25-
from autoscaler import scale_bigtable
23+
from metricscaler import _SIZE_CHANGE_STEP
24+
from metricscaler import get_cpu_load
25+
from metricscaler import main
26+
from metricscaler import scale_bigtable
2627

2728
# tests assume instance and cluster have the same ID
2829
BIGTABLE_INSTANCE = os.environ['BIGTABLE_CLUSTER']
@@ -49,7 +50,7 @@ def test_scale_bigtable():
4950
cluster.reload()
5051

5152
new_node_count = cluster.serve_nodes
52-
assert (new_node_count == (original_node_count + 2))
53+
assert (new_node_count == (original_node_count + _SIZE_CHANGE_STEP))
5354

5455
scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False)
5556
time.sleep(3)
@@ -59,10 +60,9 @@ def test_scale_bigtable():
5960

6061

6162
# Unit test for logic
62-
6363
@patch('time.sleep')
64-
@patch('autoscaler.get_cpu_load')
65-
@patch('autoscaler.scale_bigtable')
64+
@patch('metricscaler.get_cpu_load')
65+
@patch('metricscaler.scale_bigtable')
6666
def test_main(scale_bigtable, get_cpu_load, sleep):
6767
SHORT_SLEEP = 5
6868
LONG_SLEEP = 10

‎bigtable/autoscaler/requirements.txt renamed to ‎bigtable/metricscaler/requirements.txt

Copy file name to clipboard
File renamed without changes.

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.