Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 57b1bd5

Browse filesBrowse files
tseavertseaver
committed
Initial system tests for BQ.
1 parent 2d017de commit 57b1bd5
Copy full SHA for 57b1bd5

3 files changed

+226Lines changed: 226 additions & 0 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎scripts/run_system_tests.sh‎

Copy file name to clipboardExpand all lines: scripts/run_system_tests.sh
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,4 @@ fi
3838
python system_tests/run_system_test.py --package datastore
3939
python system_tests/run_system_test.py --package storage
4040
python system_tests/run_system_test.py --package pubsub
41+
python system_tests/run_system_test.py --package bigquery
Collapse file

‎system_tests/bigquery.py‎

Copy file name to clipboard
+224Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
# Copyright 2015 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import operator
16+
import time
17+
18+
import unittest2
19+
20+
from gcloud import _helpers
21+
from gcloud.environment_vars import TESTS_PROJECT
22+
from gcloud import bigquery
23+
24+
25+
_helpers.PROJECT = TESTS_PROJECT
26+
CLIENT = bigquery.Client()
27+
28+
29+
class TestPubsub(unittest2.TestCase):
30+
31+
def setUp(self):
32+
self.to_delete = []
33+
34+
def tearDown(self):
35+
for doomed in self.to_delete:
36+
doomed.delete()
37+
38+
def test_create_dataset(self):
39+
DATASET_NAME = 'system_tests'
40+
dataset = CLIENT.dataset(DATASET_NAME)
41+
self.assertFalse(dataset.exists())
42+
dataset.create()
43+
self.to_delete.append(dataset)
44+
self.assertTrue(dataset.exists())
45+
self.assertEqual(dataset.name, DATASET_NAME)
46+
47+
def test_patch_dataset(self):
48+
DATASET_NAME = 'system_tests'
49+
dataset = CLIENT.dataset(DATASET_NAME)
50+
self.assertFalse(dataset.exists())
51+
dataset.create()
52+
self.to_delete.append(dataset)
53+
self.assertTrue(dataset.exists())
54+
self.assertEqual(dataset.friendly_name, None)
55+
self.assertEqual(dataset.description, None)
56+
dataset.patch(friendly_name='Friendly', description='Description')
57+
self.assertEqual(dataset.friendly_name, 'Friendly')
58+
self.assertEqual(dataset.description, 'Description')
59+
60+
def test_update_dataset(self):
61+
DATASET_NAME = 'system_tests'
62+
dataset = CLIENT.dataset(DATASET_NAME)
63+
self.assertFalse(dataset.exists())
64+
dataset.create()
65+
self.to_delete.append(dataset)
66+
self.assertTrue(dataset.exists())
67+
after = [grant for grant in dataset.access_grants
68+
if grant.entity_id != 'projectWriters']
69+
dataset.access_grants = after
70+
dataset.update()
71+
self.assertEqual(len(dataset.access_grants), len(after))
72+
for found, expected in zip(dataset.access_grants, after):
73+
self.assertEqual(found.role, expected.role)
74+
self.assertEqual(found.entity_type, expected.entity_type)
75+
self.assertEqual(found.entity_id, expected.entity_id)
76+
77+
def test_list_datasets(self):
78+
datasets_to_create = [
79+
'new%d' % (1000 * time.time(),),
80+
'newer%d' % (1000 * time.time(),),
81+
'newest%d' % (1000 * time.time(),),
82+
]
83+
for dataset_name in datasets_to_create:
84+
dataset = CLIENT.dataset(dataset_name)
85+
dataset.create()
86+
self.to_delete.append(dataset)
87+
88+
# Retrieve the datasets.
89+
all_datasets, _ = CLIENT.list_datasets()
90+
created = [dataset for dataset in all_datasets
91+
if dataset.name in datasets_to_create and
92+
dataset.project == CLIENT.project]
93+
self.assertEqual(len(created), len(datasets_to_create))
94+
95+
def test_create_table(self):
96+
DATASET_NAME = 'system_tests'
97+
dataset = CLIENT.dataset(DATASET_NAME)
98+
self.assertFalse(dataset.exists())
99+
dataset.create()
100+
self.to_delete.append(dataset)
101+
TABLE_NAME = 'test_table'
102+
full_name = bigquery.SchemaField('full_name', 'STRING',
103+
mode='REQUIRED')
104+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
105+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
106+
self.assertFalse(table.exists())
107+
table.create()
108+
self.to_delete.insert(0, table)
109+
self.assertTrue(table.exists())
110+
self.assertEqual(table.name, TABLE_NAME)
111+
self.assertTrue(table._dataset is dataset)
112+
113+
def test_list_tables(self):
114+
DATASET_NAME = 'system_tests'
115+
dataset = CLIENT.dataset(DATASET_NAME)
116+
self.assertFalse(dataset.exists())
117+
dataset.create()
118+
self.to_delete.append(dataset)
119+
tables_to_create = [
120+
'new%d' % (1000 * time.time(),),
121+
'newer%d' % (1000 * time.time(),),
122+
'newest%d' % (1000 * time.time(),),
123+
]
124+
full_name = bigquery.SchemaField('full_name', 'STRING',
125+
mode='REQUIRED')
126+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
127+
for table_name in tables_to_create:
128+
table = dataset.table(table_name, schema=[full_name, age])
129+
table.create()
130+
self.to_delete.insert(0, table)
131+
132+
# Retrieve the tables.
133+
all_tables, _ = dataset.list_tables()
134+
created = [table for table in all_tables
135+
if table.name in tables_to_create and
136+
table._dataset.name == DATASET_NAME]
137+
self.assertEqual(len(created), len(tables_to_create))
138+
139+
def test_patch_table(self):
140+
DATASET_NAME = 'system_tests'
141+
dataset = CLIENT.dataset(DATASET_NAME)
142+
self.assertFalse(dataset.exists())
143+
dataset.create()
144+
self.to_delete.append(dataset)
145+
TABLE_NAME = 'test_table'
146+
full_name = bigquery.SchemaField('full_name', 'STRING',
147+
mode='REQUIRED')
148+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
149+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
150+
self.assertFalse(table.exists())
151+
table.create()
152+
self.to_delete.insert(0, table)
153+
self.assertTrue(table.exists())
154+
self.assertEqual(table.friendly_name, None)
155+
self.assertEqual(table.description, None)
156+
table.patch(friendly_name='Friendly', description='Description')
157+
self.assertEqual(table.friendly_name, 'Friendly')
158+
self.assertEqual(table.description, 'Description')
159+
160+
def test_update_table(self):
161+
DATASET_NAME = 'system_tests'
162+
dataset = CLIENT.dataset(DATASET_NAME)
163+
self.assertFalse(dataset.exists())
164+
dataset.create()
165+
self.to_delete.append(dataset)
166+
TABLE_NAME = 'test_table'
167+
full_name = bigquery.SchemaField('full_name', 'STRING',
168+
mode='REQUIRED')
169+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
170+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
171+
self.assertFalse(table.exists())
172+
table.create()
173+
self.to_delete.insert(0, table)
174+
self.assertTrue(table.exists())
175+
voter = bigquery.SchemaField('voter', 'BOOLEAN', mode='NULLABLE')
176+
schema = table.schema
177+
schema.append(voter)
178+
table.schema = schema
179+
table.update()
180+
self.assertEqual(len(table.schema), len(schema))
181+
for found, expected in zip(table.schema, schema):
182+
self.assertEqual(found.name, expected.name)
183+
self.assertEqual(found.field_type, expected.field_type)
184+
self.assertEqual(found.mode, expected.mode)
185+
186+
def test_load_dump_table(self):
187+
ROWS = [
188+
('Phred Phlyntstone', 32),
189+
('Bharney Rhubble', 33),
190+
('Wylma Phlyntstone', 29),
191+
('Bhettye Rhubble', 27),
192+
]
193+
ROW_IDS = range(len(ROWS))
194+
DATASET_NAME = 'system_tests'
195+
dataset = CLIENT.dataset(DATASET_NAME)
196+
self.assertFalse(dataset.exists())
197+
dataset.create()
198+
self.to_delete.append(dataset)
199+
TABLE_NAME = 'test_table'
200+
full_name = bigquery.SchemaField('full_name', 'STRING',
201+
mode='REQUIRED')
202+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
203+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
204+
self.assertFalse(table.exists())
205+
table.create()
206+
self.to_delete.insert(0, table)
207+
self.assertTrue(table.exists())
208+
209+
errors = table.insert_data(ROWS, ROW_IDS)
210+
self.assertEqual(len(errors), 0)
211+
212+
rows = ()
213+
counter = 9
214+
# Allow for 90 seconds of "warm up" before rows visible. See:
215+
# https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability
216+
217+
while len(rows) == 0 and counter > 0:
218+
rows, _, _ = table.fetch_data()
219+
if len(rows) == 0:
220+
time.sleep(10)
221+
222+
by_age = operator.itemgetter(1)
223+
self.assertEqual(sorted(rows, key=by_age),
224+
sorted(ROWS, key=by_age))
Collapse file

‎system_tests/run_system_test.py‎

Copy file name to clipboardExpand all lines: system_tests/run_system_test.py
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
'datastore': ['dataset_id', 'credentials'],
2525
'storage': ['project', 'credentials'],
2626
'pubsub': ['project', 'credentials'],
27+
'bigquery': ['project', 'credentials'],
2728
}
2829

2930

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.