46
46
47
47
48
48
def _split_names_string_builder (split_names_list : List ):
49
-
50
49
"""
51
50
_split_names_string_builder() creates a string of split-names for input to
52
51
output_example.split_names property.
53
52
54
53
"""
55
54
56
55
str1 = "["
57
- urlist_len = len (split_names_list )- 1
56
+ urlist_len = len (split_names_list ) - 1
58
57
index = 0
59
58
60
59
for element in split_names_list :
61
- if (index == urlist_len ):
62
- str1 += "\" " + element + "\" " + "]"
60
+ if (index == urlist_len ):
61
+ str1 += "\" " + element + "\" " + "]"
63
62
break
64
- str1 += "\" " + element + "\" " + ","
65
- index += 1
63
+ str1 += "\" " + element + "\" " + ","
64
+ index += 1
66
65
return str1
67
66
67
+
68
68
@component
69
69
def CopyExampleGen (
70
70
input_json_str : tfx .dsl .components .Parameter [str ],
@@ -84,23 +84,24 @@ def CopyExampleGen(
84
84
input_dict = json .loads (input_json_str )
85
85
86
86
# Parse input_dict: creates a directory from the split-names and tfrecord uris provided
87
- split_names = []
87
+ split_names = []
88
88
for key , value in input_dict .items ():
89
89
split_names .append (key )
90
90
91
- split_names_string = _split_names_string_builder (split_names )
92
- output_example .split_names = str (split_names_string )
91
+ split_names_string = _split_names_string_builder (split_names )
92
+ output_example .split_names = str (split_names_string )
93
93
94
94
# Make directories
95
- tfrecords_list = []
96
- output_example_uri = output_example .uri
95
+ tfrecords_list = []
96
+ output_example_uri = output_example .uri
97
97
98
98
for key , value in input_dict .items ():
99
99
split_value = (f"/Split-{ key } /" )
100
100
fileio .mkdir (f"{ output_example_uri } { split_value } " )
101
- tfrecords_list = fileio .glob (f"{ input_dict [key ]} *.gz" )
101
+ tfrecords_list = fileio .glob (f"{ input_dict [key ]} *.gz" )
102
102
103
103
# Copy files into directories
104
104
for tfrecord in tfrecords_list :
105
- file_name = os .path .basename (os .path .normpath (tfrecord ))
106
- fileio .copy (tfrecord , output_example .uri + split_value + file_name , True )
105
+ file_name = os .path .basename (os .path .normpath (tfrecord ))
106
+ fileio .copy (tfrecord , output_example .uri + split_value + file_name ,
107
+ True )
0 commit comments