forked from rakeshcusat/Code4Reference
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathgithubTest.py
More file actions
executable file
·283 lines (229 loc) · 10.7 KB
/
githubTest.py
File metadata and controls
executable file
·283 lines (229 loc) · 10.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
#!/usr/bin/env python
#import github3 #It is not used currently
from github import Github
from datetime import datetime
import argparse
import sys
import time
import traceback
#For sending email-notification
from smtplib import SMTP
from smtplib import SMTPException
try:
import MySQLdb
except ImportError:
print ('Could not find package "MySQLdb". Try installing it with:')
print (' sudo apt-get install python-mysqldb')
print ('')
print ('Halting.')
sys.exit(1)
#Some global variables
DB_NAME = "githubdb"
TABLE_NAME = "github_user"
DB_PSWD="root"
DB_USER="root"
DB_HOST="localhost"
MAX_ATTEMPTS = 6
#Create data base statement for scema creation.
CREATE_DATA_BASE = "CREATE DATABASE IF NOT EXISTS {db_name}".format(db_name=DB_NAME)
#Create table statement
CREATE_USERINFO_TABLE = "CREATE TABLE IF NOT EXISTS {db_name}.{table_name} ( " \
"id INTEGER NOT NULL, " \
"name VARCHAR(200), " \
"login VARCHAR(200) NOT NULL, " \
"email VARCHAR(200), " \
"location VARCHAR(200), " \
"company VARCHAR(200), " \
"repos INTEGER, " \
"created DATETIME, " \
" PRIMARY KEY (id)" \
")" \
.format(db_name=DB_NAME, table_name=TABLE_NAME)
#Select max id statement
SELECT_MAX_ID = "SELECT MAX(id) FROM {db_name}.{table_name}".format(db_name=DB_NAME, table_name=TABLE_NAME)
#Insert record statement
INSERT_RECORD = u"INSERT INTO {db_name}.{table_name} " \
u"( id, name, login, email, location, company, repos, created) " \
u"VALUES (%s, %s, %s, %s, %s, %s, %s, %s)".format(db_name=DB_NAME, table_name=TABLE_NAME)
#e-mail related variables
EMAIL_SUBJECT = "Github test notification"
EMAIL_SUBJECT_URGENT = "[URGENT]: " + EMAIL_SUBJECT
EMAIL_FROM = "notification@code4reference.com"
EMAIL_RECEIVERS = ['hamepal@gmail.com']
def listToStr(lst):
"""This method makes comma separated list item string"""
return ','.join(lst)
def send_email(sub, msg):
"""This method sends an email"""
msg_header = "From: " + EMAIL_FROM + "\n" + \
"To: " + listToStr(EMAIL_RECEIVERS) + "\n" + \
"Subject: " + sub + "\n"
msg_body = msg_header + msg
try:
smtpObj = SMTP('localhost')
smtpObj.sendmail(EMAIL_FROM, EMAIL_RECEIVERS, msg_body)
smtpObj.quit()
except SMTPException as error:
print "Error: unable to send email : {err}".format(err=error)
def get_cur_time():
"""Utility method to return the current time in string"""
return datetime.now().strftime("%d-%m-%y-%H:%M:%S")
def create_schema(db_con, db_cursor):
"""This mehtod creates the database table"""
try:
#First create the database.
db_cursor.execute(CREATE_DATA_BASE)
#Now add define table.
db_cursor.execute(CREATE_USERINFO_TABLE)
db_con.commit()
except MySQLdb.ProgrammingError as error:
print("Error : "+str(error))
except MySQLdb.IntegrityError as error:
print("Error : "+str(error))
except Exception as error:
print("Couldn't create {table} ".format(table=tableName))
print("Error : {err}".format(err=error))
def select_max_id(db_cursor):
"""This method selects max id from github_user table."""
try:
db_cursor.execute(SELECT_MAX_ID)
result = db_cursor.fetchone()
except Exception as error:
print ("Couldn't fetch max id from {table}".format(table=TABLE_NAME))
print("Error : {err}".format(err=error))
if result[0]:
print ("Max Id : {rslt}".format(rslt=result[0]))
return result[0]
else:
print ("There are no user info in db")
return 0
def insert_user_info(db_con,
db_cursor,
user_id,
name,
login,
email,
location,
company,
repos,
created_date):
"""This method inserts record in githubdb.github_user table."""
try:
db_cursor.execute(INSERT_RECORD, (user_id,
name,
login,
email,
location,
company,
repos,
created_date))
db_con.commit()
except MySQLdb.ProgrammingError as error:
print("Error : "+str(error))
except MySQLdb.IntegrityError as error:
print("Error : "+str(error))
except Exception as error:
print("Couldn't insert into {table}".format(table=TABLE_NAME))
print("Error : {err}".format(err=error))
def get_pygithub_user_details(db_con, db_cursor, args):
#Create Github object to access git API.
gh = Github(login_or_token=args.git_user, password=args.git_password, per_page=100)
while_count = 0
try:
while while_count < MAX_ATTEMPTS:
try:
for user in gh.get_users(since=select_max_id(db_cursor)):
try:
insert_user_info(db_con,
db_cursor,
user.id,
(u'' + user.name) if user.name else None,
(u'' + user.login),
(u'' + user.email) if user.email else None,
(u'' + user.location) if user.location else None,
(u'' + user.company) if user.company else None,
user.public_repos,
user.created_at)
except Exception as error:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_and_send_exception("Got excetpion , but moving forward anyways",
EMAIL_SUBJECT,
exc_type,
exc_value,
exc_traceback)
print "{cur_time} Rate limit : {limit}".format(cur_time=get_cur_time(),
limit=gh.rate_limiting)
#We should sleep for sometime to throttle the API call.
time.sleep(0.5)
if gh.rate_limiting[0] == 0:
print ("API call has exceeded the limit. We should wait for 15 mins.")
time.sleep(900)
print ("Resuming it again at {cur_time}".format(cur_time=get_cur_time()))
except RateLimitExceededException:
send_email(EMAIL_SUBJECT,
"Rate limit has exhausted, will resume after 15 mins.")
time.sleep(900)
while_count +=1
except Exception as error:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_and_send_exception("Got excetpion , terminating", EMAIL_SUBJECT_URGENT, exc_type, exc_value, exc_traceback)
print (error_message)
sys.exit(1)
#Finally send a termination email.
send_email(EMAIL_SUBJECT, "Seems everything is done")
def print_and_send_exception(initial_msg, sub, exc_type, exc_value, exc_traceback):
error_message = initial_msg + ": {err} ".format(err=repr(traceback.format_exception(exc_type,
exc_value,
exc_traceback)))
print (error_message)
send_email(sub,error_message)
def get_github3_user_details(db_con, db_cursor, args):
"""This method uses github3 library"""
gh = github3.login(username=args.git_user, password=git_password)
for user in gh.iter_all_users():
user.refresh()
try:
fhandler.write(" user: {0}, email: {1}, location: {2}\n".format(str(user), str(user.email), str(user.location)))
except:
print "Something wrong, user id : {0}".format(user.id);
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-u",
"--gituser",
dest="git_user",
help="Github user name",
required=True)
parser.add_argument("-p",
"--getpswd",
dest="git_password",
help="Github password",
required=True)
parser.add_argument("--dbuser",
dest="db_user",
default=DB_USER,
help="Database user name")
parser.add_argument("--dbpswd",
dest="db_pswd",
default=DB_PSWD,
help="Database password")
parser.add_argument("--dbhost",
dest="db_host",
default=DB_HOST,
help="Database host name")
parser.add_argument("-c",
"--createschema",
dest="create_schema",
action="store_true",
default=False,
help="Create the Data Base schema")
args = parser.parse_args()
print "args {ar}".format(ar=args)
con = MySQLdb.connect(user=args.db_user, passwd=args.db_pswd, db=DB_NAME, host=args.db_host)
db_cursor = con.cursor()
if args.create_schema:
create_schema(con, db_cursor)
get_pygithub_user_details(con, db_cursor, args)
con.close()
sys.exit(0)
if __name__ == "__main__":
main()