From 9bae3c8c2b46ab05c78f1fec7fe013bea324fd20 Mon Sep 17 00:00:00 2001 From: blkart Date: Mon, 18 Jul 2016 13:50:55 +0800 Subject: [PATCH 01/12] [bugfix] fix redmine 7627 Signed-off-by: blkart --- eayunstack_tools/doctor/cls_func.py | 4 +++- eayunstack_tools/doctor/utils.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/eayunstack_tools/doctor/cls_func.py b/eayunstack_tools/doctor/cls_func.py index 6ed56d2..f32a193 100644 --- a/eayunstack_tools/doctor/cls_func.py +++ b/eayunstack_tools/doctor/cls_func.py @@ -4,6 +4,7 @@ import re from eayunstack_tools.logger import StackLOG as LOG from eayunstack_tools.doctor.utils import run_doctor_on_nodes +from eayunstack_tools.doctor.utils import run_doctor_cmd_on_node # get node list for rabbitmq cluster @@ -111,7 +112,8 @@ def check_all_nodes(check_obj): if check_obj == 'ceph': # only need to check one node for ceph cluster ceph_node = node_list[0] - run_doctor_cmd_on_node('controller', ceph_node, check_cmd) + res = run_doctor_cmd_on_node('controller', ceph_node, check_cmd) + LOG.info(res, remote=True) else: nodes = [] for node in node_list: diff --git a/eayunstack_tools/doctor/utils.py b/eayunstack_tools/doctor/utils.py index 0df2262..399b329 100644 --- a/eayunstack_tools/doctor/utils.py +++ b/eayunstack_tools/doctor/utils.py @@ -78,6 +78,9 @@ def newfunc(*a, **kw): return decorate def run_doctor_cmd_on_node(role, node, cmd): + logging.disable(logging.NOTSET) + LOG.info('%s%s Push check cmd to %-13s (%-10s) %s%s' + % ('<', '='*2, node, role, '='*2, '>')) out, err = ssh_connect2(node, cmd, check_all=True) return out + err @@ -88,8 +91,6 @@ def run_doctor_on_nodes(node_list, check_cmd): pile = eventlet.GreenPile() result = [] for node in node_list: - LOG.info('%s%s Push check cmd to %-13s (%-10s) %s%s' - % ('<', '='*2, node['name'], node['role'], '='*2, '>')) pile.spawn(run_doctor_cmd_on_node, node['role'], node['name'], check_cmd) for node, res in zip(node_list, pile): result.append(res) From fae71216655f576667b82c5a984e7aab56c82e73 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Mon, 18 Jul 2016 11:41:01 +0800 Subject: [PATCH 02/12] Fix cls_func ceph_health, when ceph_health apear error < 150 log do not display [Bug-ES]Redmine #7628 Signed-off-by: fabian --- eayunstack_tools/doctor/cls_func.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/eayunstack_tools/doctor/cls_func.py b/eayunstack_tools/doctor/cls_func.py index f32a193..47130ad 100644 --- a/eayunstack_tools/doctor/cls_func.py +++ b/eayunstack_tools/doctor/cls_func.py @@ -50,10 +50,8 @@ def get_haproxy_nodes(): def ceph_check_health(): def _log(func, msg): func('Ceph cluster check faild !') - # FIXME: cause the log module issue, need to send error msg line - # by line - for l in msg.split('\n'): - func(l) + # so much messages, just log the first line + func(msg.splitlines()[0]) (s, o) = commands.getstatusoutput('ceph health') if s != 0: @@ -64,13 +62,9 @@ def _log(func, msg): else: (ss, oo) = commands.getstatusoutput('ceph health detail') if o.startswith('HEALTH_WARN'): - count = len(oo) - if count > 150: - _log(LOG.warn, oo.splitlines()[0]) + _log(LOG.warn, oo) else: - count = len(oo) - if count > 150: - _log(LOG.error, oo.splitlines()[0]) + _log(LOG.error, oo) # get ceph osd status From b97211fcb4dd47558ccd69ae1b71e13c610b7b72 Mon Sep 17 00:00:00 2001 From: blkart Date: Mon, 18 Jul 2016 16:11:51 +0800 Subject: [PATCH 03/12] release 1.1-9.1 1.[bugfix] fix redmine 7627 2.Fix cls_func ceph_health, when ceph_health apear error < 150 log do not display Signed-off-by: blkart --- eayunstack-tools.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eayunstack-tools.spec b/eayunstack-tools.spec index 4b0f33e..0af7b0f 100644 --- a/eayunstack-tools.spec +++ b/eayunstack-tools.spec @@ -1,6 +1,6 @@ Name: eayunstack-tools Version: 1.1 -Release: 9%{?dist} +Release: 9.1%{?dist} Summary: EayunStack Management tools Group: Application @@ -78,6 +78,9 @@ fi %changelog +* Mon Jul 18 2016 blkart 1.1-9.1 +- commit fae71216655f576667b82c5a984e7aab56c82e73 + * Tue Jun 21 2016 pq <19921207pq@gmail.com> 1.1-9 - commit 7ab007764b21d94b3db7cf2770f69665950e66fa From 16c2dc0d32ef7163bd8a901642b869ba193820d2 Mon Sep 17 00:00:00 2001 From: pengdake <19921207pq@gmail.com> Date: Thu, 11 Aug 2016 17:18:25 +0800 Subject: [PATCH 04/12] [tools][doctor][net] check external interface qos redmin 7669 Signed-off-by: pengdake <19921207pq@gmail.com> --- eayunstack_tools/doctor/net.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/eayunstack_tools/doctor/net.py b/eayunstack_tools/doctor/net.py index cc1042d..6c02fc7 100644 --- a/eayunstack_tools/doctor/net.py +++ b/eayunstack_tools/doctor/net.py @@ -61,6 +61,14 @@ def vrouter_get_gw_remote(l3_host, rid): else: return None +def vrouter_get_gw_interface(l3_host, rid): + cmd = "ip netns exec qrouter-%s ip route show | "\ + "grep 'default' | awk '{print $5}'" % (rid) + out, err = ssh_connect(l3_host, cmd) + if not err: + return out.strip('\n') + else: + return None # Some fuck juno deploy doest not support json format of port-show message def port_result_to_json(out, fmt='json'): @@ -115,10 +123,12 @@ def port_log(device_owner, s): % (device_owner, pid, l3_host)) # 2) ping external gateway to check network status + # and check external interface qos LOG.debug('check gateway for port on %s' % (l3_host)) if device_owner == 'network:router_gateway': LOG.debug('this port is external port, check external gateway') gw = vrouter_get_gw_remote(l3_host, rid) + interface = vrouter_get_gw_interface(l3_host, rid) if gw: LOG.debug("check external gateway %s on %s" % (gw, l3_host)) cmd = "ip netns exec qrouter-%s ping -c 1 %s" % (rid, gw) @@ -126,13 +136,30 @@ def port_log(device_owner, s): if not err: LOG.debug("external gateway is ok") else: - LOG.error("failed to connect external gateway on %s" % (l3_host)) + LOG.error("failed to connect external gateway on %s" + % (l3_host)) + if interface: + LOG.debug("check external interface %s qos on %s" + % (interface, l3_host)) + cmd = ("ip netns exec qrouter-%s tc qdisc show dev %s" + % (rid, interface)) + out, err = ssh_connect(l3_host, cmd) + if out: + LOG.debug("qos was found on external interface" + " %s of qrouter-%s on %s" + % (interface, rid, l3_host)) + else: + LOG.error("qos was not found on external interface" + " %s of qrouter-%s on %s" + % (interface, rid, l3_host)) + else: + LOG.error("failed to get external interface of" + " qrouter-%s on %s" % (rid, l3_host)) else: LOG.error("failed to get external gateway on %s" % (l3_host)) else: LOG.debug('this port is normal port, do not need to check gateway') - def vrouter_check_one(rid): cmd = 'neutron router-port-list %s -f csv -F id -F name' % (rid) out = run_command(cmd) From 81e7f8d7a662a1189bc1b4d58192829491f2aed5 Mon Sep 17 00:00:00 2001 From: blkart Date: Wed, 17 Aug 2016 12:21:50 +0800 Subject: [PATCH 05/12] [doctor][all] add vrouter check to doctor all command redmine 7669 Signed-off-by: blkart --- eayunstack_tools/doctor/all.py | 5 ++++- eayunstack_tools/doctor/net.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/eayunstack_tools/doctor/all.py b/eayunstack_tools/doctor/all.py index df7fcc4..2fc88d4 100644 --- a/eayunstack_tools/doctor/all.py +++ b/eayunstack_tools/doctor/all.py @@ -1,6 +1,6 @@ #check all object import logging -from eayunstack_tools.doctor import env, cls, stack +from eayunstack_tools.doctor import env, cls, stack, net from eayunstack_tools.logger import StackLOG as LOG @@ -11,6 +11,9 @@ def check_all(parser): cls.check_all() LOG.info('%s %-27s %s' % ('+'*13, 'Check OpenStack Environment', '+'*13)) stack.check_all() + LOG.info('%s %-27s %s' % ('+'*13, 'Check OpenStack Network', '+'*13)) + net.check_all() + def make(parser): '''Check All Object''' diff --git a/eayunstack_tools/doctor/net.py b/eayunstack_tools/doctor/net.py index 6c02fc7..fe00143 100644 --- a/eayunstack_tools/doctor/net.py +++ b/eayunstack_tools/doctor/net.py @@ -2,6 +2,9 @@ from eayunstack_tools.logger import fmt_excep_msg from eayunstack_tools.utils import NODE_ROLE from eayunstack_tools.sys_utils import ssh_connect, ssh_connect2 +from eayunstack_tools.utils import get_controllers_hostname +from eayunstack_tools.doctor.utils import run_doctor_cmd_on_node +import logging import commands import json import re @@ -230,6 +233,32 @@ def vrouter_check(parser): LOG.error('This check can be run only on network node') +def check_all(): + check_cmd = get_check_cmd() + if NODE_ROLE.is_fuel(): + controllers = get_controllers_hostname() + if controllers: + controller_node = controllers[0] + result = run_doctor_cmd_on_node('controller', controller_node, + check_cmd) + logging.disable(logging.NOTSET) + LOG.info(result, remote=True) + else: + LOG.error('Can not get controller node list') + elif NODE_ROLE.is_controller(): + print run_command(check_cmd) + + +def get_check_cmd(): + main_cmd = 'sudo eayunstack' + sub_cmd = 'doctor net vrouter' + if LOG.enable_debug: + check_cmd = main_cmd + ' --debug ' + sub_cmd + else: + check_cmd = main_cmd + sub_cmd + return check_cmd + + def make_vrouter(parser): '''Check Neutron virtual router''' parser.add_argument( From 9e5e7ae72be68ef51ce6c0eb0cf9cd611f629866 Mon Sep 17 00:00:00 2001 From: pengdake <19921207pq@gmail.com> Date: Thu, 18 Aug 2016 11:50:42 +0800 Subject: [PATCH 06/12] release 1.1-9.2 1.[tools][doctor][net] check external interface qos, fix redmin 7669 2.[doctor][all] add vrouter check to doctor all command Signed-off-by: pengdake <19921207pq@gmail.com> --- eayunstack-tools.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eayunstack-tools.spec b/eayunstack-tools.spec index 0af7b0f..6f1f125 100644 --- a/eayunstack-tools.spec +++ b/eayunstack-tools.spec @@ -1,6 +1,6 @@ Name: eayunstack-tools Version: 1.1 -Release: 9.1%{?dist} +Release: 9.2%{?dist} Summary: EayunStack Management tools Group: Application @@ -78,6 +78,9 @@ fi %changelog +* Wed Aug 17 2016 pq <19921207pq@gmail.com> 1.1-9.2 +- commit 81e7f8d7a662a1189bc1b4d58192829491f2aed5 + * Mon Jul 18 2016 blkart 1.1-9.1 - commit fae71216655f576667b82c5a984e7aab56c82e73 From 08374872518efbd83f544d89af888485d9cf7daf Mon Sep 17 00:00:00 2001 From: blkart Date: Mon, 22 Aug 2016 17:30:54 +0800 Subject: [PATCH 07/12] [bugfix][doctor][all] fix vrouter check failed bug redmine 7705 Signed-off-by: blkart --- eayunstack_tools/doctor/net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eayunstack_tools/doctor/net.py b/eayunstack_tools/doctor/net.py index fe00143..bc9deda 100644 --- a/eayunstack_tools/doctor/net.py +++ b/eayunstack_tools/doctor/net.py @@ -255,7 +255,7 @@ def get_check_cmd(): if LOG.enable_debug: check_cmd = main_cmd + ' --debug ' + sub_cmd else: - check_cmd = main_cmd + sub_cmd + check_cmd = main_cmd + ' ' + sub_cmd return check_cmd From 5415a2f0a16314d5c3d38e780332713e7996388c Mon Sep 17 00:00:00 2001 From: blkart Date: Mon, 22 Aug 2016 17:50:58 +0800 Subject: [PATCH 08/12] release 1.1-9.3 1.[bugfix][doctor][all] fix vrouter check failed bug Signed-off-by: blkart --- eayunstack-tools.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eayunstack-tools.spec b/eayunstack-tools.spec index 6f1f125..041d24c 100644 --- a/eayunstack-tools.spec +++ b/eayunstack-tools.spec @@ -1,6 +1,6 @@ Name: eayunstack-tools Version: 1.1 -Release: 9.2%{?dist} +Release: 9.3%{?dist} Summary: EayunStack Management tools Group: Application @@ -78,6 +78,9 @@ fi %changelog +* Mon Aug 22 2016 blkart 1.1-9.3 +- commit 08374872518efbd83f544d89af888485d9cf7daf + * Wed Aug 17 2016 pq <19921207pq@gmail.com> 1.1-9.2 - commit 81e7f8d7a662a1189bc1b4d58192829491f2aed5 From 00fe32d721979d063357da0b219476670d3ddfe0 Mon Sep 17 00:00:00 2001 From: blkart Date: Tue, 23 Aug 2016 23:48:30 +0800 Subject: [PATCH 09/12] [bugfix] replace neutron command with python SDK to fix encoding issue redmine 7707 Signed-off-by: blkart --- eayunstack_tools/doctor/net.py | 66 ++++++++------------------------ eayunstack_tools/pythonclient.py | 19 +++++++++ 2 files changed, 35 insertions(+), 50 deletions(-) diff --git a/eayunstack_tools/doctor/net.py b/eayunstack_tools/doctor/net.py index bc9deda..230578a 100644 --- a/eayunstack_tools/doctor/net.py +++ b/eayunstack_tools/doctor/net.py @@ -4,6 +4,7 @@ from eayunstack_tools.sys_utils import ssh_connect, ssh_connect2 from eayunstack_tools.utils import get_controllers_hostname from eayunstack_tools.doctor.utils import run_doctor_cmd_on_node +from eayunstack_tools.pythonclient import PythonClient import logging import commands import json @@ -22,39 +23,6 @@ def run_command(cmd): reval = out return reval - -# FIXME: some neutronclient does not support json output, hack it -def csv2dict(csv): - """Convert result format from csv to dict: -csv format: -"id","name","mac_address" -"596afd3e-b60a-41f5-97c3-39495979e6d8","","fa:16:3e:3a:ee:97" -"70cb55cd-d5cb-4c12-8ad2-8edf18c2fa94","","fa:16:3e:f7:e9:8c" - -dict format: -[{"id": "596afd3e", "name": "", "mac_address": "fa:16:3e:3a:ee:97"}, -{"id": "70cb55cd", "name": "", "mac_address": "fa:16:3e:f7:e9:8c"}] - """ - field = csv.split('\n')[0] - p = re.compile(r'"(.*)"') - column = [] - for i in field.split(','): - column.append(p.match(i).groups()[0]) - routers = [] - out = csv.split('\n')[1:] - for r in out: - router = {} - r = r.split(',') - index = 0 - for index in range(len(column)): - try: - router[column[index]] = p.match(r[index]).groups()[0] - except AttributeError: - router[column[index]] = r[index] - routers.append(router) - return routers - - def vrouter_get_gw_remote(l3_host, rid): cmd = "ip netns exec qrouter-%s ip route show | "\ "grep 'default' | awk '{print $3}'" % (rid) @@ -164,11 +132,10 @@ def port_log(device_owner, s): LOG.debug('this port is normal port, do not need to check gateway') def vrouter_check_one(rid): - cmd = 'neutron router-port-list %s -f csv -F id -F name' % (rid) - out = run_command(cmd) - ports = [] - if out: - ports = csv2dict(out) + logging.disable(logging.INFO) + pc = PythonClient() + ports = pc.neutron_router_port_list(rid) + logging.disable(logging.NOTSET) l3_host = vrouter_get_l3_host(rid) if l3_host: @@ -180,11 +147,12 @@ def vrouter_check_one(rid): def vrouter_get_l3_host(rid): - cmd = "neutron l3-agent-list-hosting-router -f csv %s" \ - % (rid) - out = run_command(cmd).strip('\r\n') - if out: - hosts = csv2dict(out) + logging.disable(logging.INFO) + pc = PythonClient() + hosts = pc.neutron_l3_agent_list_hosting_router(rid) + logging.disable(logging.NOTSET) + + if hosts: return hosts[0]['host'] else: LOG.error('can not get l3 host for router %s' % (rid)) @@ -198,13 +166,11 @@ def _vrouter_check(parser): vrouter_check_one(parser.rid) else: # 1) Get valid routers list - cmd = 'neutron router-list -f csv -F id -F name' - if parser.tid: - cmd += ' --tenant-id %s' % (parser.tid) - out = run_command(cmd) - routers = [] - if out: - routers = csv2dict(out) + tenant_id = parser.tid + logging.disable(logging.INFO) + pc = PythonClient() + routers = pc.neutron_router_list(tenant_id) + logging.disable(logging.NOTSET) # 2) Check every router one by one: .e.g. status, ip address ..., this # is done on neutron node which namespace belong to. diff --git a/eayunstack_tools/pythonclient.py b/eayunstack_tools/pythonclient.py index 5e77abc..a734573 100644 --- a/eayunstack_tools/pythonclient.py +++ b/eayunstack_tools/pythonclient.py @@ -62,6 +62,25 @@ def nova_volume(self, server_id, volume_id): def neutron_delete_port(self, port_id): self.neutronclient.delete_port(port_id) + def neutron_router_list(self, tenant_id=None): + if tenant_id: + router_list = \ + self.neutronclient.list_routers(tenant_id=tenant_id)['routers'] + else: + router_list = self.neutronclient.list_routers()['routers'] + return router_list + + def neutron_router_port_list(self, router_id): + router_port_list = \ + self.neutronclient.list_ports(device_id=router_id)['ports'] + return router_port_list + + def neutron_l3_agent_list_hosting_router(self, router_id): + l3_agent_list_hosting_router = \ + self.neutronclient.list_l3_agent_hosting_routers( + router_id)['agents'] + return l3_agent_list_hosting_router + def nova_flavor(self, flavor_id): flavor = self.novaclient.flavors.get(flavor_id) return flavor From b2136940cd0c27d7d7cf3719b6cf392e3557ba39 Mon Sep 17 00:00:00 2001 From: blkart Date: Wed, 24 Aug 2016 14:50:48 +0800 Subject: [PATCH 10/12] release 1.1-9.4 1.[bugfix] replace neutron command with python SDK to fix encoding issue Signed-off-by: blkart --- eayunstack-tools.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eayunstack-tools.spec b/eayunstack-tools.spec index 041d24c..16c69e3 100644 --- a/eayunstack-tools.spec +++ b/eayunstack-tools.spec @@ -1,6 +1,6 @@ Name: eayunstack-tools Version: 1.1 -Release: 9.3%{?dist} +Release: 9.4%{?dist} Summary: EayunStack Management tools Group: Application @@ -78,6 +78,9 @@ fi %changelog +* Wed Aug 24 2016 blkart 1.1-9.4 +- commit 00fe32d721979d063357da0b219476670d3ddfe0 + * Mon Aug 22 2016 blkart 1.1-9.3 - commit 08374872518efbd83f544d89af888485d9cf7daf From 9aecfc11f05aa3217334c9f8876ea3a9f3838639 Mon Sep 17 00:00:00 2001 From: blkart Date: Wed, 24 Aug 2016 21:37:38 +0800 Subject: [PATCH 11/12] [bugfix] the error message of port check is incomplete in fuel node redmine 7709 Signed-off-by: blkart --- eayunstack_tools/doctor/net.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eayunstack_tools/doctor/net.py b/eayunstack_tools/doctor/net.py index 230578a..a372100 100644 --- a/eayunstack_tools/doctor/net.py +++ b/eayunstack_tools/doctor/net.py @@ -87,10 +87,10 @@ def port_log(device_owner, s): # 1) check status of gateway port and interface port if detail['status'] != 'ACTIVE': - port_log(device_owner, "status of port %s[%s] on %s is down" + port_log(device_owner, "status of port %s(%s) on %s is down" % (device_owner, pid, l3_host)) if not detail['admin_state_up']: - port_log(device_owner, "admin_status of port %s[%s] on %s is down" + port_log(device_owner, "admin_status of port %s(%s) on %s is down" % (device_owner, pid, l3_host)) # 2) ping external gateway to check network status @@ -140,9 +140,9 @@ def vrouter_check_one(rid): l3_host = vrouter_get_l3_host(rid) if l3_host: for port in ports: - LOG.debug('start checking port %s[%s]' % (port['name'], port['id'])) + LOG.debug('start checking port %s(%s)' % (port['name'], port['id'])) port_check_one(port['id'], l3_host) - LOG.debug('finish checking port %s[%s]' % (port['name'], port['id'])) + LOG.debug('finish checking port %s(%s)' % (port['name'], port['id'])) # TODO: check dhcp? @@ -176,9 +176,9 @@ def _vrouter_check(parser): # is done on neutron node which namespace belong to. # tenant ID for router in routers: - LOG.info('start checking route %s[%s]' % (router['name'], router['id'])) + LOG.info('start checking route %s(%s)' % (router['name'], router['id'])) vrouter_check_one(router['id']) - LOG.info('finish checking route %s[%s]' % (router['name'], router['id'])) + LOG.info('finish checking route %s(%s)' % (router['name'], router['id'])) def vrouter_check(parser): From 4772bd913787b3b3e1463fc2c58ce6d2984df4cc Mon Sep 17 00:00:00 2001 From: blkart Date: Wed, 24 Aug 2016 22:03:12 +0800 Subject: [PATCH 12/12] release 1.1-9.5 1.[bugfix] the error message of port check is incomplete in fuel node Signed-off-by: blkart --- eayunstack-tools.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eayunstack-tools.spec b/eayunstack-tools.spec index 16c69e3..3429b80 100644 --- a/eayunstack-tools.spec +++ b/eayunstack-tools.spec @@ -1,6 +1,6 @@ Name: eayunstack-tools Version: 1.1 -Release: 9.4%{?dist} +Release: 9.5%{?dist} Summary: EayunStack Management tools Group: Application @@ -78,6 +78,9 @@ fi %changelog +* Wed Aug 24 2016 blkart 1.1-9.5 +- commit 9aecfc11f05aa3217334c9f8876ea3a9f3838639 + * Wed Aug 24 2016 blkart 1.1-9.4 - commit 00fe32d721979d063357da0b219476670d3ddfe0