diff --git a/.gitignore b/.gitignore index 2023fea..6d0b343 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.py[cod] +/.python-version /.tox __pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..092dcee --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,3 @@ +include: + - project: "repos/cloud/cicd/gitlab-ci" + file: "py3.11-bookworm-tox/gitlab-ci.yaml" diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..ca80d02 --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: gunicorn --workers 4 --bind 0.0.0.0:8000 --timeout 300 app:app + diff --git a/README.md b/README.md index 3d15b6b..05fc0f6 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,15 @@ -Keystone browser -================ +OpenStack browser +================= -Browse the Wikimedia Cloud Services OpenStack deployment. +Browse the Wikimedia Cloud VPS OpenStack deployment. -Deploy on Tool Labs +Deploy on Toolforge ------------------- ``` -$ ssh tools-dev.wmflabs.org -$ become $TOOL_NAME -$ mkdir -p $HOME/www/python -$ git clone https://phabricator.wikimedia.org/source/tool-keystone-browser.git \ - $HOME/www/python/src -$ webservice --backend=kubernetes python shell -$ python3 -m venv $HOME/www/python/venv -$ source $HOME/www/python/venv/bin/activate -$ pip install --upgrade pip -$ pip install --upgrade setuptools -$ pip install -r $HOME/www/python/src/requirements.txt -$ exit -$ webservice --backend=kubernetes python start +$ ssh dev.toolforge.org +$ become openstack-browser +$ toolforge build start https://gitlab.wikimedia.org/toolforge-repos/openstack-browser.git +$ toolforge webservice --backend=kubernetes buildservice start ``` License diff --git a/app.py b/app.py index a54a51e..9869d0f 100644 --- a/app.py +++ b/app.py @@ -19,7 +19,8 @@ # with this program. If not, see . import flask -import werkzeug.contrib.fixers +import requests +import werkzeug.middleware.proxy_fix from keystone_browser import zones from keystone_browser import glance @@ -29,250 +30,460 @@ from keystone_browser import puppetclasses from keystone_browser import proxies from keystone_browser import stats -from keystone_browser import utils +from keystone_browser import cinder +from keystone_browser import neutron +from keystone_browser import trove +from keystone_browser import octavia + +requests.utils.default_user_agent = lambda *args, **kwargs: ( + "openstack-browser (tools.openstack-browser@toolforge.org)" + + f" python-requests/{requests.__version__}" +) app = flask.Flask(__name__) -app.wsgi_app = werkzeug.contrib.fixers.ProxyFix(app.wsgi_app) +app.wsgi_app = werkzeug.middleware.proxy_fix.ProxyFix(app.wsgi_app) -@app.route('/') +@app.route("/") def home(): ctx = {} try: - cached = 'purge' not in flask.request.args - ctx.update({ - 'usage': stats.usage(cached), - }) + cached = "purge" not in flask.request.args + ctx.update( + { + "usage": stats.usage(cached), + } + ) except Exception: - app.logger.exception('Error collecting information for projects') - return flask.render_template('home.html', **ctx) + app.logger.exception("Error collecting information for projects") + return flask.render_template("home.html", **ctx) + + +@app.route("/robots.txt") +def robots_txt(): + return flask.Response( + "User-Agent: *\nDisallow: /\n", mimetype="text/plain" + ) -@app.route('/project/') +@app.route("/project/") def projects(): ctx = {} try: - ctx.update({ - 'projects': keystone.all_projects(), - }) + cached = "purge" not in flask.request.args + ctx.update( + { + "projects": keystone.all_projects(cached), + } + ) except Exception: - app.logger.exception('Error collecting information for projects') - return flask.render_template('projects.html', **ctx) + app.logger.exception("Error collecting information for projects") + return flask.render_template("projects.html", **ctx) -@app.route('/server/') +@app.route("/server/") def servers(): ctx = {} try: - ctx.update({ - 'servers': nova.all_servers(), - }) + cached = "purge" not in flask.request.args + ctx.update( + { + "servers": nova.all_servers(cached), + } + ) except Exception: - app.logger.exception('Error collecting information for projects') - return flask.render_template('servers.html', **ctx) + app.logger.exception("Error collecting information for projects") + return flask.render_template("servers.html", **ctx) + + +@app.route("/project/") +def project(project): + cached = "purge" not in flask.request.args + project_id, project_name = keystone.find_project(project) + if not project_id: + return flask.render_template("project_404.html", project=project) + if project != project_id: + return flask.redirect(flask.url_for("project", project=project_id)) -@app.route('/project/') -def project(name): - cached = 'purge' not in flask.request.args ctx = { - 'project': name, + "project_id": project_id, + "project_name": project_name, } try: - users = keystone.project_users_by_role(name) - admins = users['admin'] + users['projectadmin'] - ctx.update({ - 'project': name, - 'admins': ldap.get_users_by_uid(admins), - 'users': ldap.get_users_by_uid(users['user']), - 'servers': nova.project_servers(name), - 'flavors': nova.flavors(name), - 'images': glance.images(), - 'proxies': proxies.project_proxies(name, cached), - 'zones': zones.all_a_records(name, cached), - }) + project_data = keystone.project_data(project_id, cached) + users = keystone.project_users_by_role(project_id, cached) + # Create exclusive sets of users based on descending order of + # "power". + # member > service accounts > viewers + members = set(users["admin"]) | set(users["member"]) + service_accounts = { + role: set(uids) - members + for role, uids in users.items() + if role in keystone.SERVICE_ACCOUNT_ROLES and len(uids) > 0 + } + viewers = set(users["reader"]) - members + for uids in service_accounts.values(): + viewers = viewers - uids + + ctx.update( + { + "data": project_data, + "project_name": project_data["name"], + "members": ldap.get_users_by_uid(members, cached), + "viewers": ldap.get_users_by_uid(viewers, cached), + "service_accounts": { + role: ldap.get_users_by_uid(uids, cached) + for role, uids in service_accounts.items() + }, + "servers": nova.project_servers(project_id, cached), + "flavors": nova.flavors(project_id, cached), + "images": glance.images(cached), + "proxies": proxies.project_proxies(project_id, cached), + "zones": zones.all_dns_zones(project_id, cached), + "limits": nova.limits(project_id, cached), + "volumes": cinder.project_volumes(project_id, cached), + "cinder_limits": cinder.limits(project_id, cached), + "neutron_limits": neutron.limits(project_id, cached), + "databases": trove.project_instances(project_id, cached), + "floating_ips": neutron.floating_ips(project_id, cached), + "load_balancers": octavia.project_load_balancers( + project_id, cached + ), + } + ) except Exception: app.logger.exception( - 'Error collecting information for project "%s"', name) - return flask.render_template('project.html', **ctx) + 'Error collecting information for project "%s"', project_id + ) + return flask.render_template("project.html", **ctx) -@app.route('/user/') -def user(uid): +@app.route("/project//database/") +def project_database(project, name): + cached = "purge" not in flask.request.args + ctx = { + "project": project, + "project_name": keystone.project_name_for_id(project), + "name": name, + } + try: + instance = trove.instance(project, name, cached) + ctx.update( + { + "instance": instance, + "flavors": nova.flavors(project, cached), + } + ) + except Exception: + app.logger.exception( + 'Error collecting information for project "%s" database "%s"', + project, + name, + ) + return flask.render_template("databaseinstance.html", **ctx) + + +@app.route("/project//zone/") +def zone(project, name): + if not name.endswith("."): + return flask.redirect( + flask.url_for("zone", project=project, name=f"{name}.") + ) + + cached = "purge" not in flask.request.args ctx = { - 'uid': uid, + "project": project, + "project_name": keystone.project_name_for_id(project), + "name": name, } try: - ctx.update({ - 'user': ldap.get_users_by_uid([uid]), - 'projects': keystone.projects_for_user(uid), - }) - if ctx['user']: - ctx['user'] = ctx['user'][0] + zone = zones.zone(project, name, cached) + + ctx.update( + { + "zone": zone, + "records": zones.records(project, name, zone["id"], cached), + } + ) except Exception: app.logger.exception( - 'Error collecting information for user "%s"', uid) - return flask.render_template('user.html', **ctx) + 'Error collecting information for project "%s" zone "%s"', + project, + zone, + ) + return flask.render_template("zone.html", **ctx) + + +@app.route("/user/") +def user(uid): + ctx = { + "uid": uid, + } + try: + cached = "purge" not in flask.request.args + roles = keystone.roles_for_user(uid, cached) + ctx.update( + { + "user": ldap.get_users_by_uid([uid], cached), + "projects": roles["projects"], + "domain_roles": roles["domain_roles"], + } + ) + if ctx["user"]: + ctx["user"] = ctx["user"][0] + except Exception: + app.logger.exception('Error collecting information for user "%s"', uid) + return flask.render_template("user.html", **ctx) -@app.route('/server/') +@app.route("/server/") def server(fqdn): - name, project, tld = fqdn.split('.', 2) + name, project_key, tld = fqdn.split(".", 2) + project_id, project_name = keystone.find_project(project_key) + if project_key != project_name: + return flask.redirect( + flask.url_for("server", fqdn=f"{name}.{project_name}.{tld}") + ) + ctx = { - 'fqdn': fqdn, - 'project': project, + "fqdn": fqdn, + "project": project_name, + "project_id": project_id, } try: - ctx.update({ - 'server': nova.server(fqdn), - 'flavors': nova.flavors(project), - 'images': glance.images(), - 'puppetclasses': puppetclasses.classes(project, fqdn), - 'hiera': puppetclasses.hiera(project, fqdn), - }) - if 'user_id' in ctx['server']: - user = ldap.get_users_by_uid([ctx['server']['user_id']]) + cached = "purge" not in flask.request.args + ctx.update( + { + "server": nova.server(fqdn, cached), + "flavors": nova.flavors(project_id, cached), + "images": glance.images(cached), + "puppetclasses": puppetclasses.classes( + project_id, fqdn, cached + ), + "hiera": puppetclasses.hiera(project_id, fqdn, cached), + } + ) + if "user_id" in ctx["server"]: + user = ldap.get_users_by_uid([ctx["server"]["user_id"]], cached) if user: - ctx['owner'] = user[0] + ctx["owner"] = user[0] except Exception: app.logger.exception( - 'Error collecting information for server "%s"', fqdn) + 'Error collecting information for server "%s"', fqdn + ) - return flask.render_template('server.html', **ctx) + return flask.render_template("server.html", **ctx) -@app.route('/puppetclass/') +@app.route("/load-balancer/") +def load_balancer(lb_id): + cached = "purge" not in flask.request.args + ctx = { + "id": lb_id, + "project": None, + "project_name": None, + } + try: + lb = octavia.load_balancer(lb_id, cached) + + ctx.update( + { + "lb": lb, + "project": lb["project_id"], + "project_name": keystone.project_name_for_id(lb["project_id"]), + } + ) + except Exception: + app.logger.exception( + 'Error collecting information for load balancer "%s"', + lb_id, + ) + return flask.render_template("loadbalancer.html", **ctx) + + +@app.route("/puppetclass/") def all_puppetclasses(): ctx = {} try: - ctx.update({"puppetclasses": puppetclasses.all_classes()}) + cached = "purge" not in flask.request.args + ctx.update({"puppetclasses": puppetclasses.all_classes(cached)}) except Exception: - app.logger.exception( - 'Error collecting the list of puppet classes') + app.logger.exception("Error collecting the list of puppet classes") - return flask.render_template('puppetclasses.html', **ctx) + return flask.render_template("puppetclasses.html", **ctx) -@app.route('/puppetclass/') +@app.route("/puppetclass/") def puppetclass(classname): ctx = { - 'puppetclass': classname, + "puppetclass": classname, } try: - ctx.update({"data": puppetclasses.prefixes(classname)}) + cached = "purge" not in flask.request.args + ctx.update({"data": puppetclasses.prefixes(classname, cached)}) except Exception: app.logger.exception( - 'Error collecting information for puppet class "%s"', classname) + 'Error collecting information for puppet class "%s"', classname + ) - return flask.render_template('puppetclass.html', **ctx) + return flask.render_template("puppetclass.html", **ctx) -@app.route('/hierakey/') +@app.route("/hierakey/") def hierakey(hierakey): ctx = { - 'hierakey': hierakey, + "hierakey": hierakey, } try: - ctx.update({"data": puppetclasses.hieraprefixes(hierakey)}) + cached = "purge" not in flask.request.args + ctx.update({"data": puppetclasses.hieraprefixes(hierakey, cached)}) except Exception: app.logger.exception( - 'Error collecting information for hiera key "%s"', hierakey) + 'Error collecting information for hiera key "%s"', hierakey + ) - return flask.render_template('hierakey.html', **ctx) + return flask.render_template("hierakey.html", **ctx) -@app.route('/proxy/') +@app.route("/proxy/") def all_proxies(): - cached = 'purge' not in flask.request.args + cached = "purge" not in flask.request.args ctx = { - 'proxies': proxies.all_proxies(cached), + "proxies": proxies.all_proxies(cached), } - return flask.render_template('proxies.html', **ctx) + return flask.render_template("proxies.html", **ctx) -@app.route('/api/projects.json') +@app.route("/network/") +def networks(): + cached = "purge" not in flask.request.args + ctx = { + "networks": neutron.networks(cached), + } + return flask.render_template("networks.html", **ctx) + + +@app.route("/flavors/") +def flavors(): + cached = "purge" not in flask.request.args + ctx = { + "flavors": nova.flavors("observer", cached).values(), + } + return flask.render_template("flavors.html", **ctx) + + +@app.route("/api/projects.json") def api_projects_json(): - return flask.jsonify(projects=keystone.all_projects()) + cached = "purge" not in flask.request.args + return flask.jsonify( + projects=list(sorted(keystone.all_projects(cached).keys())) + ) -@app.route('/api/projects.txt') +@app.route("/api/projects.txt") def api_projects_txt(): + cached = "purge" not in flask.request.args return flask.Response( - '\n'.join(sorted(keystone.all_projects())), - mimetype='text/plain') + "\n".join(sorted(keystone.all_projects(cached).keys())), + mimetype="text/plain", + ) + + +@app.route("/api/project-names.json") +def api_project_names_json(): + cached = "purge" not in flask.request.args + return flask.jsonify(projects=keystone.all_projects(cached)) -@app.route('/api/dsh/project/') +@app.route("/api/dsh/project/") def api_dsh_project(name): - servers = nova.project_servers(name) + cached = "purge" not in flask.request.args + servers = nova.project_servers(name, cached) dsh = [ - "{}.{}.eqiad.wmflabs".format(server['name'], name) + "{}.{}.eqiad1.wikimedia.cloud".format(server["name"], name) for server in servers ] - return flask.Response('\n'.join(sorted(dsh)), mimetype='text/plain') + return flask.Response("\n".join(sorted(dsh)), mimetype="text/plain") -@app.route('/api/dsh/servers') +@app.route("/api/dsh/servers") def api_dsh_servers(): - servers = nova.all_servers() + cached = "purge" not in flask.request.args + servers = nova.all_servers(cached) dsh = [ - "{}.{}.eqiad.wmflabs".format(server['name'], server['tenant_id']) + "{}.{}.eqiad1.wikimedia.cloud".format( + server["name"], server["tenant_id"] + ) for server in servers ] - return flask.Response('\n'.join(sorted(dsh)), mimetype='text/plain') + return flask.Response("\n".join(sorted(dsh)), mimetype="text/plain") -@app.route('/api/dsh/puppetclass/') +@app.route("/api/dsh/puppetclass/") def api_dsh_puppet(name): - data = puppetclasses.prefixes(name) + cached = "purge" not in flask.request.args + data = puppetclasses.prefixes(name, cached) dsh = [] for project, d in data.items(): - if project == 'admin': + if project == "admin": continue try: - servers = nova.project_servers(project) + cached = "purge" not in flask.request.args + servers = nova.project_servers(project, cached) except Exception: app.logger.exception( - 'Error collecting the list of servers for %s', project) + "Error collecting the list of servers for %s", project + ) servers = [] - for prefix in d['prefixes']: - if prefix.endswith('wmflabs'): + for prefix in d["prefixes"]: + if prefix.endswith(".cloud"): dsh.append(prefix) else: - dsh.extend([ - "{}.{}.eqiad.wmflabs".format(server['name'], project) - for server in servers - if server['name'].startswith(prefix) - ]) + dsh.extend( + [ + "{}.{}.eqiad1.wikimedia.cloud".format( + server["name"], project + ) + for server in servers + if server["name"].startswith(prefix) + ] + ) - return flask.Response('\n'.join(sorted(set(dsh))), mimetype='text/plain') + return flask.Response("\n".join(sorted(set(dsh))), mimetype="text/plain") -@app.route('/api/hierakey/') +@app.route("/api/hierakey/") def api_hierakey(hierakey): - return flask.jsonify(servers=puppetclasses.hieraprefixes(hierakey)) + cached = "purge" not in flask.request.args + return flask.jsonify(servers=puppetclasses.hieraprefixes(hierakey, cached)) @app.errorhandler(404) def page_not_found(e): - return flask.redirect(flask.url_for('projects')) + return flask.redirect(flask.url_for("projects")) -@app.template_filter('contains') +@app.template_filter("contains") def contains(haystack, needle): return needle in haystack -@app.template_filter('extract_hostname') +@app.template_filter("extract_hostname") def extract_hostname(backend): """Extract a hostname from a backend description.""" - return proxies.parse_backend(backend).get('hostname', '404') - - -@app.template_test() -def ipv4addr(s): - """Is the given string an IPv4 address?""" - return utils.is_ipv4(s) + hostname = proxies.parse_backend(backend) + if not hostname: + return None + if not hostname.endswith(".wikimedia.cloud"): + return None + if ".svc." in hostname: + return None + return hostname + + +if __name__ == "__main__": + app.run(port=3000, debug=True) diff --git a/keystone_browser/cache.py b/keystone_browser/cache.py index a51576d..0a6b175 100644 --- a/keystone_browser/cache.py +++ b/keystone_browser/cache.py @@ -20,31 +20,30 @@ import hashlib import json import os -import pwd import redis class Cache(object): """Simple redis wrapper.""" - def __init__(self, enabled=True, seed=''): + + def __init__(self, enabled=True, seed=""): self.enabled = enabled - self.conn = redis.Redis( - host='tools-redis', + self.conn = redis.StrictRedis( + host="redis.svc.tools.eqiad1.wikimedia.cloud", decode_responses=True, ) - u = pwd.getpwuid(os.getuid()) self.prefix = hashlib.sha1( - '{}{}.{}'.format(seed, u.pw_name, u.pw_dir).encode('utf-8') + "{}.{}".format(seed, os.getuid()).encode("utf-8") ).hexdigest() def key(self, val): - return '{}:{}'.format(self.prefix, val) + return "{}:{}".format(self.prefix, val) def load(self, key): if self.enabled: try: - return json.loads(self.conn.get(self.key(key)) or '') + return json.loads(self.conn.get(self.key(key)) or "") except ValueError: return None else: @@ -53,7 +52,7 @@ def load(self, key): def save(self, key, data, expiry=300): if self.enabled: real_key = self.key(key) - self.conn.setex(real_key, json.dumps(data), expiry) + self.conn.setex(real_key, expiry, json.dumps(data)) -CACHE = Cache(seed='201804132043') +CACHE = Cache(seed="201804132043") diff --git a/keystone_browser/cinder.py b/keystone_browser/cinder.py new file mode 100644 index 0000000..73393fa --- /dev/null +++ b/keystone_browser/cinder.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of the Keystone browser +# +# Copyright (c) 2021 Taavi Väänänen +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import functools + +from cinderclient import client + +from . import cache +from . import keystone + + +@functools.lru_cache(maxsize=None) +def cinder_client(project, region): + return client.Client( + version="3", + session=keystone.session(project), + timeout=2, + region_name=region, + ) + + +@functools.lru_cache() +def get_regions(): + ks_client = keystone.keystone_client() + region_recs = ks_client.regions.list() + return [region.id for region in region_recs] + + +def project_volumes(project, cached=True): + key = "cinder:project-volumes:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = [] + for region in get_regions(): + cinder = cinder_client(project, region) + data.extend( + [ + volume._info + for volume in cinder.volumes.list( + detailed=True, + ) + ] + ) + cache.CACHE.save(key, data, 300) + return data + + +def limits(project, cached=True): + """Get a dict of limit details.""" + key = "cinder:limits:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = {} + for region in get_regions(): + cinder = cinder_client(project, region) + data[region] = cinder.quotas.get(project, usage=True).to_dict() + + cache.CACHE.save(key, data, 3600) + return data diff --git a/keystone_browser/glance.py b/keystone_browser/glance.py index cc47312..20bf443 100644 --- a/keystone_browser/glance.py +++ b/keystone_browser/glance.py @@ -29,20 +29,27 @@ @functools.lru_cache(maxsize=1) def glance_client(): return glanceclient.Client( - version='2', + version="2", session=keystone.session(), - interface='public', + interface="public", ) -def images(): +def images(cached=True): """Get a dict of image details indexed by id.""" - key = 'glance:images' - data = cache.CACHE.load(key) + # Images not appearing in this dict? Make sure that the 'observer' project + # can see them: + # for img in $(openstack image list --private -f value|awk '{print $1}') + # do + # glance member-create $img observer; + # glance member-update $img observer accepted; + # done + key = "glance:images" + data = None + if cached: + data = cache.CACHE.load(key) if data is None: glance = glance_client() - data = { - i['id']: i for i in glance.images.list() - } + data = {i["id"]: i for i in glance.images.list()} cache.CACHE.save(key, data, 3600) return data diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 3b81409..d5f4be1 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -20,90 +20,169 @@ import collections import functools +import yaml from keystoneauth1 import session as keystone_session from keystoneauth1.identity import v3 +from keystoneauth1 import exceptions from keystoneclient.v3 import client from . import cache +ROLES = collections.OrderedDict( + [ + # Admins + ("admin", "2cd63d467f754404bf3746fe63ee0698"), + ("member", "38676f30eaeb44518bf7e144a73c8da6"), + # Limited admin + ("designateadmin", "906f1588626d4d0993629ea3928b6fb4"), + ("glanceadmin", "1102f4ff63c3435793d0e4340bf4b04e"), + ("keystonevalidate", "f3bebf5f4b6f40fa91f3614431f2c283"), + ("object_storage", "db4c840c39d44b059ccb110851226c8d"), + # Members + ("reader", "f75a3c410bca4e96a1cf6ac103b0ccaf"), + ] +) -ROLES = collections.OrderedDict([ - ('admin', '2cd63d467f754404bf3746fe63ee0698'), - ('glanceadmin', '1102f4ff63c3435793d0e4340bf4b04e'), - ('observer', '47a8370618ea42d49f7047774e75d262'), - ('projectadmin', '4d8cad783d6342efa8414d7d36fbc034'), - ('user', 'f473273fac7146b3bdbf22e5d4504f95'), -]) + +SERVICE_ACCOUNT_ROLES = [ + "glanceadmin", + "designateadmin", + "keystonevalidate", + "object_storage", +] @functools.lru_cache(maxsize=None) -def session(project='observer'): +def session(project="observer"): """Get a session for the novaobserver user scoped to the given project.""" - # TODO: read settings from /etc/novaobserver.yaml once we get it mounted - # into the kubernetes pods () + with open("/etc/novaobserver.yaml", "r") as f: + observer_data = yaml.safe_load(f.read()) auth = v3.Password( - auth_url='http://cloudcontrol1003.wikimedia.org:5000/v3', - password='Fs6Dq2RtG8KwmM2Z', - username='novaobserver', + auth_url=observer_data["OS_AUTH_URL"], + password=observer_data["OS_PASSWORD"], + username=observer_data["OS_USERNAME"], project_id=project, - user_domain_name='Default', - project_domain_name='Default', + user_domain_name=observer_data["OS_USER_DOMAIN_ID"], + project_domain_name=observer_data["OS_PROJECT_DOMAIN_ID"], + ) + return keystone_session.Session( + auth=auth, + user_agent="openstack-browser", ) - return keystone_session.Session(auth=auth) def keystone_client(): return client.Client( session=session(), - interface='public', + interface="public", timeout=2, ) -def all_projects(): +def all_projects(cached=True): """Get a list of all project names.""" - key = 'keystone:all_projects' - data = cache.CACHE.load(key) + key = "keystone:projects_by_id" + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + keystone = keystone_client() + data = { + p.id: p.name + for p in keystone.projects.list(enabled=True, domain="default") + } + cache.CACHE.save(key, data, 300) + return data + + +def project_data(project_id, cached=True): + key = "keystone:project:{}".format(project_id) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: keystone = keystone_client() - # Ignore the magic 'admin' project - data = [ - p.name - for p in keystone.projects.list(enabled=True) - if p.name != 'admin' - ] + data = False + try: + project = keystone.projects.get(project_id) + data = { + "id": project.id, + "name": project.name, + "description": project.description, + } + except exceptions.http.NotFound: + pass cache.CACHE.save(key, data, 300) return data -def project_users_by_role(name): +def project_name_for_id(id, cached=True): + data = project_data(id, cached=cached) + if not data: + return None + return data["name"] + + +def find_project(search, cached=True): + for project_id, project_name in all_projects(cached=cached).items(): + if search == project_id or search == project_name: + return project_id, project_name + return None, None + + +def project_users_by_role(name, cached=True): """Get a dict of lists of user ids indexed by role name.""" - key = 'keystone:project_users_by_role:{}'.format(name) - data = cache.CACHE.load(key) + key = "keystone:project_users_by_role:{}".format(name) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: keystone = keystone_client() # Ignore novaadmin & novaobserver in all user lists - seen = ['novaadmin', 'novaobserver'] + ignored_users = ["novaadmin", "novaobserver"] data = {} for role_name, role_id in ROLES.items(): data[role_name] = [ - r.user['id'] for r in keystone.role_assignments.list( - project=name, role=role_id) - if r.user['id'] not in seen + r.user["id"] + for r in keystone.role_assignments.list( + project=name, role=role_id + ) + if r.user["id"] not in ignored_users ] - seen += data[role_name] cache.CACHE.save(key, data, 300) return data -def projects_for_user(uid): +def roles_for_user(uid, cached=True): """Get a list of projects that a user belongs to.""" - key = 'keystone:projects_for_user:{}'.format(uid) - data = cache.CACHE.load(key) + key = "keystone:roles_for_user:{}:v2".format(uid) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: keystone = keystone_client() - data = [ - p.name for p in keystone.projects.list(enabled=True, user=uid)] + projects = set() + domain_roles = set() + + for assignment in keystone.role_assignments.list(user=uid): + if "project" in assignment.scope: + projects.add( + ( + assignment.scope["project"]["id"], + project_name_for_id( + assignment.scope["project"]["id"], cached=cached + ), + ) + ) + elif "domain" in assignment.scope: + role_name = keystone.roles.get(assignment.role["id"]).name + domain_roles.add(role_name) + + data = { + "projects": sorted(list(projects), key=lambda project: project[1]), + "domain_roles": sorted(list(domain_roles)), + } + cache.CACHE.save(key, data, 300) return data diff --git a/keystone_browser/ldap.py b/keystone_browser/ldap.py index 99a4ae2..b31c4ce 100644 --- a/keystone_browser/ldap.py +++ b/keystone_browser/ldap.py @@ -30,12 +30,16 @@ def ldap_conn(): Return value can be used as a context manager """ - servers = ldap3.ServerPool([ - ldap3.Server('ldap-labs.eqiad.wikimedia.org'), - ldap3.Server('ldap-labs.codfw.wikimedia.org'), - ], ldap3.ROUND_ROBIN, active=True, exhaust=True) - return ldap3.Connection( - servers, read_only=True, auto_bind=True) + servers = ldap3.ServerPool( + [ + ldap3.Server("ldap-ro.eqiad.wikimedia.org", use_ssl=True), + ldap3.Server("ldap-ro.codfw.wikimedia.org", use_ssl=True), + ], + ldap3.ROUND_ROBIN, + active=True, + exhaust=True, + ) + return ldap3.Connection(servers, read_only=True, auto_bind=True) def in_list(attr, items): @@ -47,52 +51,57 @@ def in_list(attr, items): >>> in_list('uid', ['a', 'b', 'c']) '(|(uid=a)(uid=b)(uid=c))' """ - return '(|{})'.format(''.join( - ['({}={})'.format(attr, item) for item in items] - )) + return "(|{})".format( + "".join(["({}={})".format(attr, item) for item in items]) + ) -def get_users_by_uid(uids): +def get_users_by_uid(uids, cached=True): """Get a list of dicts of user information.""" if not uids: return [] - key = 'ldap:get_users_by_uid:{}'.format( - hashlib.sha1('|'.join(uids).encode('utf-8')).hexdigest()) - data = cache.CACHE.load(key) + key = "ldap:get_users_by_uid:{}".format( + hashlib.sha1("|".join(uids).encode("utf-8")).hexdigest() + ) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = [] with ldap_conn() as conn: results = conn.extend.standard.paged_search( - 'ou=people,dc=wikimedia,dc=org', - in_list('uid', uids), + "ou=people,dc=wikimedia,dc=org", + in_list("uid", uids), ldap3.SUBTREE, - attributes=['uid', 'cn'], + attributes=["uid", "cn"], paged_size=1000, time_limit=5, generator=True, ) for resp in results: - attribs = resp.get('attributes') + attribs = resp.get("attributes") # LDAP attributes come back as a dict of lists. We know that # there is only one value for each list, so unwrap it - data.append({ - 'uid': attribs['uid'][0], - 'cn': attribs['cn'][0], - }) + data.append( + { + "uid": attribs["uid"][0], + "cn": attribs["cn"][0], + } + ) cache.CACHE.save(key, data, 3600) return data def user_count(): """Get the count of all users in LDAP.""" - key = 'ldap:user_count' + key = "ldap:user_count" total_entries = cache.CACHE.load(key) if total_entries is None: total_entries = 0 with ldap_conn() as conn: results = conn.extend.standard.paged_search( - 'ou=people,dc=wikimedia,dc=org', - '(objectclass=posixaccount)', + "ou=people,dc=wikimedia,dc=org", + "(objectclass=posixaccount)", ldap3.SUBTREE, attributes=None, paged_size=1000, diff --git a/keystone_browser/neutron.py b/keystone_browser/neutron.py new file mode 100644 index 0000000..f5aeb66 --- /dev/null +++ b/keystone_browser/neutron.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of the Keystone browser +# +# Copyright (c) 2021 Taavi Väänänen +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import functools + +from neutronclient.v2_0 import client +from wmflib import dns + +from . import cache +from . import keystone +from . import utils + + +@functools.lru_cache(maxsize=None) +def neutron_client(project, region): + return client.Client( + session=keystone.session(project), + timeout=2, + region_name=region, + ) + + +@functools.lru_cache(maxsize=None) +def resolver() -> dns.Dns: + return dns.Dns() + + +@functools.lru_cache() +def get_regions(): + ks_client = keystone.keystone_client() + region_recs = ks_client.regions.list() + return [region.id for region in region_recs] + + +def limits(project, cached=True): + """Get a dict of limit details.""" + key = "neutron:limits:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = {} + for region in get_regions(): + neutron = neutron_client(project, region) + data[region] = neutron.show_quota_details(project) + cache.CACHE.save(key, data, 3600) + return data + + +def _map_ip_data(ip: dict): + data = { + "address": ip["floating_ip_address"], + "sortkey": utils.natural_sort_key(ip["floating_ip_address"]), + "description": ip["description"], + "target": ip.get("fixed_ip_address"), + "dns": [], + "target_dns": [], + } + + if data["address"]: + try: + data["dns"] = resolver().resolve_ptr(data["address"]) + except dns.DnsNotFound: + pass + + if data["target"]: + try: + data["target_dns"] = resolver().resolve_ptr(data["target"]) + except dns.DnsNotFound: + pass + + return data + + +def floating_ips(project: str, cached=True): + """Get a list of floating ips allocated to a project.""" + key = "neutron:floating_ips:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = [] + for region in get_regions(): + neutronclient = neutron_client(project, region) + data.extend( + [ + _map_ip_data(ip) + for ip in neutronclient.list_floatingips( + project_id=project + )["floatingips"] + ] + ) + + cache.CACHE.save(key, data, 3600) + + return data + + +def networks(cached=True): + key = "neutron:networks" + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = {} + for region in get_regions(): + neutronclient = neutron_client("admin", region) + subnets = { + subnet["id"]: subnet + for subnet in neutronclient.list_subnets()["subnets"] + } + for network in neutronclient.list_networks()["networks"]: + network["subnets"] = [ + subnets[subnet] for subnet in network["subnets"] + ] + data[network["id"]] = network + cache.CACHE.save(key, data, 3600) + return data diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index f2ffb0a..b84e910 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -24,92 +24,131 @@ from . import cache from . import keystone +from . import utils @functools.lru_cache(maxsize=None) def nova_client(project, region): return client.Client( - '2.12', + "2.12", session=keystone.session(project), - endpoint_type='public', + endpoint_type="public", timeout=2, - region_name=region + region_name=region, ) @functools.lru_cache() def get_regions(): - nova = client.Client('2.12', session=keystone.session, endpoint_type='public') - region_recs = nova.regions.list() + ks_client = keystone.keystone_client() + region_recs = ks_client.regions.list() return [region.id for region in region_recs] -def project_servers(project): +def project_servers(project, cached=True): """Get a list of information about servers in the given project. Data returned for each server is described at https://developer.openstack.org/api-ref/compute/?expanded=list-servers-detailed-detail#listServers """ - key = 'nova:project_servers:{}'.format(project) - data = cache.CACHE.load(key) + key = "nova:project_servers:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = [] for region in get_regions(): nova = nova_client(project, region) - data.extend([ - s._info for s in nova.servers.list( - detailed=True, - sort_keys=['display_name'], - sort_dirs=['asc'], - ) - ]) + data.extend( + [ + s._info + for s in nova.servers.list( + detailed=True, + sort_keys=["display_name"], + sort_dirs=["asc"], + ) + ] + ) + + data = list( + sorted( + data, key=lambda server: utils.natural_sort_key(server["name"]) + ) + ) + + project_name = keystone.project_name_for_id(project) + for server in data: + server["project_name"] = project_name cache.CACHE.save(key, data, 300) return data -def flavors(project): +def flavors(project, cached=True): """Get a dict of flavor details indexed by id.""" - key = 'nova:flavors:{}'.format(project) - data = cache.CACHE.load(key) + key = "nova:flavors:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = {} for region in get_regions(): nova = nova_client(project, region) for f in nova.flavors.list(): - data[f._info['id']] = f._info + data[f._info["id"]] = f._info + + cache.CACHE.save(key, data, 3600) + return data + + +def limits(project, cached=True): + """Get a dict of limit details.""" + key = "nova:limits:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = {} + for region in get_regions(): + nova = nova_client(project, region) + data[region] = nova.limits.get().to_dict() cache.CACHE.save(key, data, 3600) return data -def all_servers(): +def all_servers(cached=True): """Get a list of all servers in all projects.""" - key = 'keystone:all_servers' - data = cache.CACHE.load(key) + key = "keystone:all_servers" + data = None + if cached: + data = cache.CACHE.load(key) if data is None: data = [] all_projects = keystone.all_projects() for project in all_projects: - if project != 'admin': + if project != "admin": data += project_servers(project) cache.CACHE.save(key, data, 300) return data -def server(fqdn): +def server(fqdn, cached=True): """Get information about a server by fqdn.""" - key = 'nova:server:{}'.format(fqdn) - data = cache.CACHE.load(key) + key = "nova:server:{}".format(fqdn) + data = None + if cached: + data = cache.CACHE.load(key) if data is None: - name, project, _ = fqdn.split('.', 2) + name, project_key, _ = fqdn.split(".", 2) + project_id, _ = keystone.find_project(project_key) servers = [] for region in get_regions(): - nova = nova_client(project, region) + nova = nova_client(project_id, region) reg_servers = nova.servers.list( detailed=True, search_opts={ - 'name': '^{}$'.format(name), + "name": "^{}$".format(name), }, ) if reg_servers: diff --git a/keystone_browser/octavia.py b/keystone_browser/octavia.py new file mode 100644 index 0000000..7891220 --- /dev/null +++ b/keystone_browser/octavia.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of OpenStack browser +# +# Copyright (c) 2025 Taavi Väänänen for the Wikimedia Foundation +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +from . import cache +from . import os + + +def project_load_balancers(project_id, cached=True): + """Get a list of all database instances in a given project.""" + key = "octavia:project-lbs:{}".format(project_id) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = [] + for lb in os.session().load_balancer.load_balancers( + project_id=project_id + ): + data.append( + { + "id": lb.id, + "name": lb.name, + "provider": lb.provider, + "provisioning_status": lb.provisioning_status, + "operating_status": lb.operating_status, + "flavor_id": lb.flavor_id, + # TODO: what is the additional_vips property? + "vips": [lb.vip_address], + } + ) + cache.CACHE.save(key, data, 300) + return data + + +def load_balancer(lb_id, cached=True): + """Get a specific load balancer.""" + key = "octavia:lb:{}".format(lb_id) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + client = os.session().load_balancer + lb = client.find_load_balancer(lb_id) + if not lb: + return None + + listeners = [ + client.find_listener(listener["id"]) for listener in lb.listeners + ] + pools = [client.find_pool(pool["id"]) for pool in lb.pools] + + data = { + "id": lb.id, + "project_id": lb.project_id, + "name": lb.name, + "provider": lb.provider, + "provisioning_status": lb.provisioning_status, + "operating_status": lb.operating_status, + "flavor_id": lb.flavor_id, + "listeners": [ + { + "id": listener.id, + "protocol": listener.protocol, + "port": listener.protocol_port, + "provisioning_status": listener.provisioning_status, + "operating_status": listener.operating_status, + } + for listener in listeners + ], + "pools": [ + { + "id": pool.id, + "protocol": pool.protocol, + "provisioning_status": pool.provisioning_status, + "operating_status": pool.operating_status, + "members": [ + { + "id": member.id, + "address": member.address, + "port": member.protocol_port, + "provisioning_status": member.provisioning_status, + "operating_status": member.operating_status, + } + for member in client.members(pool) + ], + } + for pool in pools + ], + # TODO: what is the additional_vips property? + "vips": [lb.vip_address], + } + + cache.CACHE.save(key, data, 300) + return data diff --git a/keystone_browser/os.py b/keystone_browser/os.py new file mode 100644 index 0000000..bfef036 --- /dev/null +++ b/keystone_browser/os.py @@ -0,0 +1,27 @@ +# This file is part of OpenStack browser +# +# Copyright (c) 2025 Taavi Väänänen for the Wikimedia Foundation +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +import functools + +import openstack + + +@functools.lru_cache() +def session(): + """Get a session for the novaobserver user.""" + return openstack.connect(cloud="novaobserver") diff --git a/keystone_browser/proxies.py b/keystone_browser/proxies.py index 618a053..9c4db69 100644 --- a/keystone_browser/proxies.py +++ b/keystone_browser/proxies.py @@ -19,50 +19,47 @@ # with this program. If not, see . import functools -import re import socket - -import requests +from urllib.parse import urlparse from . import cache from . import keystone from . import utils -RE_BACKEND = re.compile(r'^https?://(?P[^:]+):(?P\d+)$') - - @functools.lru_cache(maxsize=1) def url_template(): """Get the url template for accessing the proxy service.""" c = keystone.keystone_client() - proxy = c.services.list(type='proxy')[0] - endpoint = c.endpoints.list( - service=proxy.id, interface='public', enabled=True)[0] - # Secret magic! The endpoint provided by keystone is private and we can't - # access it. There's an alternative public read-only endpoint on port 5669 - # though. So, swap in 5669 for the port we got from keystone. - return re.sub(r':[0-9]+/', ':5669/', endpoint.url) + proxy = c.services.list(type="proxy")[0] + endpoint = c.endpoints.list(service=proxy.id, interface="public")[0] + return endpoint.url + + +@functools.lru_cache(maxsize=None) +def proxy_client(project): + proxy_url = url_template().replace("$(tenant_id)s", project) + session = keystone.session(project) + return proxy_url, session def project_proxies(project, cached=True): """Get a list of proxies for a project.""" - key = 'proxies:{}'.format(project) + key = "proxies:{}".format(project) data = None if cached: data = cache.CACHE.load(key) if data is None: - base_url = url_template().replace('$(tenant_id)s', project) - url = '{}/mapping'.format(base_url) - req = requests.get(url, verify=False) + proxy_url, session = proxy_client(project) + req = session.get(f"{proxy_url}/mapping", raise_exc=False) if req.status_code != 200: data = [] else: - data = req.json()['routes'] + data = req.json()["routes"] # Some of the domain names have a . appended at the end, # strip them out so the URLs look right for route in data: - route['domain'] = route['domain'].rstrip('.') + route["domain"] = route["domain"].rstrip(".") cache.CACHE.save(key, data, 3600) return data @@ -73,15 +70,17 @@ def all_proxies(cached=True): Each proxy in the list will be a dict containing project, domain, and backends keys. """ - key = 'proxies:all' + key = "proxies:all" data = None if cached: data = cache.CACHE.load(key) if data is None: data = [ - dict(project=project, **proxy) - for project in keystone.all_projects() - for proxy in project_proxies(project, cached) + dict(project=project_id, project_name=project_name, **proxy) + for project_id, project_name in keystone.all_projects( + cached + ).items() + for proxy in project_proxies(project_id, cached) ] cache.CACHE.save(key, data, 3600) return data @@ -90,12 +89,11 @@ def all_proxies(cached=True): @functools.lru_cache(maxsize=1024) def parse_backend(backend): """Parse a proxy backend specification.""" - m = RE_BACKEND.match(backend) - if not m: - return {'hostname': backend} - data = m.groupdict() - if utils.is_ipv4(data['host']): - data['hostname'] = socket.getfqdn(data['host']) - else: - data['hostname'] = data['host'] - return data + url = urlparse(backend) + if not url or not url.hostname: + return None + + if not utils.is_ip(url.hostname): + return None + + return socket.getfqdn(url.hostname) diff --git a/keystone_browser/puppetclasses.py b/keystone_browser/puppetclasses.py index ab31906..0e5f372 100644 --- a/keystone_browser/puppetclasses.py +++ b/keystone_browser/puppetclasses.py @@ -20,7 +20,6 @@ import functools -import requests import yaml from . import cache @@ -29,80 +28,110 @@ @functools.lru_cache(maxsize=1) def url_template(): - """Get the url template for accessing the proxy service.""" - return "http://labs-puppetmaster.wikimedia.org:8100/v1" + """Get the url template for accessing the Puppet ENC service.""" + c = keystone.keystone_client() + proxy = c.services.list(type="puppet-enc")[0] + endpoint = c.endpoints.list(service=proxy.id, interface="public")[0] + + return endpoint.url.replace("/$(project_id)s", "") + + +@functools.lru_cache(maxsize=None) +def puppet_enc_client(project="observer"): + session = keystone.session(project) + return url_template(), session def prefixes(classname, cached=True): - """Return a dict of {: [prefixes]} for a given puppet class - """ + """Return a dict of {: [prefixes]} for a given puppet class""" - key = 'puppetprefixes:{}'.format(classname) + key = "puppetprefixes:{}".format(classname) data = None if cached: data = cache.CACHE.load(key) if data is None: - url = url_template() + "/prefix/" + classname - req = requests.get(url, verify=False) + base_url, session = puppet_enc_client() + req = session.get( + f"{base_url}/prefix/{classname}", + raise_exc=False, + headers={"Accept": "application/x-yaml"}, + ) if req.status_code != 200: - data = [] + data_with_ids = {} else: - data = yaml.safe_load(req.text) + data_with_ids = yaml.safe_load(req.text) + + data = { + (keystone.project_name_for_id(key) or key): value + for key, value in data_with_ids.items() + } + cache.CACHE.save(key, data, 1200) return data def all_classes(cached=True): - """Return a list of all used puppet classes - """ + """Return a list of all used puppet classes""" - key = 'all_puppetclasses' + key = "all_puppetclasses" data = None if cached: data = cache.CACHE.load(key) if data is None: - url = url_template() + '/roles' - req = requests.get(url, verify=False) + base_url, session = puppet_enc_client() + req = session.get( + f"{base_url}/roles", + raise_exc=False, + headers={"Accept": "application/x-yaml"}, + ) if req.status_code != 200: data = [] else: data = yaml.safe_load(req.text) + cache.CACHE.save(key, data, 1200) - return data['roles'] + return data["roles"] def project_prefixes(project, cached=True): - """Return a dict of [prefixes] for a given project - """ + """Return a dict of [prefixes] for a given project""" - key = 'puppetprojectprefixess:{}'.format(project) + key = "puppetprojectprefixess:{}".format(project) data = None if cached: data = cache.CACHE.load(key) if data is None: - url = url_template() + "/" + project + "/prefix" - req = requests.get(url, verify=False) + base_url, session = puppet_enc_client(project) + req = session.get( + f"{base_url}/{project}/prefix", + raise_exc=False, + headers={"Accept": "application/x-yaml"}, + ) if req.status_code != 200: data = [] else: data = yaml.safe_load(req.text) cache.CACHE.save(key, data, 1200) - return data['prefixes'] + return data["prefixes"] def config(project, fqdn, cached=True): """Get full puppet config for a prefix. - Returns a dict with 'roles' and 'hiera' keys. + Returns a dict with 'roles' and 'hiera' keys. """ - key = 'puppetconfig:{}'.format(fqdn) + key = "puppetconfig:{}".format(fqdn) data = None if cached: data = cache.CACHE.load(key) if data is None: - url = url_template() + '/' + project + "/node/" + fqdn - req = requests.get(url, verify=False) + base_url, session = puppet_enc_client(project) + req = session.get( + f"{base_url}/{project}/node/{fqdn}", + raise_exc=False, + headers={"Accept": "application/x-yaml"}, + ) if req.status_code != 200: data = [] else: @@ -112,52 +141,49 @@ def config(project, fqdn, cached=True): def classes(project, fqdn, cached=True): - """Return a list of puppet classes for the given project and fqdn - """ - return config(project, fqdn, cached)['roles'] + """Return a list of puppet classes for the given project and fqdn""" + return config(project, fqdn, cached)["roles"] def hiera(project, fqdn, cached=True): - """Return a list of puppet classes for the given project and fqdn - """ + """Return a list of puppet classes for the given project and fqdn""" """Return a list of puppet classes for the given project and fqdn """ - return config(project, fqdn, cached)['hiera'] + return config(project, fqdn, cached)["hiera"] def giant_hiera_dict(cached=True): """Gather up the hiera config for every possible instance. - This is incredibly slow and expensive, but it'll get all - the caches warmed up! + This is incredibly slow and expensive, but it'll get all + the caches warmed up! - Make a dict of the form - {hiera_key: - {project_id: - {fqdn: hiera_value}}} + Make a dict of the form + {hiera_key: + {project_name: + {fqdn: hiera_value}}} """ - key = 'completehieradictt:' + key = "completehieradictt:" data = None if cached: data = cache.CACHE.load(key) if data is None: data = {} - for project in keystone.all_projects(): - for prefix in project_prefixes(project): - hieradata = hiera(project, prefix, cached) + for project_id, project_name in keystone.all_projects().items(): + for prefix in project_prefixes(project_id): + hieradata = hiera(project_id, prefix, cached) for key in hieradata.keys(): if key not in data: data[key] = {} - if project not in data[key]: - data[key][project] = {} + if project_id not in data[key]: + data[key][project_name] = {} if key in data: - data[key][project][prefix] = hieradata[key] + data[key][project_name][prefix] = hieradata[key] cache.CACHE.save(key, data, 1200) return data def hieraprefixes(hierakey, cached=True): - """dict of {: {prefix: value}} for a given hiera key - """ + """dict of {: {prefix: value}} for a given hiera key""" return giant_hiera_dict(cached).get(hierakey, {}) diff --git a/keystone_browser/stats.py b/keystone_browser/stats.py index 4cd7578..54cec49 100644 --- a/keystone_browser/stats.py +++ b/keystone_browser/stats.py @@ -21,34 +21,38 @@ import datetime from . import cache +from . import cinder from . import keystone from . import ldap from . import nova def usage(cached=True): - key = 'stats:usage' + key = "stats:usage" data = None if cached: data = cache.CACHE.load(key) if data is None: - projects = [p for p in keystone.all_projects() if p != 'admin'] + projects = [p for p in keystone.all_projects().keys() if p != "admin"] data = collections.defaultdict(int) - data['projects'] = len(projects) + data["projects"] = len(projects) for p in projects: types = collections.defaultdict(int) for s in nova.project_servers(p): - data['instances'] += 1 - types[s['flavor']['id']] += 1 + data["instances"] += 1 + types[s["flavor"]["id"]] += 1 for label, flavor in nova.flavors(p).items(): - data['ram'] += types[label] * flavor['ram'] - data['vcpus'] += types[label] * flavor['vcpus'] - data['disk'] += types[label] * flavor['disk'] + data["ram"] += types[label] * flavor["ram"] + data["vcpus"] += types[label] * flavor["vcpus"] + data["disk"] += types[label] * flavor["disk"] - data['users'] = ldap.user_count() - data['generated'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + for volume in cinder.project_volumes(p): + data["disk"] += volume["size"] + + data["users"] = ldap.user_count() + data["generated"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") # Cache for 25 hours cache.CACHE.save(key, data, 90000) return data diff --git a/keystone_browser/trove.py b/keystone_browser/trove.py new file mode 100644 index 0000000..a0ee5b3 --- /dev/null +++ b/keystone_browser/trove.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of the Keystone browser +# +# Copyright (c) 2021 Taavi Väänänen +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import functools + +from troveclient.v1 import client +from troveclient.apiclient.exceptions import UnprocessableEntity + +from . import cache +from . import keystone + + +@functools.lru_cache(maxsize=None) +def trove_client(project, region): + return client.Client( + session=keystone.session(project), + timeout=2, + region_name=region, + ) + + +@functools.lru_cache() +def get_regions(): + ks_client = keystone.keystone_client() + region_recs = ks_client.regions.list() + return [region.id for region in region_recs] + + +# TODO: failing with troveclient.apiclient.exceptions.Unauthorized: +# User does not have admin privileges. (HTTP 401) +# def limits(project, cached=True): +# """Get a dict of limit details.""" +# key = "trove:limits:{}".format(project) +# data = None +# if cached: +# data = cache.CACHE.load(key) +# if data is None: +# data = {} +# for region in get_regions(): +# trove = trove_client(project, region) +# data[region] = trove.quota.show(project) +# cache.CACHE.save(key, data, 3600) +# return data + + +def project_instances(project, cached=True): + """Get a list of all database instances in a given project.""" + key = "trove:project-instances:{}".format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = [] + for region in get_regions(): + trove = trove_client(project, region) + data.extend( + [instance._info for instance in trove.instances.list()] + ) + cache.CACHE.save(key, data, 300) + return data + + +def instance(project, name, cached=True): + key = "trove:instance:{}:{}".format(project, name) + data = None + if cached: + data = cache.CACHE.load(key) + + if data is None: + for region in get_regions(): + trove = trove_client(project, region) + instances = [ + instance + for instance in trove.instances.list(include_clustered=False) + if instance.name == name + ] + if instances: + instance = instances[0] + data = { + "name": instance.name, + "id": instance.id, + "hostname": instance.hostname, + "datastore": instance.datastore, + "flavor": instance.flavor["id"], + "status": instance.status, + "operating_status": instance.operating_status, + "size": instance.volume["size"], + } + + try: + data["databases"] = [ + database.name + for database in trove.databases.list(instance) + ] + except UnprocessableEntity: + data["databases"] = [] + + cache.CACHE.save(key, data, 300) + + return data diff --git a/keystone_browser/utils.py b/keystone_browser/utils.py index 6679648..8248eed 100644 --- a/keystone_browser/utils.py +++ b/keystone_browser/utils.py @@ -18,14 +18,25 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +import ipaddress import re +from typing import List, Union -RE_IPV4ADDR = re.compile( - r'^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}' - r'(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$' -) +RE_DIGIT = re.compile("(\\d+)") -def is_ipv4(s): - """Is the given string an IPv4 address?""" - return RE_IPV4ADDR.match(s) is not None +def is_ip(s): + """Is the given string an IP address?""" + try: + ipaddress.ip_address(s) + return True + except ValueError: + return False + + +def natural_sort_key(element: str) -> List[Union[str, int]]: + """Changes "name-12.something.com" into ["name-", 12, ".something.com"].""" + return [ + int(mychunk) if mychunk.isdigit() else mychunk + for mychunk in RE_DIGIT.split(element) + ] diff --git a/keystone_browser/utils_test.py b/keystone_browser/utils_test.py new file mode 100644 index 0000000..8185dc6 --- /dev/null +++ b/keystone_browser/utils_test.py @@ -0,0 +1,7 @@ +from . import utils + + +def test_is_ip(): + assert utils.is_ip("192.0.2.1") + assert utils.is_ip("::1") + assert not utils.is_ip("example.com") diff --git a/keystone_browser/zones.py b/keystone_browser/zones.py index b1e82ed..7d6644c 100644 --- a/keystone_browser/zones.py +++ b/keystone_browser/zones.py @@ -19,96 +19,75 @@ # with this program. If not, see . import functools -import operator from designateclient.v2 import client as designate_client from . import cache from . import keystone -from . import nova +from . import utils @functools.lru_cache(maxsize=None) def client(project): - return designate_client.Client( - session=keystone.session(project)) + return designate_client.Client(session=keystone.session(project)) -@functools.lru_cache(maxsize=None) -def _raw_zones(project): - """Return list of designate 'zone' objects owned by a project. - - Note that in designate, dns domains are called 'Zones' because the word - 'Domain' was used by Keystone for some totally other thing. - """ - return client(project).zones.list() - - -def zones(project): - """Return a simple list of zones owned by a project.""" - raw_zones = _raw_zones(project) - return [zone['name'] for zone in raw_zones] +def zone(project, zone, cached): + key = "zones:by-name:{}:{}".format(project, zone) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + for z in client(project).zones.list(): + if z["name"] == zone: + data = z + cache.CACHE.save(key, data, 3600) + break + return data -@functools.lru_cache(maxsize=None) -def _raw_recordsets(project, zone): - """Return list of designate 'recordset' objects for a given - project and zone name. - """ - for z in _raw_zones(project): - if z['name'] == zone: - return client(project).recordsets.list(z['id']) - return [] +def _format_record_name(name: str, zone: str) -> str: + if name == zone: + return "@" + return name.removesuffix(f".{zone}") -@functools.lru_cache(maxsize=None) -def a_records(project, zone): - """Return a list of dns A records for a given project and zone. +def records(project, zone_name, zone_id, cached): + """Return a list of dns records for a given zone ID. Each record is in the format described at https://developer.openstack.org/api-ref/dns/?expanded=list-all-recordsets-owned-by-project-detail """ - raw_recordsets = _raw_recordsets(project, zone) - return [r for r in raw_recordsets if r['type'] == 'A'] - - -@functools.lru_cache(maxsize=None) -def floating_ips(project): - """Get a list of floating ips allocated to a project.""" - novaclient = nova.nova_client(project) - ips = novaclient.floating_ips.list() - return [ip.ip for ip in ips] - - -@functools.lru_cache(maxsize=None) -def wmflabsdotorg_a_records(project): - """Get a list of *.wmflabs.org records matching IPs allocated to - a project. - - Records under wmflabs.org are a special cased because they are all owned - by a special 'wmflabsdotorg' project. - """ - return [ - r for r in a_records('wmflabsdotorg', 'wmflabs.org.') - if r['records'][0] in floating_ips(project) - ] - + key = "zones:records:{}:{}".format(project, zone_id) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + raw_recordsets = client(project).recordsets.list(zone_id) + data = [ + { + "name": _format_record_name(r["name"], zone_name), + "type": r["type"], + "records": r["records"], + "status": r["status"], + "sortkey": utils.natural_sort_key( + r["name"].removesuffix(zone_id) + ), + } + for r in raw_recordsets + if (r["type"] not in ("NS", "SOA") or r["name"] != zone_name) + ] + cache.CACHE.save(key, data, 3600) + return data -def all_a_records(project, cached=True): - """Get all the A records associated with a project. - Returns a dict keyed by host with values being lists of ip addresses. - """ - key = 'zones:A:{}'.format(project) +def all_dns_zones(project, cached=True): + """Get all the DNS zones for a specific project.""" + key = "zones:per-project:{}".format(project) data = None if cached: data = cache.CACHE.load(key) if data is None: - data = functools.reduce( - operator.add, - [a_records(project, zone) for zone in zones(project)], - [] - ) - data += wmflabsdotorg_a_records(project) + data = [zone["name"] for zone in client(project).zones.list()] cache.CACHE.save(key, data, 3600) return data diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4477e86 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +line-length = 79 +target_version = ['py311'] +include = '\.pyi?$' diff --git a/requirements.txt b/requirements.txt index b6dcdff..1cf88b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,15 @@ +dnspython flask +gunicorn ldap3 +openstacksdk +python-cinderclient python-glanceclient python-keystoneclient -python-novaclient==7.1.0 # v8.0.0 dropped support for nova-network +python-neutronclient +python-novaclient python-designateclient +python-troveclient +pyyaml redis +wmflib diff --git a/templates/databaseinstance.html b/templates/databaseinstance.html new file mode 100644 index 0000000..6b4b4c7 --- /dev/null +++ b/templates/databaseinstance.html @@ -0,0 +1,78 @@ +{% extends "layout.html" %} + +{% block title %}Database instance {{ name }} - {{ super() }}{% endblock %} + +{% block content %} + + + + +{% if instance %} +{% set flavor = flavors[instance.flavor]|default('') %} +
+
+ +
+
+
Project
+
{{ project }}
+
Instance name
+
{{ instance.name }}
+
Instance ID
+
{{ instance.id }}
+
Host
+
{{ instance.hostname }}
+
Status
+
+ {% if instance.status == "ACTIVE" %} + Active, + {% else %} + {{ instance.status }}, + {% endif %} + + {% if instance.operating_status == "HEALTHY" %} + Healthy + {% else %} + {{ instance.operating_status }} + {% endif %} +
+
Type
+
{{ instance.datastore.type }} {{ instance.datastore.version }}
+
Storage size
+
{{ instance.size }}G
+
Resources
+
+ {{ flavor.vcpus|default('-') }} CPUs, + {{ flavor.ram|default('-') }}M RAM + ({{ flavor.name }}) +
+ {% if instance.databases %} +
Databases
+
+
    + {% for database in instance.databases | sort %} +
  • {{ database }}
  • + {% endfor %} +
+
+ {% endif %} +
+
+
+
+{% else %} +

Unknown database instance '{{ name }}'. Are you just guessing?

+{% endif %} +{% endblock %} diff --git a/templates/flavors.html b/templates/flavors.html new file mode 100644 index 0000000..d65cdf6 --- /dev/null +++ b/templates/flavors.html @@ -0,0 +1,47 @@ +{% extends "layout.html" %} + +{% block title %}Flavors - {{ super() }}{% endblock %} + +{% block content %} +
+
+ + + + + + + + + + + {% for flavor in flavors|sort(attribute="vcpus") %} + + + + + + {% endfor %} + +
NameCPUsRAM
{{ flavor.name|default('UNKNOWN') }}{{ flavor.vcpus|default('-') }}{{ flavor.ram|default('-') }}M
+
+
+{% endblock %} + +{% block css %} + +{% endblock %} +{% block js %} +{{ super() }} + + +{% endblock %} diff --git a/templates/hierakey.html b/templates/hierakey.html index 76ebd96..7591a8e 100644 --- a/templates/hierakey.html +++ b/templates/hierakey.html @@ -23,14 +23,14 @@

{% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

Project: {{ project }}

+

Project: {{ project }}

    {% if '_' in prefixes.keys() %}
  • All project instances
  • {% endif %} {% for prefix in prefixes.keys()|sort() %} {% if prefix != '_' %} - {% if prefix.endswith('wmflabs') %} + {% if prefix.endswith('.cloud') %}
  • {{ prefix }}
  • {% else %}
  • {{ prefix }}*
  • diff --git a/templates/home.html b/templates/home.html index 91413dd..07a28a9 100644 --- a/templates/home.html +++ b/templates/home.html @@ -2,10 +2,13 @@ {% block content %}
    +{% if usage %}
    - - Usage +

    + + Usage +

    @@ -39,5 +42,18 @@ Last updated: {{ usage.generated }} UTC +{% endif %} {% endblock %} +{% block css %} + +{% endblock %} +{% block js %} +{{ super() }} + + +{% endblock %} diff --git a/templates/layout.html b/templates/layout.html index 4edadc1..164342e 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -35,12 +35,23 @@
  • Projects
  • {% endwith %} {% with servers = url_for('servers') %} -
  • Servers
  • +
  • Servers
  • {% endwith %}
  • Proxies
  • {% with classes = url_for('all_puppetclasses') %}
  • Puppet Classes
  • {% endwith %} + {% with flavors = url_for('flavors') %} + {% with networks = url_for('networks') %} + + {% endwith %} + {% endwith %}
    + + + + + + + + + {% for network in networks.values()|sort(attribute='name') %} + {% for subnet in network.subnets|sort(attribute='cidr') %} + + {% if loop.first %} + {% set rowspan = network.subnets|length %} + + {% endif %} + + + + {% endfor %} + {% endfor %} + +
    NetworkSubnetSubnet CIDR
    {{ network.name }}{{ subnet.name }}{{ subnet.cidr }}
    +
    +
    +{% endblock %} + +{% block css %} + +{% endblock %} +{% block js %} +{{ super() }} + + +{% endblock %} diff --git a/templates/project.html b/templates/project.html index af85fee..0a8e9b9 100644 --- a/templates/project.html +++ b/templates/project.html @@ -1,19 +1,39 @@ {% extends "layout.html" %} -{% block title %}Project {{ project }} - {{ super() }}{% endblock %} +{% block title %}Project {{ project_name }} - {{ super() }}{% endblock %} {% block content %} -{% if admins or users or servers or proxies or zones %} +{% if data %}
    +
    + +
    +
    +
    Name
    +
    {{ data.name }}
    +
    ID
    +
    {{ data.id }}
    + {% if data.description %} +
    Description
    +
    {{ data.description }}
    + {% endif %} +
    +
    +
    - {% if admins %} +
    + +
    + + {% for region, region_limits in limits.items() %} + {% set cinder_region_limits = cinder_limits[region] %} + {% set neutron_region_limits = neutron_limits[region]['quota'] %} +
    {{ region }}
    +
    + {{ region_limits['absolute']['totalInstancesUsed'] }} / {{ region_limits['absolute']['maxTotalInstances'] }} instances. + {{ region_limits['absolute']['totalCoresUsed'] }} / {{ region_limits['absolute']['maxTotalCores'] }} VCPUs. + {{ region_limits['absolute']['totalRAMUsed'] / 1024 }} GB / {{ region_limits['absolute']['maxTotalRAMSize'] / 1024 }} GB RAM. + {{ neutron_region_limits['floatingip']['used'] }} / {{ neutron_region_limits['floatingip']['limit'] }} floating IPs. + {{ neutron_region_limits['security_group']['used'] }} / {{ neutron_region_limits['security_group']['limit'] }} security groups. + {{ cinder_region_limits['volumes']['in_use'] }} / {{ cinder_region_limits['volumes']['limit'] }} volumes. + {{ cinder_region_limits['gigabytes']['in_use'] }} / {{ cinder_region_limits['gigabytes']['limit'] }} GB volume space. +
    + {% endfor %} +
    +
    + {% if members %}
      - {% for u in admins|sort(attribute='cn') %} + {% for u in members|sort(attribute='cn') %}
    • {{ u.cn }}
    • {% endfor %}
    {% endif %} - {% if users %} + {% if viewers %}
    -