From 53fb118966da2b40c4e32191511f1f91d20265cb Mon Sep 17 00:00:00 2001 From: Brooke Storm Date: Wed, 22 Aug 2018 16:34:21 -0400 Subject: [PATCH 001/174] Last commit needs a fixup to work right -- regions come from keystone --- keystone_browser/nova.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index f2ffb0a..cccf0fc 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -39,8 +39,8 @@ def nova_client(project, region): @functools.lru_cache() def get_regions(): - nova = client.Client('2.12', session=keystone.session, endpoint_type='public') - region_recs = nova.regions.list() + ks_client = keystone.keystone_client() + region_recs = ks_client.regions.list() return [region.id for region in region_recs] From 18280b20ab237acd4e80614158368e184726472c Mon Sep 17 00:00:00 2001 From: Brooke Storm Date: Wed, 22 Aug 2018 16:46:06 -0400 Subject: [PATCH 002/174] zones: the ip-getter uses the nova_client interface as well --- keystone_browser/zones.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/keystone_browser/zones.py b/keystone_browser/zones.py index b1e82ed..03bdefb 100644 --- a/keystone_browser/zones.py +++ b/keystone_browser/zones.py @@ -75,9 +75,11 @@ def a_records(project, zone): @functools.lru_cache(maxsize=None) def floating_ips(project): """Get a list of floating ips allocated to a project.""" - novaclient = nova.nova_client(project) - ips = novaclient.floating_ips.list() - return [ip.ip for ip in ips] + ips = [] + for region in nova.get_regions(): + novaclient = nova.nova_client(project, region) + ips.extend([ip.ip for ip in novaclient.floating_ips.list()]) + return ips @functools.lru_cache(maxsize=None) From fe243fa5b1906e66d68e2f9e8a775deedb47f3d5 Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Mon, 1 Oct 2018 17:31:31 -0600 Subject: [PATCH 003/174] Support multiple SDNs The legacy 'main' region had a single software-defined network named 'public'. The new 'eqiad1' region currently has a SDN named 'lan-flat-cloudinstances2b', but in theory could have additional networks globally or per project in the future. Update display templates dealing with addresses to iterate all networks returned by the API rather than hardcoding checks just for the legacy 'public' network. Bug: T205869 --- templates/project.html | 10 ++++++---- templates/server.html | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/templates/project.html b/templates/project.html index af85fee..db8d55b 100644 --- a/templates/project.html +++ b/templates/project.html @@ -167,11 +167,13 @@

{{ fqdn }} {{ flavor.name|default('UNKNOWN') }} {{ image.name|default('UNKNOWN') }} - {% if 'public' in server.addresses %} - {{ server.addresses.public[0].addr }} - {% else %} - - + {% for sdn, interfaces in server.addresses.items() %} + {% for interface in interfaces %} + {% if interface['OS-EXT-IPS:type'] == 'fixed' %} + {{ interface.addr }} {% endif %} + {% endfor %} + {% endfor %} {{ flavor.vcpus|default('-') }} {{ flavor.ram|default('-') }}M {{ flavor.disk|default('-') }}G diff --git a/templates/server.html b/templates/server.html index de855ff..f639688 100644 --- a/templates/server.html +++ b/templates/server.html @@ -33,13 +33,13 @@

Instance Id
{{ server.id }}
IP
- {% if 'public' in server.addresses %} - {% for interface in server.addresses.public %} + {% for sdn, interfaces in server.addresses.items() %} + {% for interface in interfaces %}
{{ interface.addr }} [{{ interface['OS-EXT-IPS:type'] }}]
{% endfor %} {% else %}
-
- {% endif %} + {% endfor %}
Status
{{ server.status }}
Type
From 88f9554e9bccfcd78ccb895ddd2246409ee06088 Mon Sep 17 00:00:00 2001 From: Alex Monk Date: Thu, 1 Nov 2018 16:01:08 +0000 Subject: [PATCH 004/174] Show instance hypervisors Bug: T208502 --- templates/server.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/server.html b/templates/server.html index f639688..a6f93c0 100644 --- a/templates/server.html +++ b/templates/server.html @@ -42,6 +42,8 @@

{% endfor %}
Status
{{ server.status }}
+
Host hypervisor
+
{{ server['OS-EXT-SRV-ATTR:hypervisor_hostname'] }}
Type
{{ flavor.name|default('UNKNOWN') }}
Image
From 2beb3e12b9ef6fb55191ea80428cc3d3a2e8d840 Mon Sep 17 00:00:00 2001 From: Alex Monk Date: Thu, 1 Nov 2018 20:56:03 +0000 Subject: [PATCH 005/174] Display limits for projects Bug: T164332 --- app.py | 1 + keystone_browser/nova.py | 14 ++++++++++++++ templates/project.html | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/app.py b/app.py index a54a51e..513a204 100644 --- a/app.py +++ b/app.py @@ -91,6 +91,7 @@ def project(name): 'images': glance.images(), 'proxies': proxies.project_proxies(name, cached), 'zones': zones.all_a_records(name, cached), + 'limits': nova.limits(name), }) except Exception: app.logger.exception( diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index cccf0fc..698a01b 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -83,6 +83,20 @@ def flavors(project): return data +def limits(project): + """Get a dict of limit details.""" + key = 'nova:limits:{}'.format(project) + data = cache.CACHE.load(key) + if data is None: + data = {} + for region in get_regions(): + nova = nova_client(project, region) + data[region] = nova.limits.get().to_dict() + + cache.CACHE.save(key, data, 3600) + return data + + def all_servers(): """Get a list of all servers in all projects.""" key = 'keystone:all_servers' diff --git a/templates/project.html b/templates/project.html index db8d55b..eeb4a83 100644 --- a/templates/project.html +++ b/templates/project.html @@ -29,6 +29,27 @@

+
+ +
+ + {% for region, region_limits in limits.items() %} +
{{ region }}
+
+ {{ region_limits['absolute']['totalInstancesUsed'] }} / {{ region_limits['absolute']['maxTotalInstances'] }} instances. + {{ region_limits['absolute']['totalCoresUsed'] }} / {{ region_limits['absolute']['maxTotalCores'] }} VCPUs. + {{ region_limits['absolute']['totalRAMUsed'] / 1024 }} GB / {{ region_limits['absolute']['maxTotalRAMSize'] / 1024 }} GB RAM. + {{ region_limits['absolute']['totalFloatingIpsUsed'] }} / {{ region_limits['absolute']['maxTotalFloatingIps'] }} floating IPs. + {{ region_limits['absolute']['totalSecurityGroupsUsed'] }} / {{ region_limits['absolute']['maxSecurityGroups'] }} security groups. +
+ {% endfor %} +
+
{% if admins %}
From cd61ba5e7a5fec9c6d161b659d558265b5494240 Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Tue, 8 Sep 2020 11:07:32 -0600 Subject: [PATCH 013/174] Update hardcoded fqdns Change hardcoded FQDN completion from `eqiad.wmflabs` to `eqiad1.wikimedia.cloud`. Bug: T262293 --- app.py | 10 +++++++--- templates/project.html | 2 +- templates/servers.html | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 513a204..b949eec 100644 --- a/app.py +++ b/app.py @@ -208,7 +208,7 @@ def api_projects_txt(): def api_dsh_project(name): servers = nova.project_servers(name) dsh = [ - "{}.{}.eqiad.wmflabs".format(server['name'], name) + "{}.{}.eqiad1.wikimedia.cloud".format(server['name'], name) for server in servers ] return flask.Response('\n'.join(sorted(dsh)), mimetype='text/plain') @@ -218,7 +218,9 @@ def api_dsh_project(name): def api_dsh_servers(): servers = nova.all_servers() dsh = [ - "{}.{}.eqiad.wmflabs".format(server['name'], server['tenant_id']) + "{}.{}.eqiad1.wikimedia.cloud".format( + server['name'], server['tenant_id'] + ) for server in servers ] return flask.Response('\n'.join(sorted(dsh)), mimetype='text/plain') @@ -244,7 +246,9 @@ def api_dsh_puppet(name): dsh.append(prefix) else: dsh.extend([ - "{}.{}.eqiad.wmflabs".format(server['name'], project) + "{}.{}.eqiad1.wikimedia.cloud".format( + server['name'], project + ) for server in servers if server['name'].startswith(prefix) ]) diff --git a/templates/project.html b/templates/project.html index ef024bb..69bf0bd 100644 --- a/templates/project.html +++ b/templates/project.html @@ -184,7 +184,7 @@

{% for server in servers|sort(attribute='name') %} {% set image = images[server.image.id]|default('') %} {% set flavor = flavors[server.flavor.id]|default('') %} - {% set fqdn = server.name ~ '.' ~ project ~ '.eqiad.wmflabs' %} + {% set fqdn = server.name ~ '.' ~ project ~ '.eqiad1.wikimedia.cloud' %} {{ fqdn }} {{ flavor.name|default('UNKNOWN') }} diff --git a/templates/servers.html b/templates/servers.html index 345c04d..907098e 100644 --- a/templates/servers.html +++ b/templates/servers.html @@ -10,7 +10,7 @@
    {% for server in servers %} - {% set fqdn = server.name ~ '.' ~ server.tenant_id ~ '.eqiad.wmflabs' %} + {% set fqdn = server.name ~ '.' ~ server.tenant_id ~ '.eqiad1.wikimedia.cloud' %}
  • {{ fqdn }}
    • {% endfor %}
From d1e97d862bc6f13d880654c8269c579f5eece8d0 Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Tue, 8 Sep 2020 11:15:11 -0600 Subject: [PATCH 014/174] Update for modern versions of flask The ProxyFix middleware was moved from werkzeug.contrib.fixers to werkzeug.middleware.proxy_fix in the 0.15.0 release. --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index b949eec..de538c8 100644 --- a/app.py +++ b/app.py @@ -19,7 +19,7 @@ # with this program. If not, see . import flask -import werkzeug.contrib.fixers +import werkzeug.middleware.proxy_fix from keystone_browser import zones from keystone_browser import glance @@ -33,7 +33,7 @@ app = flask.Flask(__name__) -app.wsgi_app = werkzeug.contrib.fixers.ProxyFix(app.wsgi_app) +app.wsgi_app = werkzeug.middleware.proxy_fix.ProxyFix(app.wsgi_app) @app.route('/') From e66b14994b7b021a3ab111e51e1edf3f29e8687f Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Tue, 8 Sep 2020 11:19:02 -0600 Subject: [PATCH 015/174] Bump Python version to 3.7 and format with black --- app.py | 231 ++++++++++++++++-------------- keystone_browser/cache.py | 13 +- keystone_browser/glance.py | 10 +- keystone_browser/keystone.py | 51 +++---- keystone_browser/ldap.py | 51 ++++--- keystone_browser/nova.py | 41 +++--- keystone_browser/proxies.py | 29 ++-- keystone_browser/puppetclasses.py | 54 ++++--- keystone_browser/stats.py | 20 +-- keystone_browser/utils.py | 4 +- keystone_browser/zones.py | 22 +-- pyproject.toml | 4 + tox.ini | 4 +- 13 files changed, 282 insertions(+), 252 deletions(-) create mode 100644 pyproject.toml diff --git a/app.py b/app.py index de538c8..4105fcc 100644 --- a/app.py +++ b/app.py @@ -36,245 +36,262 @@ app.wsgi_app = werkzeug.middleware.proxy_fix.ProxyFix(app.wsgi_app) -@app.route('/') +@app.route("/") def home(): ctx = {} try: - cached = 'purge' not in flask.request.args - ctx.update({ - 'usage': stats.usage(cached), - }) + cached = "purge" not in flask.request.args + ctx.update( + { + "usage": stats.usage(cached), + } + ) except Exception: - app.logger.exception('Error collecting information for projects') - return flask.render_template('home.html', **ctx) + app.logger.exception("Error collecting information for projects") + return flask.render_template("home.html", **ctx) -@app.route('/project/') +@app.route("/project/") def projects(): ctx = {} try: - ctx.update({ - 'projects': keystone.all_projects(), - }) + ctx.update( + { + "projects": keystone.all_projects(), + } + ) except Exception: - app.logger.exception('Error collecting information for projects') - return flask.render_template('projects.html', **ctx) + app.logger.exception("Error collecting information for projects") + return flask.render_template("projects.html", **ctx) -@app.route('/server/') +@app.route("/server/") def servers(): ctx = {} try: - ctx.update({ - 'servers': nova.all_servers(), - }) + ctx.update( + { + "servers": nova.all_servers(), + } + ) except Exception: - app.logger.exception('Error collecting information for projects') - return flask.render_template('servers.html', **ctx) + app.logger.exception("Error collecting information for projects") + return flask.render_template("servers.html", **ctx) -@app.route('/project/') +@app.route("/project/") def project(name): - cached = 'purge' not in flask.request.args + cached = "purge" not in flask.request.args ctx = { - 'project': name, + "project": name, } try: users = keystone.project_users_by_role(name) - admins = users['admin'] + users['projectadmin'] - ctx.update({ - 'project': name, - 'admins': ldap.get_users_by_uid(admins), - 'users': ldap.get_users_by_uid(users['user']), - 'servers': nova.project_servers(name), - 'flavors': nova.flavors(name), - 'images': glance.images(), - 'proxies': proxies.project_proxies(name, cached), - 'zones': zones.all_a_records(name, cached), - 'limits': nova.limits(name), - }) + admins = users["admin"] + users["projectadmin"] + ctx.update( + { + "project": name, + "admins": ldap.get_users_by_uid(admins), + "users": ldap.get_users_by_uid(users["user"]), + "servers": nova.project_servers(name), + "flavors": nova.flavors(name), + "images": glance.images(), + "proxies": proxies.project_proxies(name, cached), + "zones": zones.all_a_records(name, cached), + "limits": nova.limits(name), + } + ) except Exception: app.logger.exception( - 'Error collecting information for project "%s"', name) - return flask.render_template('project.html', **ctx) + 'Error collecting information for project "%s"', name + ) + return flask.render_template("project.html", **ctx) -@app.route('/user/') +@app.route("/user/") def user(uid): ctx = { - 'uid': uid, + "uid": uid, } try: - ctx.update({ - 'user': ldap.get_users_by_uid([uid]), - 'projects': keystone.projects_for_user(uid), - }) - if ctx['user']: - ctx['user'] = ctx['user'][0] + ctx.update( + { + "user": ldap.get_users_by_uid([uid]), + "projects": keystone.projects_for_user(uid), + } + ) + if ctx["user"]: + ctx["user"] = ctx["user"][0] except Exception: - app.logger.exception( - 'Error collecting information for user "%s"', uid) - return flask.render_template('user.html', **ctx) + app.logger.exception('Error collecting information for user "%s"', uid) + return flask.render_template("user.html", **ctx) -@app.route('/server/') +@app.route("/server/") def server(fqdn): - name, project, tld = fqdn.split('.', 2) + name, project, tld = fqdn.split(".", 2) ctx = { - 'fqdn': fqdn, - 'project': project, + "fqdn": fqdn, + "project": project, } try: - ctx.update({ - 'server': nova.server(fqdn), - 'flavors': nova.flavors(project), - 'images': glance.images(), - 'puppetclasses': puppetclasses.classes(project, fqdn), - 'hiera': puppetclasses.hiera(project, fqdn), - }) - if 'user_id' in ctx['server']: - user = ldap.get_users_by_uid([ctx['server']['user_id']]) + ctx.update( + { + "server": nova.server(fqdn), + "flavors": nova.flavors(project), + "images": glance.images(), + "puppetclasses": puppetclasses.classes(project, fqdn), + "hiera": puppetclasses.hiera(project, fqdn), + } + ) + if "user_id" in ctx["server"]: + user = ldap.get_users_by_uid([ctx["server"]["user_id"]]) if user: - ctx['owner'] = user[0] + ctx["owner"] = user[0] except Exception: app.logger.exception( - 'Error collecting information for server "%s"', fqdn) + 'Error collecting information for server "%s"', fqdn + ) - return flask.render_template('server.html', **ctx) + return flask.render_template("server.html", **ctx) -@app.route('/puppetclass/') +@app.route("/puppetclass/") def all_puppetclasses(): ctx = {} try: ctx.update({"puppetclasses": puppetclasses.all_classes()}) except Exception: - app.logger.exception( - 'Error collecting the list of puppet classes') + app.logger.exception("Error collecting the list of puppet classes") - return flask.render_template('puppetclasses.html', **ctx) + return flask.render_template("puppetclasses.html", **ctx) -@app.route('/puppetclass/') +@app.route("/puppetclass/") def puppetclass(classname): ctx = { - 'puppetclass': classname, + "puppetclass": classname, } try: ctx.update({"data": puppetclasses.prefixes(classname)}) except Exception: app.logger.exception( - 'Error collecting information for puppet class "%s"', classname) + 'Error collecting information for puppet class "%s"', classname + ) - return flask.render_template('puppetclass.html', **ctx) + return flask.render_template("puppetclass.html", **ctx) -@app.route('/hierakey/') +@app.route("/hierakey/") def hierakey(hierakey): ctx = { - 'hierakey': hierakey, + "hierakey": hierakey, } try: ctx.update({"data": puppetclasses.hieraprefixes(hierakey)}) except Exception: app.logger.exception( - 'Error collecting information for hiera key "%s"', hierakey) + 'Error collecting information for hiera key "%s"', hierakey + ) - return flask.render_template('hierakey.html', **ctx) + return flask.render_template("hierakey.html", **ctx) -@app.route('/proxy/') +@app.route("/proxy/") def all_proxies(): - cached = 'purge' not in flask.request.args + cached = "purge" not in flask.request.args ctx = { - 'proxies': proxies.all_proxies(cached), + "proxies": proxies.all_proxies(cached), } - return flask.render_template('proxies.html', **ctx) + return flask.render_template("proxies.html", **ctx) -@app.route('/api/projects.json') +@app.route("/api/projects.json") def api_projects_json(): return flask.jsonify(projects=keystone.all_projects()) -@app.route('/api/projects.txt') +@app.route("/api/projects.txt") def api_projects_txt(): return flask.Response( - '\n'.join(sorted(keystone.all_projects())), - mimetype='text/plain') + "\n".join(sorted(keystone.all_projects())), mimetype="text/plain" + ) -@app.route('/api/dsh/project/') +@app.route("/api/dsh/project/") def api_dsh_project(name): servers = nova.project_servers(name) dsh = [ - "{}.{}.eqiad1.wikimedia.cloud".format(server['name'], name) + "{}.{}.eqiad1.wikimedia.cloud".format(server["name"], name) for server in servers ] - return flask.Response('\n'.join(sorted(dsh)), mimetype='text/plain') + return flask.Response("\n".join(sorted(dsh)), mimetype="text/plain") -@app.route('/api/dsh/servers') +@app.route("/api/dsh/servers") def api_dsh_servers(): servers = nova.all_servers() dsh = [ "{}.{}.eqiad1.wikimedia.cloud".format( - server['name'], server['tenant_id'] + server["name"], server["tenant_id"] ) for server in servers ] - return flask.Response('\n'.join(sorted(dsh)), mimetype='text/plain') + return flask.Response("\n".join(sorted(dsh)), mimetype="text/plain") -@app.route('/api/dsh/puppetclass/') +@app.route("/api/dsh/puppetclass/") def api_dsh_puppet(name): data = puppetclasses.prefixes(name) dsh = [] for project, d in data.items(): - if project == 'admin': + if project == "admin": continue try: servers = nova.project_servers(project) except Exception: app.logger.exception( - 'Error collecting the list of servers for %s', project) + "Error collecting the list of servers for %s", project + ) servers = [] - for prefix in d['prefixes']: - if prefix.endswith('wmflabs'): + for prefix in d["prefixes"]: + if prefix.endswith("wmflabs"): dsh.append(prefix) else: - dsh.extend([ - "{}.{}.eqiad1.wikimedia.cloud".format( - server['name'], project - ) - for server in servers - if server['name'].startswith(prefix) - ]) + dsh.extend( + [ + "{}.{}.eqiad1.wikimedia.cloud".format( + server["name"], project + ) + for server in servers + if server["name"].startswith(prefix) + ] + ) - return flask.Response('\n'.join(sorted(set(dsh))), mimetype='text/plain') + return flask.Response("\n".join(sorted(set(dsh))), mimetype="text/plain") -@app.route('/api/hierakey/') +@app.route("/api/hierakey/") def api_hierakey(hierakey): return flask.jsonify(servers=puppetclasses.hieraprefixes(hierakey)) @app.errorhandler(404) def page_not_found(e): - return flask.redirect(flask.url_for('projects')) + return flask.redirect(flask.url_for("projects")) -@app.template_filter('contains') +@app.template_filter("contains") def contains(haystack, needle): return needle in haystack -@app.template_filter('extract_hostname') +@app.template_filter("extract_hostname") def extract_hostname(backend): """Extract a hostname from a backend description.""" - return proxies.parse_backend(backend).get('hostname', '404') + return proxies.parse_backend(backend).get("hostname", "404") @app.template_test() diff --git a/keystone_browser/cache.py b/keystone_browser/cache.py index a2d86c1..99bcf17 100644 --- a/keystone_browser/cache.py +++ b/keystone_browser/cache.py @@ -27,24 +27,25 @@ class Cache(object): """Simple redis wrapper.""" - def __init__(self, enabled=True, seed=''): + + def __init__(self, enabled=True, seed=""): self.enabled = enabled self.conn = redis.StrictRedis( - host='tools-redis.svc.eqiad.wmflabs', + host="tools-redis.svc.eqiad.wmflabs", decode_responses=True, ) u = pwd.getpwuid(os.getuid()) self.prefix = hashlib.sha1( - '{}{}.{}'.format(seed, u.pw_name, u.pw_dir).encode('utf-8') + "{}{}.{}".format(seed, u.pw_name, u.pw_dir).encode("utf-8") ).hexdigest() def key(self, val): - return '{}:{}'.format(self.prefix, val) + return "{}:{}".format(self.prefix, val) def load(self, key): if self.enabled: try: - return json.loads(self.conn.get(self.key(key)) or '') + return json.loads(self.conn.get(self.key(key)) or "") except ValueError: return None else: @@ -56,4 +57,4 @@ def save(self, key, data, expiry=300): self.conn.setex(real_key, expiry, json.dumps(data)) -CACHE = Cache(seed='201804132043') +CACHE = Cache(seed="201804132043") diff --git a/keystone_browser/glance.py b/keystone_browser/glance.py index 8063c7e..4c240b7 100644 --- a/keystone_browser/glance.py +++ b/keystone_browser/glance.py @@ -29,9 +29,9 @@ @functools.lru_cache(maxsize=1) def glance_client(): return glanceclient.Client( - version='2', + version="2", session=keystone.session(), - interface='public', + interface="public", ) @@ -44,12 +44,10 @@ def images(): # glance member-create $img observer; # glance member-update $img observer accepted; # done - key = 'glance:images' + key = "glance:images" data = cache.CACHE.load(key) if data is None: glance = glance_client() - data = { - i['id']: i for i in glance.images.list() - } + data = {i["id"]: i for i in glance.images.list()} cache.CACHE.save(key, data, 3600) return data diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 3b81409..884e88d 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -28,27 +28,29 @@ from . import cache -ROLES = collections.OrderedDict([ - ('admin', '2cd63d467f754404bf3746fe63ee0698'), - ('glanceadmin', '1102f4ff63c3435793d0e4340bf4b04e'), - ('observer', '47a8370618ea42d49f7047774e75d262'), - ('projectadmin', '4d8cad783d6342efa8414d7d36fbc034'), - ('user', 'f473273fac7146b3bdbf22e5d4504f95'), -]) +ROLES = collections.OrderedDict( + [ + ("admin", "2cd63d467f754404bf3746fe63ee0698"), + ("glanceadmin", "1102f4ff63c3435793d0e4340bf4b04e"), + ("observer", "47a8370618ea42d49f7047774e75d262"), + ("projectadmin", "4d8cad783d6342efa8414d7d36fbc034"), + ("user", "f473273fac7146b3bdbf22e5d4504f95"), + ] +) @functools.lru_cache(maxsize=None) -def session(project='observer'): +def session(project="observer"): """Get a session for the novaobserver user scoped to the given project.""" # TODO: read settings from /etc/novaobserver.yaml once we get it mounted # into the kubernetes pods () auth = v3.Password( - auth_url='http://cloudcontrol1003.wikimedia.org:5000/v3', - password='Fs6Dq2RtG8KwmM2Z', - username='novaobserver', + auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3", + password="Fs6Dq2RtG8KwmM2Z", + username="novaobserver", project_id=project, - user_domain_name='Default', - project_domain_name='Default', + user_domain_name="Default", + project_domain_name="Default", ) return keystone_session.Session(auth=auth) @@ -56,14 +58,14 @@ def session(project='observer'): def keystone_client(): return client.Client( session=session(), - interface='public', + interface="public", timeout=2, ) def all_projects(): """Get a list of all project names.""" - key = 'keystone:all_projects' + key = "keystone:all_projects" data = cache.CACHE.load(key) if data is None: keystone = keystone_client() @@ -71,7 +73,7 @@ def all_projects(): data = [ p.name for p in keystone.projects.list(enabled=True) - if p.name != 'admin' + if p.name != "admin" ] cache.CACHE.save(key, data, 300) return data @@ -79,18 +81,20 @@ def all_projects(): def project_users_by_role(name): """Get a dict of lists of user ids indexed by role name.""" - key = 'keystone:project_users_by_role:{}'.format(name) + key = "keystone:project_users_by_role:{}".format(name) data = cache.CACHE.load(key) if data is None: keystone = keystone_client() # Ignore novaadmin & novaobserver in all user lists - seen = ['novaadmin', 'novaobserver'] + seen = ["novaadmin", "novaobserver"] data = {} for role_name, role_id in ROLES.items(): data[role_name] = [ - r.user['id'] for r in keystone.role_assignments.list( - project=name, role=role_id) - if r.user['id'] not in seen + r.user["id"] + for r in keystone.role_assignments.list( + project=name, role=role_id + ) + if r.user["id"] not in seen ] seen += data[role_name] cache.CACHE.save(key, data, 300) @@ -99,11 +103,10 @@ def project_users_by_role(name): def projects_for_user(uid): """Get a list of projects that a user belongs to.""" - key = 'keystone:projects_for_user:{}'.format(uid) + key = "keystone:projects_for_user:{}".format(uid) data = cache.CACHE.load(key) if data is None: keystone = keystone_client() - data = [ - p.name for p in keystone.projects.list(enabled=True, user=uid)] + data = [p.name for p in keystone.projects.list(enabled=True, user=uid)] cache.CACHE.save(key, data, 300) return data diff --git a/keystone_browser/ldap.py b/keystone_browser/ldap.py index 99a4ae2..aeac495 100644 --- a/keystone_browser/ldap.py +++ b/keystone_browser/ldap.py @@ -30,12 +30,16 @@ def ldap_conn(): Return value can be used as a context manager """ - servers = ldap3.ServerPool([ - ldap3.Server('ldap-labs.eqiad.wikimedia.org'), - ldap3.Server('ldap-labs.codfw.wikimedia.org'), - ], ldap3.ROUND_ROBIN, active=True, exhaust=True) - return ldap3.Connection( - servers, read_only=True, auto_bind=True) + servers = ldap3.ServerPool( + [ + ldap3.Server("ldap-labs.eqiad.wikimedia.org"), + ldap3.Server("ldap-labs.codfw.wikimedia.org"), + ], + ldap3.ROUND_ROBIN, + active=True, + exhaust=True, + ) + return ldap3.Connection(servers, read_only=True, auto_bind=True) def in_list(attr, items): @@ -47,52 +51,55 @@ def in_list(attr, items): >>> in_list('uid', ['a', 'b', 'c']) '(|(uid=a)(uid=b)(uid=c))' """ - return '(|{})'.format(''.join( - ['({}={})'.format(attr, item) for item in items] - )) + return "(|{})".format( + "".join(["({}={})".format(attr, item) for item in items]) + ) def get_users_by_uid(uids): """Get a list of dicts of user information.""" if not uids: return [] - key = 'ldap:get_users_by_uid:{}'.format( - hashlib.sha1('|'.join(uids).encode('utf-8')).hexdigest()) + key = "ldap:get_users_by_uid:{}".format( + hashlib.sha1("|".join(uids).encode("utf-8")).hexdigest() + ) data = cache.CACHE.load(key) if data is None: data = [] with ldap_conn() as conn: results = conn.extend.standard.paged_search( - 'ou=people,dc=wikimedia,dc=org', - in_list('uid', uids), + "ou=people,dc=wikimedia,dc=org", + in_list("uid", uids), ldap3.SUBTREE, - attributes=['uid', 'cn'], + attributes=["uid", "cn"], paged_size=1000, time_limit=5, generator=True, ) for resp in results: - attribs = resp.get('attributes') + attribs = resp.get("attributes") # LDAP attributes come back as a dict of lists. We know that # there is only one value for each list, so unwrap it - data.append({ - 'uid': attribs['uid'][0], - 'cn': attribs['cn'][0], - }) + data.append( + { + "uid": attribs["uid"][0], + "cn": attribs["cn"][0], + } + ) cache.CACHE.save(key, data, 3600) return data def user_count(): """Get the count of all users in LDAP.""" - key = 'ldap:user_count' + key = "ldap:user_count" total_entries = cache.CACHE.load(key) if total_entries is None: total_entries = 0 with ldap_conn() as conn: results = conn.extend.standard.paged_search( - 'ou=people,dc=wikimedia,dc=org', - '(objectclass=posixaccount)', + "ou=people,dc=wikimedia,dc=org", + "(objectclass=posixaccount)", ldap3.SUBTREE, attributes=None, paged_size=1000, diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index 698a01b..dd4ef1b 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -29,11 +29,11 @@ @functools.lru_cache(maxsize=None) def nova_client(project, region): return client.Client( - '2.12', + "2.12", session=keystone.session(project), - endpoint_type='public', + endpoint_type="public", timeout=2, - region_name=region + region_name=region, ) @@ -50,19 +50,22 @@ def project_servers(project): Data returned for each server is described at https://developer.openstack.org/api-ref/compute/?expanded=list-servers-detailed-detail#listServers """ - key = 'nova:project_servers:{}'.format(project) + key = "nova:project_servers:{}".format(project) data = cache.CACHE.load(key) if data is None: data = [] for region in get_regions(): nova = nova_client(project, region) - data.extend([ - s._info for s in nova.servers.list( - detailed=True, - sort_keys=['display_name'], - sort_dirs=['asc'], - ) - ]) + data.extend( + [ + s._info + for s in nova.servers.list( + detailed=True, + sort_keys=["display_name"], + sort_dirs=["asc"], + ) + ] + ) cache.CACHE.save(key, data, 300) return data @@ -70,14 +73,14 @@ def project_servers(project): def flavors(project): """Get a dict of flavor details indexed by id.""" - key = 'nova:flavors:{}'.format(project) + key = "nova:flavors:{}".format(project) data = cache.CACHE.load(key) if data is None: data = {} for region in get_regions(): nova = nova_client(project, region) for f in nova.flavors.list(): - data[f._info['id']] = f._info + data[f._info["id"]] = f._info cache.CACHE.save(key, data, 3600) return data @@ -85,7 +88,7 @@ def flavors(project): def limits(project): """Get a dict of limit details.""" - key = 'nova:limits:{}'.format(project) + key = "nova:limits:{}".format(project) data = cache.CACHE.load(key) if data is None: data = {} @@ -99,13 +102,13 @@ def limits(project): def all_servers(): """Get a list of all servers in all projects.""" - key = 'keystone:all_servers' + key = "keystone:all_servers" data = cache.CACHE.load(key) if data is None: data = [] all_projects = keystone.all_projects() for project in all_projects: - if project != 'admin': + if project != "admin": data += project_servers(project) cache.CACHE.save(key, data, 300) return data @@ -113,17 +116,17 @@ def all_servers(): def server(fqdn): """Get information about a server by fqdn.""" - key = 'nova:server:{}'.format(fqdn) + key = "nova:server:{}".format(fqdn) data = cache.CACHE.load(key) if data is None: - name, project, _ = fqdn.split('.', 2) + name, project, _ = fqdn.split(".", 2) servers = [] for region in get_regions(): nova = nova_client(project, region) reg_servers = nova.servers.list( detailed=True, search_opts={ - 'name': '^{}$'.format(name), + "name": "^{}$".format(name), }, ) if reg_servers: diff --git a/keystone_browser/proxies.py b/keystone_browser/proxies.py index 618a053..d5d6452 100644 --- a/keystone_browser/proxies.py +++ b/keystone_browser/proxies.py @@ -29,40 +29,41 @@ from . import utils -RE_BACKEND = re.compile(r'^https?://(?P[^:]+):(?P\d+)$') +RE_BACKEND = re.compile(r"^https?://(?P[^:]+):(?P\d+)$") @functools.lru_cache(maxsize=1) def url_template(): """Get the url template for accessing the proxy service.""" c = keystone.keystone_client() - proxy = c.services.list(type='proxy')[0] + proxy = c.services.list(type="proxy")[0] endpoint = c.endpoints.list( - service=proxy.id, interface='public', enabled=True)[0] + service=proxy.id, interface="public", enabled=True + )[0] # Secret magic! The endpoint provided by keystone is private and we can't # access it. There's an alternative public read-only endpoint on port 5669 # though. So, swap in 5669 for the port we got from keystone. - return re.sub(r':[0-9]+/', ':5669/', endpoint.url) + return re.sub(r":[0-9]+/", ":5669/", endpoint.url) def project_proxies(project, cached=True): """Get a list of proxies for a project.""" - key = 'proxies:{}'.format(project) + key = "proxies:{}".format(project) data = None if cached: data = cache.CACHE.load(key) if data is None: - base_url = url_template().replace('$(tenant_id)s', project) - url = '{}/mapping'.format(base_url) + base_url = url_template().replace("$(tenant_id)s", project) + url = "{}/mapping".format(base_url) req = requests.get(url, verify=False) if req.status_code != 200: data = [] else: - data = req.json()['routes'] + data = req.json()["routes"] # Some of the domain names have a . appended at the end, # strip them out so the URLs look right for route in data: - route['domain'] = route['domain'].rstrip('.') + route["domain"] = route["domain"].rstrip(".") cache.CACHE.save(key, data, 3600) return data @@ -73,7 +74,7 @@ def all_proxies(cached=True): Each proxy in the list will be a dict containing project, domain, and backends keys. """ - key = 'proxies:all' + key = "proxies:all" data = None if cached: data = cache.CACHE.load(key) @@ -92,10 +93,10 @@ def parse_backend(backend): """Parse a proxy backend specification.""" m = RE_BACKEND.match(backend) if not m: - return {'hostname': backend} + return {"hostname": backend} data = m.groupdict() - if utils.is_ipv4(data['host']): - data['hostname'] = socket.getfqdn(data['host']) + if utils.is_ipv4(data["host"]): + data["hostname"] = socket.getfqdn(data["host"]) else: - data['hostname'] = data['host'] + data["hostname"] = data["host"] return data diff --git a/keystone_browser/puppetclasses.py b/keystone_browser/puppetclasses.py index cd8c55a..3982a07 100644 --- a/keystone_browser/puppetclasses.py +++ b/keystone_browser/puppetclasses.py @@ -34,10 +34,9 @@ def url_template(): def prefixes(classname, cached=True): - """Return a dict of {: [prefixes]} for a given puppet class - """ + """Return a dict of {: [prefixes]} for a given puppet class""" - key = 'puppetprefixes:{}'.format(classname) + key = "puppetprefixes:{}".format(classname) data = None if cached: data = cache.CACHE.load(key) @@ -53,29 +52,27 @@ def prefixes(classname, cached=True): def all_classes(cached=True): - """Return a list of all used puppet classes - """ + """Return a list of all used puppet classes""" - key = 'all_puppetclasses' + key = "all_puppetclasses" data = None if cached: data = cache.CACHE.load(key) if data is None: - url = url_template() + '/roles' + url = url_template() + "/roles" req = requests.get(url, verify=False) if req.status_code != 200: data = [] else: data = yaml.safe_load(req.text) cache.CACHE.save(key, data, 1200) - return data['roles'] + return data["roles"] def project_prefixes(project, cached=True): - """Return a dict of [prefixes] for a given project - """ + """Return a dict of [prefixes] for a given project""" - key = 'puppetprojectprefixess:{}'.format(project) + key = "puppetprojectprefixess:{}".format(project) data = None if cached: data = cache.CACHE.load(key) @@ -87,21 +84,21 @@ def project_prefixes(project, cached=True): else: data = yaml.safe_load(req.text) cache.CACHE.save(key, data, 1200) - return data['prefixes'] + return data["prefixes"] def config(project, fqdn, cached=True): """Get full puppet config for a prefix. - Returns a dict with 'roles' and 'hiera' keys. + Returns a dict with 'roles' and 'hiera' keys. """ - key = 'puppetconfig:{}'.format(fqdn) + key = "puppetconfig:{}".format(fqdn) data = None if cached: data = cache.CACHE.load(key) if data is None: - url = url_template() + '/' + project + "/node/" + fqdn + url = url_template() + "/" + project + "/node/" + fqdn req = requests.get(url, verify=False) if req.status_code != 200: data = [] @@ -112,32 +109,30 @@ def config(project, fqdn, cached=True): def classes(project, fqdn, cached=True): - """Return a list of puppet classes for the given project and fqdn - """ - return config(project, fqdn, cached)['roles'] + """Return a list of puppet classes for the given project and fqdn""" + return config(project, fqdn, cached)["roles"] def hiera(project, fqdn, cached=True): - """Return a list of puppet classes for the given project and fqdn - """ + """Return a list of puppet classes for the given project and fqdn""" """Return a list of puppet classes for the given project and fqdn """ - return config(project, fqdn, cached)['hiera'] + return config(project, fqdn, cached)["hiera"] def giant_hiera_dict(cached=True): """Gather up the hiera config for every possible instance. - This is incredibly slow and expensive, but it'll get all - the caches warmed up! + This is incredibly slow and expensive, but it'll get all + the caches warmed up! - Make a dict of the form - {hiera_key: - {project_id: - {fqdn: hiera_value}}} + Make a dict of the form + {hiera_key: + {project_id: + {fqdn: hiera_value}}} """ - key = 'completehieradictt:' + key = "completehieradictt:" data = None if cached: data = cache.CACHE.load(key) @@ -158,6 +153,5 @@ def giant_hiera_dict(cached=True): def hieraprefixes(hierakey, cached=True): - """dict of {: {prefix: value}} for a given hiera key - """ + """dict of {: {prefix: value}} for a given hiera key""" return giant_hiera_dict(cached).get(hierakey, {}) diff --git a/keystone_browser/stats.py b/keystone_browser/stats.py index 4cd7578..eb60c90 100644 --- a/keystone_browser/stats.py +++ b/keystone_browser/stats.py @@ -27,28 +27,28 @@ def usage(cached=True): - key = 'stats:usage' + key = "stats:usage" data = None if cached: data = cache.CACHE.load(key) if data is None: - projects = [p for p in keystone.all_projects() if p != 'admin'] + projects = [p for p in keystone.all_projects() if p != "admin"] data = collections.defaultdict(int) - data['projects'] = len(projects) + data["projects"] = len(projects) for p in projects: types = collections.defaultdict(int) for s in nova.project_servers(p): - data['instances'] += 1 - types[s['flavor']['id']] += 1 + data["instances"] += 1 + types[s["flavor"]["id"]] += 1 for label, flavor in nova.flavors(p).items(): - data['ram'] += types[label] * flavor['ram'] - data['vcpus'] += types[label] * flavor['vcpus'] - data['disk'] += types[label] * flavor['disk'] + data["ram"] += types[label] * flavor["ram"] + data["vcpus"] += types[label] * flavor["vcpus"] + data["disk"] += types[label] * flavor["disk"] - data['users'] = ldap.user_count() - data['generated'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + data["users"] = ldap.user_count() + data["generated"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") # Cache for 25 hours cache.CACHE.save(key, data, 90000) return data diff --git a/keystone_browser/utils.py b/keystone_browser/utils.py index 6679648..46e7e36 100644 --- a/keystone_browser/utils.py +++ b/keystone_browser/utils.py @@ -21,8 +21,8 @@ import re RE_IPV4ADDR = re.compile( - r'^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}' - r'(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$' + r"^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}" + r"(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$" ) diff --git a/keystone_browser/zones.py b/keystone_browser/zones.py index 03bdefb..cd1d229 100644 --- a/keystone_browser/zones.py +++ b/keystone_browser/zones.py @@ -30,8 +30,7 @@ @functools.lru_cache(maxsize=None) def client(project): - return designate_client.Client( - session=keystone.session(project)) + return designate_client.Client(session=keystone.session(project)) @functools.lru_cache(maxsize=None) @@ -47,17 +46,17 @@ def _raw_zones(project): def zones(project): """Return a simple list of zones owned by a project.""" raw_zones = _raw_zones(project) - return [zone['name'] for zone in raw_zones] + return [zone["name"] for zone in raw_zones] @functools.lru_cache(maxsize=None) def _raw_recordsets(project, zone): """Return list of designate 'recordset' objects for a given - project and zone name. + project and zone name. """ for z in _raw_zones(project): - if z['name'] == zone: - return client(project).recordsets.list(z['id']) + if z["name"] == zone: + return client(project).recordsets.list(z["id"]) return [] @@ -69,7 +68,7 @@ def a_records(project, zone): https://developer.openstack.org/api-ref/dns/?expanded=list-all-recordsets-owned-by-project-detail """ raw_recordsets = _raw_recordsets(project, zone) - return [r for r in raw_recordsets if r['type'] == 'A'] + return [r for r in raw_recordsets if r["type"] == "A"] @functools.lru_cache(maxsize=None) @@ -91,8 +90,9 @@ def wmflabsdotorg_a_records(project): by a special 'wmflabsdotorg' project. """ return [ - r for r in a_records('wmflabsdotorg', 'wmflabs.org.') - if r['records'][0] in floating_ips(project) + r + for r in a_records("wmflabsdotorg", "wmflabs.org.") + if r["records"][0] in floating_ips(project) ] @@ -101,7 +101,7 @@ def all_a_records(project, cached=True): Returns a dict keyed by host with values being lists of ip addresses. """ - key = 'zones:A:{}'.format(project) + key = "zones:A:{}".format(project) data = None if cached: data = cache.CACHE.load(key) @@ -109,7 +109,7 @@ def all_a_records(project, cached=True): data = functools.reduce( operator.add, [a_records(project, zone) for zone in zones(project)], - [] + [], ) data += wmflabsdotorg_a_records(project) cache.CACHE.save(key, data, 3600) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7d2ace1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +line-length = 79 +target_version = ['py37'] +include = '\.pyi?$' diff --git a/tox.ini b/tox.ini index 140a05a..3f35be0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,16 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py3 +envlist = py37 [testenv] commands = flake8 nosetests --with-doctest + black --check --diff . deps = -r{toxinidir}/requirements.txt + black flake8 nose From ec02307236b7ba22d6620343e0f298aefe09db2f Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Tue, 8 Sep 2020 11:43:19 -0600 Subject: [PATCH 016/174] Include .cloud in puppet prefix checks Bug: T262293 --- app.py | 2 +- templates/hierakey.html | 2 +- templates/puppetclass.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 4105fcc..512257f 100644 --- a/app.py +++ b/app.py @@ -257,7 +257,7 @@ def api_dsh_puppet(name): servers = [] for prefix in d["prefixes"]: - if prefix.endswith("wmflabs"): + if prefix.endswith(".wmflabs") or prefix.endswith(".cloud"): dsh.append(prefix) else: dsh.extend( diff --git a/templates/hierakey.html b/templates/hierakey.html index 76ebd96..0d886c6 100644 --- a/templates/hierakey.html +++ b/templates/hierakey.html @@ -30,7 +30,7 @@

Project: {{ {% endif %} {% for prefix in prefixes.keys()|sort() %} {% if prefix != '_' %} - {% if prefix.endswith('wmflabs') %} + {% if prefix.endswith('.wmflabs') or prefix.endswith('.cloud') %}
  • {{ prefix }}
    • {% else %}
  • {{ prefix }}*
    • diff --git a/templates/puppetclass.html b/templates/puppetclass.html index e9cfc21..d5d04e3 100644 --- a/templates/puppetclass.html +++ b/templates/puppetclass.html @@ -30,7 +30,7 @@

    Project: {{ {% endif %} {% for prefix in prefixes.prefixes|sort() %} {% if prefix != '_' %} - {% if prefix.endswith('wmflabs') %} + {% if prefix.endswith('.wmflabs') or prefix.endswith('.cloud') %}
  • {{ prefix }}
    • {% else %}
  • {{ prefix }}*
    • From 551312aab977c5bbfe47d7058daf8d5e963c4671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Thu, 25 Mar 2021 18:51:51 +0200 Subject: [PATCH 017/174] Display Cinder volumes Summary: Display Cinder volume information in individual project pages. Closes T276569. Test Plan: Tested in the `majavah-test` Toolforge tool. Reviewers: bd808, aborrero, Majavah Reviewed By: Majavah Tags: PHID-PROJ-7sul7hj32btacinsnubq, #tool-openstack-browser Maniphest Tasks: T276569 Revert Plan: `git revert` Differential Revision: https://phabricator.wikimedia.org/D1190 --- app.py | 2 ++ keystone_browser/cinder.py | 64 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + templates/project.html | 44 ++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 keystone_browser/cinder.py diff --git a/app.py b/app.py index 512257f..c6417c0 100644 --- a/app.py +++ b/app.py @@ -30,6 +30,7 @@ from keystone_browser import proxies from keystone_browser import stats from keystone_browser import utils +from keystone_browser import cinder app = flask.Flask(__name__) @@ -99,6 +100,7 @@ def project(name): "proxies": proxies.project_proxies(name, cached), "zones": zones.all_a_records(name, cached), "limits": nova.limits(name), + "volumes": cinder.project_volumes(name, cached), } ) except Exception: diff --git a/keystone_browser/cinder.py b/keystone_browser/cinder.py new file mode 100644 index 0000000..b823a01 --- /dev/null +++ b/keystone_browser/cinder.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of the Keystone browser +# +# Copyright (c) 2021 Taavi Väänänen +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import functools + +from cinderclient import client + +from . import cache +from . import keystone + + +@functools.lru_cache(maxsize=None) +def cinder_client(project, region): + return client.Client( + version="3", + session=keystone.session(project), + timeout=2, + region_name=region, + ) + + +@functools.lru_cache() +def get_regions(): + ks_client = keystone.keystone_client() + region_recs = ks_client.regions.list() + return [region.id for region in region_recs] + + +def project_volumes(project, cached=True): + key = 'cinder:project-volumes:{}'.format(project) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = [] + for region in get_regions(): + cinder = cinder_client(project, region) + data.extend( + [ + volume._info + for volume in cinder.volumes.list( + detailed=True, + ) + ] + ) + cache.CACHE.save(key, data, 300) + return data diff --git a/requirements.txt b/requirements.txt index b6dcdff..6b4de3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ flask ldap3 +python-cinderclient python-glanceclient python-keystoneclient python-novaclient==7.1.0 # v8.0.0 dropped support for nova-network diff --git a/templates/project.html b/templates/project.html index 69bf0bd..997c153 100644 --- a/templates/project.html +++ b/templates/project.html @@ -158,6 +158,50 @@

    {% endif %} + {% if volumes %} +
    + + + + + + + + + + + {% for volume in volumes|sort(attribute='name') %} + + + + + + {% endfor %} + +
    NameSizeStatus
    {{ volume.name }}{{ volume.size|default('-') }}G + {% if volume.status == "in-use" %} + Attached to + {% for attachment in volume.attachments %} + {% for server in servers %} + {% if server.id == attachment.server_id %} + {% set fqdn = server.name ~ '.' ~ project ~ '.eqiad1.wikimedia.cloud' %} + {{ fqdn }} + {% endif %} + {% endfor %} + {% endfor %} + {% elif volume.status == "available" %} + Unattached + {% else %} + {{ volume.status }} + {% endif %} +
    +
    + {% endif %} {% if servers %}
    {% endif %} + {% if databases %} +
    + + + + + + + + + + + + + {% for database in databases|sort(attribute='name') %} + {% set flavor = flavors[database.flavor.id]|default('') %} + + + + + + + + {% endfor %} + +
    NameTypeSizeResourcesStatus
    {{ database.name }}{{ database.datastore.type }} {{ database.datastore.version }}{{ database.volume.size|default('-') }}G + {{ flavor.vcpus|default('-') }} CPUs, + {{ flavor.ram|default('-') }}M RAM + + {% if database.status == "HEALTHY" %} + Healthy + {% else %} + {{ database.status }} + {% endif %} +
    +
    + {% endif %} {% if servers %}
    + {% if domain_roles %} +
    +
    + + System-wide roles +
    +
    +
      + {% for role in domain_roles %} +
    • {{ role }}
      • + {% endfor %} +
    +
    +
    + {% endif %}
    From 9e5ec8af52db55f917a594f63a1c5b25675286f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 16 Apr 2022 09:35:49 +0300 Subject: [PATCH 046/174] Don't show an empty project list --- templates/user.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/user.html b/templates/user.html index c3a570e..6c0cb69 100644 --- a/templates/user.html +++ b/templates/user.html @@ -38,6 +38,7 @@

    User: {{ user.cn }}

    {% endif %} + {% if projects %}
    @@ -51,6 +52,7 @@

    User: {{ user.cn }}

    + {% endif %}
    {% else %}

    Unknown user '{{ uid }}'. Are you just guessing?

    From 0bfd2798e266216b6c2e98f5dc369d3945184462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 16 Apr 2022 09:40:23 +0300 Subject: [PATCH 047/174] reformat with black --- keystone_browser/keystone.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index aaa1aec..835b4fb 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -75,10 +75,7 @@ def all_projects(cached=True): data = cache.CACHE.load(key) if data is None: keystone = keystone_client() - data = [ - p.name - for p in keystone.projects.list(enabled=True) - ] + data = [p.name for p in keystone.projects.list(enabled=True)] cache.CACHE.save(key, data, 300) return data From 0737f6184f5e0dd4804e2fc0e921edbe9c932953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 16 Apr 2022 13:30:56 +0300 Subject: [PATCH 048/174] show service accounts in project --- app.py | 7 +++++++ keystone_browser/keystone.py | 5 +++++ templates/project.html | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/app.py b/app.py index f386e31..21f6c9a 100644 --- a/app.py +++ b/app.py @@ -101,11 +101,18 @@ def project(name): try: users = keystone.project_users_by_role(name) admins = users["admin"] + users["projectadmin"] + service_accounts = { + role: ldap.get_users_by_uid(users[members], cached) + for role, members in users.items() + if role in keystone.SERVICE_ACCOUNT_ROLES and len(members) > 0 + } + ctx.update( { "project": name, "admins": ldap.get_users_by_uid(admins, cached), "users": ldap.get_users_by_uid(users["user"], cached), + "service_accounts": service_accounts, "servers": nova.project_servers(name, cached), "flavors": nova.flavors(name, cached), "images": glance.images(cached), diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 835b4fb..4c37294 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -35,11 +35,16 @@ ("glanceadmin", "1102f4ff63c3435793d0e4340bf4b04e"), ("observer", "47a8370618ea42d49f7047774e75d262"), ("projectadmin", "4d8cad783d6342efa8414d7d36fbc034"), + ("designateadmin", "906f1588626d4d0993629ea3928b6fb4"), + ("keystonevalidate", "f3bebf5f4b6f40fa91f3614431f2c283"), ("user", "f473273fac7146b3bdbf22e5d4504f95"), ] ) +SERVICE_ACCOUNT_ROLES = ["glanceadmin", "designateadmin", "keystonevalidate"] + + @functools.lru_cache(maxsize=None) def session(project="observer"): """Get a session for the novaobserver user scoped to the given project.""" diff --git a/templates/project.html b/templates/project.html index bcfc30d..c9c515f 100644 --- a/templates/project.html +++ b/templates/project.html @@ -90,6 +90,32 @@

    {% endif %} + {% if service_accounts %} +
    + +
    +
      + {% for role, users in roles.items() %} +
      {{ role }}
      +
      +
        + {% for u in users|sort(attribute='cn') %} +
      • {{ u.cn }}
        • + {% endfor %} +
      +
      + {% endfor %} +
    +
    +
    + {% endif %} {% if proxies %}
    {% endif %} + {% if floating_ips %} +
    + + + + + + + + + + + + {% for floating_ip in floating_ips|sort(attribute='sortkey') %} + + + + + + + {% endfor %} + +
    AddressDescriptionMapped toDNS names
    {{ floating_ip.address }}{{ floating_ip.description }} + {% if floating_ip.target %} + {{ floating_ip.target }} + {% else %} + (unused) + {% endif %} + +
      + {% for name in floating_ip.dns|sort %} +
    • {{ name }}
      • + {% endfor %} +
    +
    +
    + {% endif %} {% if zones %}
    @@ -57,9 +71,17 @@

    {% for record in records|sort(attribute='sortkey') %} - {{ record.name }} - {% if record.status != "ACTIVE" %} + {% if record.name == "@" %} + {{ record.name }} + {% else %} + {{ record.name }} + {% endif %} + + + {% if record.status == "ACTIVE" %} + Active + {% else %} {{ record.status }} {% endif %} From f5f1b13edba52867de07d3be973a93feca20b9c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Tue, 4 Jul 2023 20:26:23 +0300 Subject: [PATCH 111/174] zones: monospace * (star) too --- templates/zone.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/zone.html b/templates/zone.html index b865df5..6c4e5dd 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -72,7 +72,7 @@

    {% for record in records|sort(attribute='sortkey') %} - {% if record.name == "@" %} + {% if record.name == "@" or record.name == "*" %} {{ record.name }} {% else %} {{ record.name }} From 162bee7f62868121c7929ffefd867b7cf456e7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Tue, 4 Jul 2023 20:26:23 +0300 Subject: [PATCH 112/174] zones: display almost all records --- keystone_browser/zones.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/keystone_browser/zones.py b/keystone_browser/zones.py index 48e2842..7d6644c 100644 --- a/keystone_browser/zones.py +++ b/keystone_browser/zones.py @@ -75,10 +75,7 @@ def records(project, zone_name, zone_id, cached): ), } for r in raw_recordsets - if ( - r["type"] in ["A", "AAAA", "CNAME", "PTR", "TXT"] - or (r["type"] == "NS" and r["name"] != zone_name) - ) + if (r["type"] not in ("NS", "SOA") or r["name"] != zone_name) ] cache.CACHE.save(key, data, 3600) return data From 8f0f878fd7b498169ed9fc8681ee2149c5f40e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 19 Jul 2023 18:59:56 +0300 Subject: [PATCH 113/174] project: link to alertmanager --- templates/project.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/project.html b/templates/project.html index 6513785..4ee450c 100644 --- a/templates/project.html +++ b/templates/project.html @@ -25,6 +25,7 @@

    From 5cdaaa37979c8dd6bec6874a4dcc136477689cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 19 Jul 2023 19:03:14 +0300 Subject: [PATCH 114/174] database: Improve status display --- templates/databaseinstance.html | 12 +++++++++++- templates/project.html | 10 ++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/templates/databaseinstance.html b/templates/databaseinstance.html index 55c0f20..c89dd08 100644 --- a/templates/databaseinstance.html +++ b/templates/databaseinstance.html @@ -36,7 +36,17 @@

    {{ instance.hostname }}
    Status
    - {{ instance.status }} {{ instance.operating_status }} + {% if instance.status == "ACTIVE" %} + Active, + {% else %} + {{ instance.status }}, + {% endif %} + + {% if instance.operating_status == "HEALTHY" %} + Healthy + {% else %} + {{ instance.operating_status }} + {% endif %}
    Type
    {{ instance.datastore.type }} {{ instance.datastore.version }}
    diff --git a/templates/project.html b/templates/project.html index 4ee450c..576a606 100644 --- a/templates/project.html +++ b/templates/project.html @@ -297,10 +297,16 @@

    {{ flavor.ram|default('-') }}M RAM - {% if database.status == "HEALTHY" %} + {% if database.status == "ACTIVE" %} + Active, + {% else %} + {{ database.status }}, + {% endif %} + + {% if database.operating_status == "HEALTHY" %} Healthy {% else %} - {{ database.status }} + {{ database.operating_status }} {% endif %} From 6ddfe1060c27b5fb42006745e54818a783faab95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 19 Jul 2023 19:07:35 +0300 Subject: [PATCH 115/174] proxies: Don't link .svc. addresses to instances --- app.py | 9 ++++++++- templates/project.html | 6 +++--- templates/proxies.html | 6 +++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index 4798d54..02ab21f 100644 --- a/app.py +++ b/app.py @@ -394,7 +394,14 @@ def contains(haystack, needle): @app.template_filter("extract_hostname") def extract_hostname(backend): """Extract a hostname from a backend description.""" - return proxies.parse_backend(backend).get("hostname", "404") + hostname = proxies.parse_backend(backend).get("hostname") + if not hostname: + return None + if not hostname.enswith(".wikimedia.cloud"): + return None + if ".svc." in hostname: + return None + return hostname @app.template_test() diff --git a/templates/project.html b/templates/project.html index 576a606..bb44afb 100644 --- a/templates/project.html +++ b/templates/project.html @@ -140,10 +140,10 @@

    {% for backend in proxy.backends %} {% with fqdn=backend|extract_hostname %} - {% if fqdn is ipv4addr %} - UNKNOWN - {% else %} + {% if fqdn %} {{ fqdn }} + {% else %} + Unknown {% endif %} {% endwith %} {% endfor %} diff --git a/templates/proxies.html b/templates/proxies.html index 5afdc8b..3d83e9c 100644 --- a/templates/proxies.html +++ b/templates/proxies.html @@ -33,10 +33,10 @@

    {% for backend in proxy.backends %} {% with fqdn=backend|extract_hostname %} - {% if fqdn is ipv4addr %} - UNKNOWN - {% else %} + {% if fqdn %} {{ fqdn }} + {% else %} + Unknown {% endif %} {% endwith %} {% endfor %} From 95277cc7befca43526c281bcb80adb493698f916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 19 Jul 2023 19:14:44 +0300 Subject: [PATCH 116/174] app: fix typo --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 02ab21f..da57abf 100644 --- a/app.py +++ b/app.py @@ -397,7 +397,7 @@ def extract_hostname(backend): hostname = proxies.parse_backend(backend).get("hostname") if not hostname: return None - if not hostname.enswith(".wikimedia.cloud"): + if not hostname.endswith(".wikimedia.cloud"): return None if ".svc." in hostname: return None From f1a07fd83aa9afa7c748628fc228da7bb3ecaba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Fri, 10 Nov 2023 17:05:05 +0200 Subject: [PATCH 117/174] display project data --- app.py | 1 + keystone_browser/keystone.py | 18 ++++++++++++++++++ templates/project.html | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/app.py b/app.py index da57abf..9a4a6e8 100644 --- a/app.py +++ b/app.py @@ -122,6 +122,7 @@ def project(name): ctx.update( { "project": name, + "data": keystone.project_data(name, cached), "members": ldap.get_users_by_uid(members, cached), "viewers": ldap.get_users_by_uid(viewers, cached), "service_accounts": { diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index d985cc3..a07f199 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -90,6 +90,24 @@ def all_projects(cached=True): return data +def project_data(project_id, cached=True): + key = "keystone:project:{}".format(project_id) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + keystone = keystone_client() + project = keystone.projects.get(project_id) + if project: + data = { + "id": project.id, + "name": project.name, + "description": project.description, + } + cache.CACHE.save(key, data, 300) + return data + + def project_users_by_role(name, cached=True): """Get a dict of lists of user ids indexed by role name.""" key = "keystone:project_users_by_role:{}".format(name) diff --git a/templates/project.html b/templates/project.html index bb44afb..794922b 100644 --- a/templates/project.html +++ b/templates/project.html @@ -14,6 +14,24 @@

    Project: +
    + +
    +
    +
    ID
    +
    {{ data.id }}
    +
    Name
    +
    {{ data.name }}
    +
    Description
    +
    {{ data.description }}
    +
    +
    +
    From 4249330f6a2c21c55b3c1dc1ad1412656cfc4375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Tue, 30 Jan 2024 15:04:40 +0200 Subject: [PATCH 119/174] templates: Fix zone status variable --- templates/zone.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/zone.html b/templates/zone.html index 6c4e5dd..4376a88 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -44,7 +44,7 @@

    {% if zone.status == "ACTIVE" %} Active {% else %} - {{ record.status }} + {{ zone.status }} {% endif %} From 42685b723c12f8011f0209d1f2dd5c7967fb7b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 20 Mar 2024 18:14:26 +0200 Subject: [PATCH 120/174] Add link to phab --- templates/layout.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/layout.html b/templates/layout.html index c2d6fcf..842e9e5 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -75,6 +75,7 @@
    view source + report issues
    From bee453a4ee40077df1e084ac73d43952850d7c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 24 Apr 2024 13:49:42 +0300 Subject: [PATCH 121/174] zone: Handle URLs with trailing dot cut off --- app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app.py b/app.py index 9a4a6e8..cda95f8 100644 --- a/app.py +++ b/app.py @@ -175,6 +175,11 @@ def project_database(project, name): @app.route("/project//zone/") def zone(project, name): + if not name.endswith("."): + return flask.redirect( + flask.url_for("zone", project=project, name=f"{name}.") + ) + cached = "purge" not in flask.request.args ctx = { "project": project, From 550852440aff2d86d19c7a2c3ed0a4a8082537ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 24 Apr 2024 13:54:31 +0300 Subject: [PATCH 122/174] templates: Fix typo --- templates/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/layout.html b/templates/layout.html index 842e9e5..7cfc2cb 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -75,7 +75,7 @@
    view source - report issues + report issues
    From fb9d3e96ff3697c6d774a6c600c0d9bbb9c644cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:01:13 +0300 Subject: [PATCH 123/174] Support projects where id != name --- app.py | 35 +++++++++++++++---------------- keystone_browser/keystone.py | 8 +++---- keystone_browser/proxies.py | 2 +- keystone_browser/puppetclasses.py | 2 +- keystone_browser/stats.py | 2 +- templates/databaseinstance.html | 4 ++-- templates/hierakey.html | 2 +- templates/project.html | 22 +++++++++---------- templates/projects.html | 6 +++--- templates/proxies.html | 2 +- templates/puppetclass.html | 2 +- templates/server.html | 4 ++-- templates/user.html | 2 +- templates/zone.html | 4 ++-- 14 files changed, 48 insertions(+), 49 deletions(-) diff --git a/app.py b/app.py index cda95f8..af77b38 100644 --- a/app.py +++ b/app.py @@ -99,14 +99,14 @@ def servers(): return flask.render_template("servers.html", **ctx) -@app.route("/project/") -def project(name): +@app.route("/project/") +def project(project_id): cached = "purge" not in flask.request.args ctx = { - "project": name, + "project_id": project_id, } try: - users = keystone.project_users_by_role(name, cached) + users = keystone.project_users_by_role(project_id, cached) # Create exclusive sets of users based on descending order of "power". # member > service accounts > viewers members = set(users["admin"]) | set(users["member"]) @@ -121,25 +121,24 @@ def project(name): ctx.update( { - "project": name, - "data": keystone.project_data(name, cached), + "data": keystone.project_data(project_id, cached), "members": ldap.get_users_by_uid(members, cached), "viewers": ldap.get_users_by_uid(viewers, cached), "service_accounts": { role: ldap.get_users_by_uid(uids, cached) for role, uids in service_accounts.items() }, - "servers": nova.project_servers(name, cached), - "flavors": nova.flavors(name, cached), + "servers": nova.project_servers(project_id, cached), + "flavors": nova.flavors(project_id, cached), "images": glance.images(cached), - "proxies": proxies.project_proxies(name, cached), - "zones": zones.all_dns_zones(name, cached), - "limits": nova.limits(name, cached), - "volumes": cinder.project_volumes(name, cached), - "cinder_limits": cinder.limits(name, cached), - "neutron_limits": neutron.limits(name, cached), - "databases": trove.project_instances(name, cached), - "floating_ips": neutron.floating_ips(name, cached), + "proxies": proxies.project_proxies(project_id, cached), + "zones": zones.all_dns_zones(project_id, cached), + "limits": nova.limits(project_id, cached), + "volumes": cinder.project_volumes(project_id, cached), + "cinder_limits": cinder.limits(project_id, cached), + "neutron_limits": neutron.limits(project_id, cached), + "databases": trove.project_instances(project_id, cached), + "floating_ips": neutron.floating_ips(project_id, cached), } ) except Exception: @@ -311,14 +310,14 @@ def all_proxies(): @app.route("/api/projects.json") def api_projects_json(): cached = "purge" not in flask.request.args - return flask.jsonify(projects=keystone.all_projects(cached)) + return flask.jsonify(projects=keystone.all_projects(cached).keys()) @app.route("/api/projects.txt") def api_projects_txt(): cached = "purge" not in flask.request.args return flask.Response( - "\n".join(sorted(keystone.all_projects(cached))), mimetype="text/plain" + "\n".join(sorted(keystone.all_projects(cached).keys())), mimetype="text/plain" ) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index a07f199..4ed9b1b 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -76,16 +76,16 @@ def keystone_client(): def all_projects(cached=True): """Get a list of all project names.""" - key = "keystone:all_projects" + key = "keystone:projects_by_id" data = None if cached: data = cache.CACHE.load(key) if data is None: keystone = keystone_client() - data = [ - p.name + data = { + p.id: p.name for p in keystone.projects.list(enabled=True, domain="default") - ] + } cache.CACHE.save(key, data, 300) return data diff --git a/keystone_browser/proxies.py b/keystone_browser/proxies.py index 234270a..e83b053 100644 --- a/keystone_browser/proxies.py +++ b/keystone_browser/proxies.py @@ -82,7 +82,7 @@ def all_proxies(cached=True): if data is None: data = [ dict(project=project, **proxy) - for project in keystone.all_projects() + for project in keystone.all_projects().keys() for proxy in project_proxies(project, cached) ] cache.CACHE.save(key, data, 3600) diff --git a/keystone_browser/puppetclasses.py b/keystone_browser/puppetclasses.py index 98b6fb3..ef7205f 100644 --- a/keystone_browser/puppetclasses.py +++ b/keystone_browser/puppetclasses.py @@ -165,7 +165,7 @@ def giant_hiera_dict(cached=True): data = cache.CACHE.load(key) if data is None: data = {} - for project in keystone.all_projects(): + for project in keystone.all_projects().keys(): for prefix in project_prefixes(project): hieradata = hiera(project, prefix, cached) for key in hieradata.keys(): diff --git a/keystone_browser/stats.py b/keystone_browser/stats.py index 9d18918..54cec49 100644 --- a/keystone_browser/stats.py +++ b/keystone_browser/stats.py @@ -33,7 +33,7 @@ def usage(cached=True): if cached: data = cache.CACHE.load(key) if data is None: - projects = [p for p in keystone.all_projects() if p != "admin"] + projects = [p for p in keystone.all_projects().keys() if p != "admin"] data = collections.defaultdict(int) data["projects"] = len(projects) diff --git a/templates/databaseinstance.html b/templates/databaseinstance.html index c89dd08..f43afc8 100644 --- a/templates/databaseinstance.html +++ b/templates/databaseinstance.html @@ -5,7 +5,7 @@ {% block content %}
    1. Projects
      2. -
    2. {{ project }}
      3. +
    3. {{ project }}
    4. Databases
    5. {{ name }}
    @@ -27,7 +27,7 @@

    Project
    -
    {{ project }}
    +
    {{ project }}
    Instance name
    {{ instance.name }}
    Instance ID
    diff --git a/templates/hierakey.html b/templates/hierakey.html index b4b9577..f67cfab 100644 --- a/templates/hierakey.html +++ b/templates/hierakey.html @@ -23,7 +23,7 @@

    {% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

    Project: {{ project }}

    +

    Project: {{ project }}

      {% if '_' in prefixes.keys() %}
    • All project instances
      • diff --git a/templates/project.html b/templates/project.html index e4ae8ed..9809e64 100644 --- a/templates/project.html +++ b/templates/project.html @@ -1,15 +1,15 @@ {% extends "layout.html" %} -{% block title %}Project {{ project }} - {{ super() }}{% endblock %} +{% block title %}Project {{ data.name }} - {{ super() }}{% endblock %} {% block content %}
      1. Projects
        2. -
      2. {{ project }}
        3. +
      3. {{ data.name }}
      -

      Project: {{ project|capitalize }}

      +

      Project: {{ data.name }}

      {% if admins or users or servers or proxies or zones %} @@ -43,9 +43,9 @@

    @@ -236,7 +236,7 @@

    {% for zone in zones|sort %} - {{ zone }} + {{ zone }} {% endfor %} @@ -270,7 +270,7 @@

    {% for attachment in volume.attachments %} {% for server in servers %} {% if server.id == attachment.server_id %} - {% set fqdn = server.name ~ '.' ~ project ~ '.eqiad1.wikimedia.cloud' %} + {% set fqdn = server.name ~ '.' ~ data.id ~ '.eqiad1.wikimedia.cloud' %} {{ fqdn }} {% endif %} {% endfor %} @@ -309,7 +309,7 @@

    {% for database in databases|sort(attribute='name') %} {% set flavor = flavors[database.flavor.id]|default('') %} - {{ database.name }} + {{ database.name }} {{ database.datastore.type }} {{ database.datastore.version }} {{ database.volume.size|default('-') }}G @@ -361,7 +361,7 @@

    {% for server in servers %} {% set image = images[server.image.id]|default('') %} {% set flavor = flavors[server.flavor.id]|default('') %} - {% set fqdn = server.name ~ '.' ~ project ~ '.eqiad1.wikimedia.cloud' %} + {% set fqdn = server.name ~ '.' ~ data.id ~ '.eqiad1.wikimedia.cloud' %} {{ fqdn }} {{ flavor.name|default('UNKNOWN') }} @@ -386,7 +386,7 @@

    {% endif %} {% else %} -

    Unknown project '{{ project }}'. Are you just guessing?

    +

    Unknown project '{{ project_id }}'. Are you just guessing?

    {% endif %} {% endblock %} diff --git a/templates/projects.html b/templates/projects.html index d227e97..b1e50c2 100644 --- a/templates/projects.html +++ b/templates/projects.html @@ -9,9 +9,9 @@
      - {% for project in projects %} - {% if project != 'admin' %} -
    • {{ project }}
      • + {% for project_id, project_name in projects %} + {% if project_id != 'admin' %} +
    • {{ project_name }}
      • {% endif %} {% endfor %}
    diff --git a/templates/proxies.html b/templates/proxies.html index 3d83e9c..f5a288c 100644 --- a/templates/proxies.html +++ b/templates/proxies.html @@ -23,7 +23,7 @@

    {% for proxy in proxies|sort(attribute='project') %} - {{ proxy.project }} + {{ proxy.project }} {{ proxy.domain }} {% for backend in proxy.backends %} diff --git a/templates/puppetclass.html b/templates/puppetclass.html index 04dc7e2..5c06c10 100644 --- a/templates/puppetclass.html +++ b/templates/puppetclass.html @@ -23,7 +23,7 @@

    {% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

    Project: {{ project }}

    +

    Project: {{ project }}

      {% if '_' in prefixes.prefixes %}
    • All project instances
      • diff --git a/templates/server.html b/templates/server.html index c30270d..638c788 100644 --- a/templates/server.html +++ b/templates/server.html @@ -5,7 +5,7 @@ {% block content %}
      1. Projects
        2. -
      2. {{ project }}
        3. +
      3. {{ project }}
      4. {{ fqdn }}
      @@ -27,7 +27,7 @@

      Project
      -
      {{ server.tenant_id }}
      +
      {{ server.tenant_id }}
      Instance name
      {{ server.name }}
      Instance Id
      diff --git a/templates/user.html b/templates/user.html index 6c0cb69..821a9d5 100644 --- a/templates/user.html +++ b/templates/user.html @@ -47,7 +47,7 @@

      User: {{ user.cn }}

      diff --git a/templates/zone.html b/templates/zone.html index 4376a88..5f414e5 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -5,7 +5,7 @@ {% block content %}
      1. Projects
        2. -
      2. {{ project }}
        3. +
      3. {{ project }}
      4. DNS zones
      5. {{ name }}
      @@ -26,7 +26,7 @@

      Project
      -
      {{ project }}
      +
      {{ project }}
      Zone ID
      {{ zone.id }}
      Type
      From 99bab86244a870319e4507580837c57ed796e468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:04:15 +0300 Subject: [PATCH 124/174] build: Fix CI include (and bump to Bookworm) --- .gitlab-ci.yml | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56ce3f8..092dcee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,2 +1,3 @@ include: - - https://gitlab.wikimedia.org/repos/cloud/cicd/gitlab-ci/-/raw/main/py3.9-bullseye-tox/gitlab-ci.yaml \ No newline at end of file + - project: "repos/cloud/cicd/gitlab-ci" + file: "py3.11-bookworm-tox/gitlab-ci.yaml" diff --git a/pyproject.toml b/pyproject.toml index 2813dfc..4477e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ [tool.black] line-length = 79 -target_version = ['py39'] +target_version = ['py311'] include = '\.pyi?$' From f53b2a45f891e26c85e8b68e279fe8bd445a2d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:06:03 +0300 Subject: [PATCH 125/174] build: Run black first Since it has an autofixer. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3e1debe..39d2003 100644 --- a/tox.ini +++ b/tox.ini @@ -5,8 +5,8 @@ envlist = py3 [testenv] commands = - flake8 black --check --diff . + flake8 deps = -r{toxinidir}/requirements.txt black From d5ce99da897fd2934dfa95a076a61c8d9defe85a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:06:16 +0300 Subject: [PATCH 126/174] templates: Add missing .items() --- templates/projects.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/projects.html b/templates/projects.html index b1e50c2..b64e515 100644 --- a/templates/projects.html +++ b/templates/projects.html @@ -9,7 +9,7 @@
        - {% for project_id, project_name in projects %} + {% for project_id, project_name in projects.items() %} {% if project_id != 'admin' %}
      • {{ project_name }}
        • {% endif %} From 6fd66c03078c8bddadde2b43646ee97edcbb5396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:06:30 +0300 Subject: [PATCH 127/174] app: Fix variable name and style --- app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index af77b38..eb042e7 100644 --- a/app.py +++ b/app.py @@ -143,7 +143,7 @@ def project(project_id): ) except Exception: app.logger.exception( - 'Error collecting information for project "%s"', name + 'Error collecting information for project "%s"', project_id ) return flask.render_template("project.html", **ctx) @@ -317,7 +317,8 @@ def api_projects_json(): def api_projects_txt(): cached = "purge" not in flask.request.args return flask.Response( - "\n".join(sorted(keystone.all_projects(cached).keys())), mimetype="text/plain" + "\n".join(sorted(keystone.all_projects(cached).keys())), + mimetype="text/plain", ) From 11c9067fe50ec54f16574ee5097bf75967e16bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:11:14 +0300 Subject: [PATCH 128/174] templates: Sort projects by name --- templates/projects.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/projects.html b/templates/projects.html index b64e515..d7b11c8 100644 --- a/templates/projects.html +++ b/templates/projects.html @@ -9,7 +9,7 @@
        - {% for project_id, project_name in projects.items() %} + {% for project_id, project_name in projects|dictsort(by='value') %} {% if project_id != 'admin' %}
      • {{ project_name }}
        • {% endif %} From 0a92113746fa6a8eef3e1229c7b1944f7f3f2dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:14:26 +0300 Subject: [PATCH 129/174] Show project names in user project list --- keystone_browser/keystone.py | 11 ++++++++--- templates/user.html | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 4ed9b1b..e92877f 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -133,7 +133,7 @@ def project_users_by_role(name, cached=True): def roles_for_user(uid, cached=True): """Get a list of projects that a user belongs to.""" - key = "keystone:roles_for_user:{}".format(uid) + key = "keystone:roles_for_user:{}:v2".format(uid) data = None if cached: data = cache.CACHE.load(key) @@ -144,13 +144,18 @@ def roles_for_user(uid, cached=True): for assignment in keystone.role_assignments.list(user=uid): if "project" in assignment.scope: - projects.add(assignment.scope["project"]["id"]) + projects.add( + ( + assignment.scope["project"]["id"], + assignment.scope["project"]["name"], + ) + ) elif "domain" in assignment.scope: role_name = keystone.roles.get(assignment.role["id"]).name domain_roles.add(role_name) data = { - "projects": sorted(list(projects)), + "projects": sorted(list(projects), key=lambda project: project[1]), "domain_roles": sorted(list(domain_roles)), } diff --git a/templates/user.html b/templates/user.html index 821a9d5..36b1450 100644 --- a/templates/user.html +++ b/templates/user.html @@ -46,8 +46,8 @@

        User: {{ user.cn }}

      From e3f3896b97176bc2db66df15f77458e81e844387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 5 Jun 2024 12:19:16 +0300 Subject: [PATCH 130/174] keystone: Fix user project loading --- keystone_browser/keystone.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index e92877f..a882cb8 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -147,7 +147,10 @@ def roles_for_user(uid, cached=True): projects.add( ( assignment.scope["project"]["id"], - assignment.scope["project"]["name"], + # TODO: replace with project name, + # just assignment.scope["project"]["name"], + # does not work sadly + assignment.scope["project"]["id"], ) ) elif "domain" in assignment.scope: From 4656287adfebcde8884218c324289d64d9b682c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 31 Aug 2024 17:44:37 +0300 Subject: [PATCH 131/174] app: Fix "Object of type dict_keys is not JSON serializable" error Bug: T373742 --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index eb042e7..c8978a8 100644 --- a/app.py +++ b/app.py @@ -310,7 +310,7 @@ def all_proxies(): @app.route("/api/projects.json") def api_projects_json(): cached = "purge" not in flask.request.args - return flask.jsonify(projects=keystone.all_projects(cached).keys()) + return flask.jsonify(projects=list(sorted(keystone.all_projects(cached).keys()))) @app.route("/api/projects.txt") From a8c076ab4dfb6a95173f2464e0fa7d9548368657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 31 Aug 2024 17:46:26 +0300 Subject: [PATCH 132/174] app: Format with black --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index c8978a8..fe50097 100644 --- a/app.py +++ b/app.py @@ -310,7 +310,9 @@ def all_proxies(): @app.route("/api/projects.json") def api_projects_json(): cached = "purge" not in flask.request.args - return flask.jsonify(projects=list(sorted(keystone.all_projects(cached).keys()))) + return flask.jsonify( + projects=list(sorted(keystone.all_projects(cached).keys())) + ) @app.route("/api/projects.txt") From 9db1eab9557faf0bcfe4d023edb0cdeb9b77bff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 31 Aug 2024 17:51:36 +0300 Subject: [PATCH 133/174] app: Add API endpoint for project names --- app.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app.py b/app.py index fe50097..9ac9941 100644 --- a/app.py +++ b/app.py @@ -324,6 +324,14 @@ def api_projects_txt(): ) +@app.route("/api/project-names.json") +def api_project_names_json(): + cached = "purge" not in flask.request.args + return flask.jsonify( + projects=dict(sorted(keystone.all_projects(cached))) + ) + + @app.route("/api/dsh/project/") def api_dsh_project(name): cached = "purge" not in flask.request.args From 1c23e105c9f9f201a422feb9b79bf54ce31a56df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 31 Aug 2024 17:54:37 +0300 Subject: [PATCH 134/174] app: Try to Python Not being able to test this is hard.. --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 9ac9941..e47a280 100644 --- a/app.py +++ b/app.py @@ -328,7 +328,7 @@ def api_projects_txt(): def api_project_names_json(): cached = "purge" not in flask.request.args return flask.jsonify( - projects=dict(sorted(keystone.all_projects(cached))) + projects=sorted(keystone.all_projects(cached)) ) From 734e7fdb07c622c399501fad2f1f3874014aa5b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 31 Aug 2024 17:57:01 +0300 Subject: [PATCH 135/174] app: Don't turn it into a list --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index e47a280..44896f0 100644 --- a/app.py +++ b/app.py @@ -328,7 +328,7 @@ def api_projects_txt(): def api_project_names_json(): cached = "purge" not in flask.request.args return flask.jsonify( - projects=sorted(keystone.all_projects(cached)) + projects=keystone.all_projects(cached) ) From 0c2df0ef151cefa11186cbb53e50f8a40a91338d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Mon, 30 Sep 2024 21:05:09 +0300 Subject: [PATCH 136/174] Format with black --- app.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app.py b/app.py index 44896f0..d1a24a5 100644 --- a/app.py +++ b/app.py @@ -327,9 +327,7 @@ def api_projects_txt(): @app.route("/api/project-names.json") def api_project_names_json(): cached = "purge" not in flask.request.args - return flask.jsonify( - projects=keystone.all_projects(cached) - ) + return flask.jsonify(projects=keystone.all_projects(cached)) @app.route("/api/dsh/project/") From fdc71f8fcfdecf0711d9d8a04e5ac69bafe623dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Mon, 30 Sep 2024 21:04:33 +0300 Subject: [PATCH 137/174] keystone: Fix error page for non-existent projects --- app.py | 72 +++++++++++++++++++----------------- keystone_browser/keystone.py | 7 +++- templates/project.html | 10 ++--- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/app.py b/app.py index d1a24a5..a6ef989 100644 --- a/app.py +++ b/app.py @@ -104,43 +104,47 @@ def project(project_id): cached = "purge" not in flask.request.args ctx = { "project_id": project_id, + "project_name": project_id, } try: - users = keystone.project_users_by_role(project_id, cached) - # Create exclusive sets of users based on descending order of "power". - # member > service accounts > viewers - members = set(users["admin"]) | set(users["member"]) - service_accounts = { - role: set(uids) - members - for role, uids in users.items() - if role in keystone.SERVICE_ACCOUNT_ROLES and len(uids) > 0 - } - viewers = set(users["reader"]) - members - for uids in service_accounts.values(): - viewers = viewers - uids - - ctx.update( - { - "data": keystone.project_data(project_id, cached), - "members": ldap.get_users_by_uid(members, cached), - "viewers": ldap.get_users_by_uid(viewers, cached), - "service_accounts": { - role: ldap.get_users_by_uid(uids, cached) - for role, uids in service_accounts.items() - }, - "servers": nova.project_servers(project_id, cached), - "flavors": nova.flavors(project_id, cached), - "images": glance.images(cached), - "proxies": proxies.project_proxies(project_id, cached), - "zones": zones.all_dns_zones(project_id, cached), - "limits": nova.limits(project_id, cached), - "volumes": cinder.project_volumes(project_id, cached), - "cinder_limits": cinder.limits(project_id, cached), - "neutron_limits": neutron.limits(project_id, cached), - "databases": trove.project_instances(project_id, cached), - "floating_ips": neutron.floating_ips(project_id, cached), + project_data = keystone.project_data(project_id, cached) + if project_data: + users = keystone.project_users_by_role(project_id, cached) + # Create exclusive sets of users based on descending order of "power". + # member > service accounts > viewers + members = set(users["admin"]) | set(users["member"]) + service_accounts = { + role: set(uids) - members + for role, uids in users.items() + if role in keystone.SERVICE_ACCOUNT_ROLES and len(uids) > 0 } - ) + viewers = set(users["reader"]) - members + for uids in service_accounts.values(): + viewers = viewers - uids + + ctx.update( + { + "data": project_data, + "project_name": project_data["name"], + "members": ldap.get_users_by_uid(members, cached), + "viewers": ldap.get_users_by_uid(viewers, cached), + "service_accounts": { + role: ldap.get_users_by_uid(uids, cached) + for role, uids in service_accounts.items() + }, + "servers": nova.project_servers(project_id, cached), + "flavors": nova.flavors(project_id, cached), + "images": glance.images(cached), + "proxies": proxies.project_proxies(project_id, cached), + "zones": zones.all_dns_zones(project_id, cached), + "limits": nova.limits(project_id, cached), + "volumes": cinder.project_volumes(project_id, cached), + "cinder_limits": cinder.limits(project_id, cached), + "neutron_limits": neutron.limits(project_id, cached), + "databases": trove.project_instances(project_id, cached), + "floating_ips": neutron.floating_ips(project_id, cached), + } + ) except Exception: app.logger.exception( 'Error collecting information for project "%s"', project_id diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index a882cb8..603c2bb 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -97,13 +97,16 @@ def project_data(project_id, cached=True): data = cache.CACHE.load(key) if data is None: keystone = keystone_client() - project = keystone.projects.get(project_id) - if project: + data = False + try: + project = keystone.projects.get(project_id) data = { "id": project.id, "name": project.name, "description": project.description, } + except keystoneauth1.exceptions.http.NotFound: + pass cache.CACHE.save(key, data, 300) return data diff --git a/templates/project.html b/templates/project.html index 9809e64..c714e3d 100644 --- a/templates/project.html +++ b/templates/project.html @@ -1,18 +1,18 @@ {% extends "layout.html" %} -{% block title %}Project {{ data.name }} - {{ super() }}{% endblock %} +{% block title %}Project {{ project_name }} - {{ super() }}{% endblock %} {% block content %}
      1. Projects
        2. -
      2. {{ data.name }}
        3. +
      3. {{ project_name }}
      -

      Project: {{ data.name }}

      +

      Project: {{ data.name }}

      -{% if admins or users or servers or proxies or zones %} +{% if data %}
      {% else %} -

      Unknown project '{{ project_id }}'. Are you just guessing?

      +

      Unknown project '{{ project_name }}'. Are you just guessing?

      {% endif %} {% endblock %} From 26e175092f697c58bfeccce01c8312535b6bba68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Mon, 30 Sep 2024 21:08:27 +0300 Subject: [PATCH 138/174] Fix one more spot --- templates/project.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/project.html b/templates/project.html index c714e3d..1c3821b 100644 --- a/templates/project.html +++ b/templates/project.html @@ -9,7 +9,7 @@
      -

      Project: {{ data.name }}

      +

      Project: {{ project_name }}

      {% if data %} From 40d20b9170068f2211bbe088b462e822e62f25bc Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 14:51:56 -0600 Subject: [PATCH 139/174] Fix a dangling lib reference --- keystone_browser/keystone.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 603c2bb..e5284f5 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -24,6 +24,7 @@ from keystoneauth1 import session as keystone_session from keystoneauth1.identity import v3 +from keystoneauth1 import exceptions from keystoneclient.v3 import client from . import cache @@ -105,7 +106,7 @@ def project_data(project_id, cached=True): "name": project.name, "description": project.description, } - except keystoneauth1.exceptions.http.NotFound: + except exceptions.http.NotFound: pass cache.CACHE.save(key, data, 300) return data From 786758a7a3bf346136a2ef7ab6c9ac80d1d89698 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 14:10:24 -0600 Subject: [PATCH 140/174] Project page: Show project name up top, then ID Bug: T366679 --- templates/project.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/project.html b/templates/project.html index 1c3821b..963c31c 100644 --- a/templates/project.html +++ b/templates/project.html @@ -23,10 +23,10 @@

      -
      ID
      -
      {{ data.id }}
      Name
      {{ data.name }}
      +
      ID
      +
      {{ data.id }}
      {% if data.description %}
      Description
      {{ data.description }}
      From e699fec6a493793b62d31783b0d1cee90968d1dd Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 14:16:53 -0600 Subject: [PATCH 141/174] roles_for_user: include project name in project records This should allow us to display project names in the user project memberships page. Bug: T366679 --- keystone_browser/keystone.py | 11 +++++++---- keystone_browser/nova.py | 5 +++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index e5284f5..a76e053 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -112,6 +112,10 @@ def project_data(project_id, cached=True): return data +def project_name_for_id(id, cached=True): + return project_data(id, cached=cached)["name"] + + def project_users_by_role(name, cached=True): """Get a dict of lists of user ids indexed by role name.""" key = "keystone:project_users_by_role:{}".format(name) @@ -151,10 +155,9 @@ def roles_for_user(uid, cached=True): projects.add( ( assignment.scope["project"]["id"], - # TODO: replace with project name, - # just assignment.scope["project"]["name"], - # does not work sadly - assignment.scope["project"]["id"], + project_name_for_id( + assignment.scope["project"]["id"], cached=cached + ), ) ) elif "domain" in assignment.scope: diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index a52d494..e33a0a9 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -76,6 +76,11 @@ def project_servers(project, cached=True): ) ) + for server in data: + server["project_name"] = keystone.project_name_for_id( + server["project_id"] + ) + cache.CACHE.save(key, data, 300) return data From 51915432d4cd919a493f55ccd6a5bfef906ffdd7 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 14:44:21 -0600 Subject: [PATCH 142/174] Display server fqdns with the project name rather than the project ID. Bug: T366679 --- keystone_browser/keystone.py | 5 +++++ keystone_browser/nova.py | 5 +++-- templates/servers.html | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index a76e053..93638ad 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -116,6 +116,11 @@ def project_name_for_id(id, cached=True): return project_data(id, cached=cached)["name"] +def project_id_for_name(id, cached=True): + id_for_name = {p["id"]: p["name"] for p in project_data(id, cached=cached)} + return id_for_name(id) + + def project_users_by_role(name, cached=True): """Get a dict of lists of user ids indexed by role name.""" key = "keystone:project_users_by_role:{}".format(name) diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index e33a0a9..a7d51f9 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -141,10 +141,11 @@ def server(fqdn, cached=True): if cached: data = cache.CACHE.load(key) if data is None: - name, project, _ = fqdn.split(".", 2) + name, project_name, _ = fqdn.split(".", 2) + project_id = keystone.project_id_for_name(project_name) servers = [] for region in get_regions(): - nova = nova_client(project, region) + nova = nova_client(project_id, region) reg_servers = nova.servers.list( detailed=True, search_opts={ diff --git a/templates/servers.html b/templates/servers.html index 907098e..0f483e0 100644 --- a/templates/servers.html +++ b/templates/servers.html @@ -10,7 +10,7 @@
        {% for server in servers %} - {% set fqdn = server.name ~ '.' ~ server.tenant_id ~ '.eqiad1.wikimedia.cloud' %} + {% set fqdn = server.name ~ '.' ~ server.project_id ~ '.eqiad1.wikimedia.cloud' %}
      • {{ fqdn }}
        • {% endfor %}
      From ed0c138a4567fa60745a83b45ca42504469f18df Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 14:47:07 -0600 Subject: [PATCH 143/174] Display project name (rather than ID) on zone pages Bug: T366679 --- templates/zone.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/zone.html b/templates/zone.html index 5f414e5..44637db 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -5,7 +5,7 @@ {% block content %}
      1. Projects
        2. -
      2. {{ project }}
        3. +
      3. {{ project_name }}
      4. DNS zones
      5. {{ name }}
      From 934e1c47026020aec65f18ac169ae5c261249e70 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 14:48:53 -0600 Subject: [PATCH 144/174] Display project name rather than ID on database instance pages Bug: T366679 --- app.py | 2 ++ templates/databaseinstance.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index a6ef989..53e6686 100644 --- a/app.py +++ b/app.py @@ -157,6 +157,7 @@ def project_database(project, name): cached = "purge" not in flask.request.args ctx = { "project": project, + "project_name": keystone.project_name_for_id(project), "name": name, } try: @@ -186,6 +187,7 @@ def zone(project, name): cached = "purge" not in flask.request.args ctx = { "project": project, + "project_name": keystone.project_name_for_id("project"), "name": name, } try: diff --git a/templates/databaseinstance.html b/templates/databaseinstance.html index f43afc8..ca68620 100644 --- a/templates/databaseinstance.html +++ b/templates/databaseinstance.html @@ -5,7 +5,7 @@ {% block content %}
      1. Projects
        2. -
      2. {{ project }}
        3. +
      3. {{ project_name }}
      4. Databases
      5. {{ name }}
      From a92d1c1c4e4fde92369bb979eeb18d36b2bb7f79 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 15:07:54 -0600 Subject: [PATCH 145/174] flake8 fix --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 53e6686..26eab8b 100644 --- a/app.py +++ b/app.py @@ -110,7 +110,8 @@ def project(project_id): project_data = keystone.project_data(project_id, cached) if project_data: users = keystone.project_users_by_role(project_id, cached) - # Create exclusive sets of users based on descending order of "power". + # Create exclusive sets of users based on descending order of + # "power". # member > service accounts > viewers members = set(users["admin"]) | set(users["member"]) service_accounts = { From dc7563fcf11fd60019a242f6a3f00a739e976d34 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 15:31:26 -0600 Subject: [PATCH 146/174] Correct per-server project name lookup --- keystone_browser/nova.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index a7d51f9..89e2fe1 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -77,9 +77,7 @@ def project_servers(project, cached=True): ) for server in data: - server["project_name"] = keystone.project_name_for_id( - server["project_id"] - ) + server["project_name"] = keystone.project_name_for_id(project) cache.CACHE.save(key, data, 300) return data From 3184e6b68052697ab75f5c1ecb69a8cf00a24ce5 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 15:38:11 -0600 Subject: [PATCH 147/174] Correct server fqdn on project panel --- templates/project.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/project.html b/templates/project.html index 963c31c..4d5e0cb 100644 --- a/templates/project.html +++ b/templates/project.html @@ -361,7 +361,7 @@

      {% for server in servers %} {% set image = images[server.image.id]|default('') %} {% set flavor = flavors[server.flavor.id]|default('') %} - {% set fqdn = server.name ~ '.' ~ data.id ~ '.eqiad1.wikimedia.cloud' %} + {% set fqdn = server.name ~ '.' ~ server.project_name ~ '.eqiad1.wikimedia.cloud' %} {{ fqdn }} {{ flavor.name|default('UNKNOWN') }} From 28f7919831ae039d7952bc3aee0c7899c7227266 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 17:04:55 -0600 Subject: [PATCH 148/174] Fix logic error in project_id_for_name --- keystone_browser/keystone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 93638ad..dc2b011 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -117,7 +117,7 @@ def project_name_for_id(id, cached=True): def project_id_for_name(id, cached=True): - id_for_name = {p["id"]: p["name"] for p in project_data(id, cached=cached)} + id_for_name = {p["id"]: p["name"] for p in all_projects(cached=cached)} return id_for_name(id) From b04256607092e33c0a9defd5ec910662f5266f37 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 17:22:24 -0600 Subject: [PATCH 149/174] Fix logic error in project_id_for_name, again! --- keystone_browser/keystone.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index dc2b011..9cd550e 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -116,9 +116,11 @@ def project_name_for_id(id, cached=True): return project_data(id, cached=cached)["name"] -def project_id_for_name(id, cached=True): - id_for_name = {p["id"]: p["name"] for p in all_projects(cached=cached)} - return id_for_name(id) +def project_id_for_name(name, cached=True): + for key, value in all_projects(cached=cached).items(): + if value == name: + return key + return None def project_users_by_role(name, cached=True): From 9725a0bb8f475239161d640190163945a8c55b84 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 17:32:33 -0600 Subject: [PATCH 150/174] server page: clarify project_id/project_name distinction --- app.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 26eab8b..2a5f9c1 100644 --- a/app.py +++ b/app.py @@ -233,20 +233,23 @@ def user(uid): @app.route("/server/") def server(fqdn): - name, project, tld = fqdn.split(".", 2) + name, project_name, tld = fqdn.split(".", 2) + project_id = keystone.project_id_for_name(project_name) ctx = { "fqdn": fqdn, - "project": project, + "project": project_name, } try: cached = "purge" not in flask.request.args ctx.update( { "server": nova.server(fqdn, cached), - "flavors": nova.flavors(project, cached), + "flavors": nova.flavors(project_id, cached), "images": glance.images(cached), - "puppetclasses": puppetclasses.classes(project, fqdn, cached), - "hiera": puppetclasses.hiera(project, fqdn, cached), + "puppetclasses": puppetclasses.classes( + project_id, fqdn, cached + ), + "hiera": puppetclasses.hiera(project_id, fqdn, cached), } ) if "user_id" in ctx["server"]: From f15a948b65856e03427a38fcfe2196d0f6a2a7fd Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Tue, 25 Feb 2025 18:22:51 -0600 Subject: [PATCH 151/174] Fix project name lookup for zone panel --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 2a5f9c1..1e6aaea 100644 --- a/app.py +++ b/app.py @@ -188,7 +188,7 @@ def zone(project, name): cached = "purge" not in flask.request.args ctx = { "project": project, - "project_name": keystone.project_name_for_id("project"), + "project_name": keystone.project_name_for_id(project), "name": name, } try: From 697335221d714e0bf137a237d4346c01d0797b85 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Wed, 26 Feb 2025 12:29:18 +0000 Subject: [PATCH 152/174] Puppetclasses * Replace project ids with project names in hiera dict * forgot this one! * flask.geturl * flask.redirect * Previous gambit to overload project() is misguided, just create a different route. * Fix project page urls for hiera and puppetclass templates * Create puppetclasses dict with project names as dict keys. * project() endpoint can now accept a project name or an ID --- app.py | 10 +++++++++- keystone_browser/puppetclasses.py | 26 +++++++++++++++++--------- templates/hierakey.html | 2 +- templates/puppetclass.html | 2 +- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 1e6aaea..5f909db 100644 --- a/app.py +++ b/app.py @@ -99,12 +99,20 @@ def servers(): return flask.render_template("servers.html", **ctx) +@app.route("/projectbyname/") +def projectbyname(project_name): + project_id = keystone.project_id_for_name(project_name) + return flask.redirect(flask.url_for("project", project_id=project_id)) + + @app.route("/project/") def project(project_id): cached = "purge" not in flask.request.args + project_name = keystone.project_name_for_id(project_id) + ctx = { "project_id": project_id, - "project_name": project_id, + "project_name": project_name, } try: project_data = keystone.project_data(project_id, cached) diff --git a/keystone_browser/puppetclasses.py b/keystone_browser/puppetclasses.py index ef7205f..43d9b5b 100644 --- a/keystone_browser/puppetclasses.py +++ b/keystone_browser/puppetclasses.py @@ -59,9 +59,15 @@ def prefixes(classname, cached=True): headers={"Accept": "application/x-yaml"}, ) if req.status_code != 200: - data = [] + data_with_ids = [] else: - data = yaml.safe_load(req.text) + data_with_ids = yaml.safe_load(req.text) + + data = { + keystone.project_name_for_id(key): value + for key, value in data_with_ids.items() + } + cache.CACHE.save(key, data, 1200) return data @@ -84,6 +90,7 @@ def all_classes(cached=True): data = [] else: data = yaml.safe_load(req.text) + cache.CACHE.save(key, data, 1200) return data["roles"] @@ -156,7 +163,7 @@ def giant_hiera_dict(cached=True): Make a dict of the form {hiera_key: - {project_id: + {project_name: {fqdn: hiera_value}}} """ key = "completehieradictt:" @@ -165,16 +172,17 @@ def giant_hiera_dict(cached=True): data = cache.CACHE.load(key) if data is None: data = {} - for project in keystone.all_projects().keys(): - for prefix in project_prefixes(project): - hieradata = hiera(project, prefix, cached) + for project_id in keystone.all_projects().keys(): + project_name = keystone.project_name_for_id(project_id) + for prefix in project_prefixes(project_id): + hieradata = hiera(project_id, prefix, cached) for key in hieradata.keys(): if key not in data: data[key] = {} - if project not in data[key]: - data[key][project] = {} + if project_id not in data[key]: + data[key][project_name] = {} if key in data: - data[key][project][prefix] = hieradata[key] + data[key][project_name][prefix] = hieradata[key] cache.CACHE.save(key, data, 1200) return data diff --git a/templates/hierakey.html b/templates/hierakey.html index f67cfab..5490b06 100644 --- a/templates/hierakey.html +++ b/templates/hierakey.html @@ -23,7 +23,7 @@

      {% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

      Project: {{ project }}

      +

      Project: {{ project }}

        {% if '_' in prefixes.keys() %}
      • All project instances
        • diff --git a/templates/puppetclass.html b/templates/puppetclass.html index 5c06c10..afaa7c0 100644 --- a/templates/puppetclass.html +++ b/templates/puppetclass.html @@ -23,7 +23,7 @@

        {% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

        Project: {{ project }}

        +

        Project: {{ project }}

          {% if '_' in prefixes.prefixes %}
        • All project instances
          • From f2b98712ddda1dc8a290efbb7dfe0e4836f6ac41 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Wed, 26 Feb 2025 06:45:37 -0600 Subject: [PATCH 153/174] stamp out a few more user-displayed project IDs --- templates/project.html | 2 +- templates/server.html | 4 ++-- templates/zone.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/project.html b/templates/project.html index 4d5e0cb..5afc66f 100644 --- a/templates/project.html +++ b/templates/project.html @@ -270,7 +270,7 @@

          {% for attachment in volume.attachments %} {% for server in servers %} {% if server.id == attachment.server_id %} - {% set fqdn = server.name ~ '.' ~ data.id ~ '.eqiad1.wikimedia.cloud' %} + {% set fqdn = server.name ~ '.' ~ project_name ~ '.eqiad1.wikimedia.cloud' %} {{ fqdn }} {% endif %} {% endfor %} diff --git a/templates/server.html b/templates/server.html index 638c788..d62fad4 100644 --- a/templates/server.html +++ b/templates/server.html @@ -5,7 +5,7 @@ {% block content %}
          1. Projects
            2. -
          2. {{ project }}
            3. +
          3. {{ project }}
          4. {{ fqdn }}
          @@ -27,7 +27,7 @@

          Project
          -
          {{ server.tenant_id }}
          +
          {{ project }}
          Instance name
          {{ server.name }}
          Instance Id
          diff --git a/templates/zone.html b/templates/zone.html index 44637db..b4e7ba6 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -26,7 +26,7 @@

          Project
          -
          {{ project }}
          +
          {{ project_name }}
          Zone ID
          {{ zone.id }}
          Type
          From 353e92046f1ccb772f3305e59849e0050050bee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Thu, 27 Mar 2025 12:51:29 +0200 Subject: [PATCH 154/174] Show basic network information --- app.py | 9 +++++++ keystone_browser/neutron.py | 24 ++++++++++++++++- templates/layout.html | 3 +++ templates/networks.html | 52 +++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 templates/networks.html diff --git a/app.py b/app.py index 5f909db..6dc4e66 100644 --- a/app.py +++ b/app.py @@ -325,6 +325,15 @@ def all_proxies(): return flask.render_template("proxies.html", **ctx) +@app.route("/network/") +def networks(): + cached = "purge" not in flask.request.args + ctx = { + "networks": neutron.networks(cached), + } + return flask.render_template("networks.html", **ctx) + + @app.route("/api/projects.json") def api_projects_json(): cached = "purge" not in flask.request.args diff --git a/keystone_browser/neutron.py b/keystone_browser/neutron.py index a086dfc..f5aeb66 100644 --- a/keystone_browser/neutron.py +++ b/keystone_browser/neutron.py @@ -96,9 +96,9 @@ def floating_ips(project: str, cached=True): if cached: data = cache.CACHE.load(key) if data is None: + data = [] for region in get_regions(): neutronclient = neutron_client(project, region) - data = [] data.extend( [ _map_ip_data(ip) @@ -111,3 +111,25 @@ def floating_ips(project: str, cached=True): cache.CACHE.save(key, data, 3600) return data + + +def networks(cached=True): + key = "neutron:networks" + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = {} + for region in get_regions(): + neutronclient = neutron_client("admin", region) + subnets = { + subnet["id"]: subnet + for subnet in neutronclient.list_subnets()["subnets"] + } + for network in neutronclient.list_networks()["networks"]: + network["subnets"] = [ + subnets[subnet] for subnet in network["subnets"] + ] + data[network["id"]] = network + cache.CACHE.save(key, data, 3600) + return data diff --git a/templates/layout.html b/templates/layout.html index 7cfc2cb..60e0bcc 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -41,6 +41,9 @@ {% with classes = url_for('all_puppetclasses') %}
        • Puppet Classes
          • {% endwith %} + {% with networks = url_for('networks') %} +
        • Networks
          • + {% endwith %}

          {% block navright %}{% endblock %} diff --git a/templates/networks.html b/templates/networks.html new file mode 100644 index 0000000..4d5462b --- /dev/null +++ b/templates/networks.html @@ -0,0 +1,52 @@ +{% extends "layout.html" %} + +{% block title %}Networks - {{ super() }}{% endblock %} + +{% block content %} +
          +
          + + + + + + + + + + + {% for network in networks.values()|sort(attribute='name') %} + {% for subnet in network.subnets|sort(attribute='cidr') %} + + {% if loop.first %} + {% set rowspan = network.subnets|length %} + + {% endif %} + + + + {% endfor %} + {% endfor %} + +
          NetworkSubnetSubnet CIDR
          {{ network.name }}{{ subnet.name }}{{ subnet.cidr }}
          +
          +
          +{% endblock %} + +{% block css %} + +{% endblock %} +{% block js %} +{{ super() }} + + +{% endblock %} From e0c7819d1323b5887168abdce8890019c0c4ab0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Thu, 24 Apr 2025 15:08:07 +0300 Subject: [PATCH 155/174] Accept more project ID/name combinations --- app.py | 23 ++++++++++++----------- keystone_browser/keystone.py | 8 ++++---- keystone_browser/nova.py | 4 ++-- templates/databaseinstance.html | 4 ++-- templates/hierakey.html | 2 +- templates/projects.html | 2 +- templates/proxies.html | 2 +- templates/puppetclass.html | 2 +- templates/server.html | 4 ++-- templates/user.html | 2 +- templates/zone.html | 8 ++++++-- 11 files changed, 33 insertions(+), 28 deletions(-) diff --git a/app.py b/app.py index 6dc4e66..2b88eaf 100644 --- a/app.py +++ b/app.py @@ -99,16 +99,12 @@ def servers(): return flask.render_template("servers.html", **ctx) -@app.route("/projectbyname/") -def projectbyname(project_name): - project_id = keystone.project_id_for_name(project_name) - return flask.redirect(flask.url_for("project", project_id=project_id)) - - -@app.route("/project/") -def project(project_id): +@app.route("/project/") +def project(project): cached = "purge" not in flask.request.args - project_name = keystone.project_name_for_id(project_id) + project_id, project_name = keystone.find_project(project) + if project != project_id: + return flask.redirect(flask.url_for("project", project=project_id)) ctx = { "project_id": project_id, @@ -241,8 +237,13 @@ def user(uid): @app.route("/server/") def server(fqdn): - name, project_name, tld = fqdn.split(".", 2) - project_id = keystone.project_id_for_name(project_name) + name, project_key, tld = fqdn.split(".", 2) + project_id, project_name = keystone.find_project(project_key) + if project_key != project_name: + return flask.redirect( + flask.url_for("server", fqdn=f"{name}.{project_name}.{tld}") + ) + ctx = { "fqdn": fqdn, "project": project_name, diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 9cd550e..5c58b0c 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -116,10 +116,10 @@ def project_name_for_id(id, cached=True): return project_data(id, cached=cached)["name"] -def project_id_for_name(name, cached=True): - for key, value in all_projects(cached=cached).items(): - if value == name: - return key +def find_project(search, cached=True): + for project_id, project_name in all_projects(cached=cached).items(): + if search == project_id or search == project_name: + return project_id, project_name return None diff --git a/keystone_browser/nova.py b/keystone_browser/nova.py index 89e2fe1..60e46cd 100644 --- a/keystone_browser/nova.py +++ b/keystone_browser/nova.py @@ -139,8 +139,8 @@ def server(fqdn, cached=True): if cached: data = cache.CACHE.load(key) if data is None: - name, project_name, _ = fqdn.split(".", 2) - project_id = keystone.project_id_for_name(project_name) + name, project_key, _ = fqdn.split(".", 2) + project_id, _ = keystone.find_project(project_key) servers = [] for region in get_regions(): nova = nova_client(project_id, region) diff --git a/templates/databaseinstance.html b/templates/databaseinstance.html index ca68620..6b4b4c7 100644 --- a/templates/databaseinstance.html +++ b/templates/databaseinstance.html @@ -5,7 +5,7 @@ {% block content %}
          1. Projects
            2. -
          2. {{ project_name }}
            3. +
          3. {{ project_name }}
          4. Databases
          5. {{ name }}
          @@ -27,7 +27,7 @@

          Project
          -
          {{ project }}
          +
          {{ project }}
          Instance name
          {{ instance.name }}
          Instance ID
          diff --git a/templates/hierakey.html b/templates/hierakey.html index 5490b06..7591a8e 100644 --- a/templates/hierakey.html +++ b/templates/hierakey.html @@ -23,7 +23,7 @@

          {% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

          Project: {{ project }}

          +

          Project: {{ project }}

            {% if '_' in prefixes.keys() %}
          • All project instances
            • diff --git a/templates/projects.html b/templates/projects.html index d7b11c8..e1ca8a5 100644 --- a/templates/projects.html +++ b/templates/projects.html @@ -11,7 +11,7 @@ diff --git a/templates/proxies.html b/templates/proxies.html index f5a288c..190a32f 100644 --- a/templates/proxies.html +++ b/templates/proxies.html @@ -23,7 +23,7 @@

            {% for proxy in proxies|sort(attribute='project') %} - {{ proxy.project }} + {{ proxy.project }} {{ proxy.domain }} {% for backend in proxy.backends %} diff --git a/templates/puppetclass.html b/templates/puppetclass.html index afaa7c0..3033af0 100644 --- a/templates/puppetclass.html +++ b/templates/puppetclass.html @@ -23,7 +23,7 @@

            {% for project, prefixes in data.items()|sort() %} {% if project != 'admin' %} -

            Project: {{ project }}

            +

            Project: {{ project }}

              {% if '_' in prefixes.prefixes %}
            • All project instances
              • diff --git a/templates/server.html b/templates/server.html index d62fad4..e64531f 100644 --- a/templates/server.html +++ b/templates/server.html @@ -5,7 +5,7 @@ {% block content %}
              1. Projects
                2. -
              2. {{ project }}
                3. +
              3. {{ project }}
              4. {{ fqdn }}
              @@ -27,7 +27,7 @@

              Project
              -
              {{ project }}
              +
              {{ project }}
              Instance name
              {{ server.name }}
              Instance Id
              diff --git a/templates/user.html b/templates/user.html index 36b1450..d5680f2 100644 --- a/templates/user.html +++ b/templates/user.html @@ -47,7 +47,7 @@

              User: {{ user.cn }}

              diff --git a/templates/zone.html b/templates/zone.html index b4e7ba6..a77ad9b 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -5,7 +5,7 @@ {% block content %}
              1. Projects
                2. -
              2. {{ project_name }}
                3. +
              3. {{ project_name }}
              4. DNS zones
              5. {{ name }}
              @@ -26,7 +26,7 @@

              Project
              -
              {{ project_name }}
              +
              {{ project_name }}
              Zone ID
              {{ zone.id }}
              Type
              @@ -47,6 +47,10 @@

              {{ zone.status }} {% endif %} + {% if zone.description %} +
              Description
              +
              {{ zone.description }}
              + {% endif %}

              From da922c52978f5f342d6654c393124214e57094de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Fri, 25 Apr 2025 13:46:46 +0300 Subject: [PATCH 156/174] zone: Sort by type as well --- templates/zone.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/zone.html b/templates/zone.html index a77ad9b..80538e7 100644 --- a/templates/zone.html +++ b/templates/zone.html @@ -73,7 +73,7 @@

              - {% for record in records|sort(attribute='sortkey') %} + {% for record in records|sort(attribute='sortkey,type') %} {% if record.name == "@" or record.name == "*" %} From 074ead63cff708b2a496181c5bb8ae19b1ea8f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 7 May 2025 12:47:55 +0300 Subject: [PATCH 157/174] proxies: Display project name instead of ID --- keystone_browser/proxies.py | 8 +++++--- templates/proxies.html | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/keystone_browser/proxies.py b/keystone_browser/proxies.py index e83b053..5b7c4f9 100644 --- a/keystone_browser/proxies.py +++ b/keystone_browser/proxies.py @@ -81,9 +81,11 @@ def all_proxies(cached=True): data = cache.CACHE.load(key) if data is None: data = [ - dict(project=project, **proxy) - for project in keystone.all_projects().keys() - for proxy in project_proxies(project, cached) + dict(project=project_id, project_name=project_name, **proxy) + for project_id, project_name in keystone.all_projects( + cached + ).items() + for proxy in project_proxies(project_id, cached) ] cache.CACHE.save(key, data, 3600) return data diff --git a/templates/proxies.html b/templates/proxies.html index 190a32f..7192ff6 100644 --- a/templates/proxies.html +++ b/templates/proxies.html @@ -21,9 +21,9 @@

              - {% for proxy in proxies|sort(attribute='project') %} + {% for proxy in proxies|sort(attribute='project_name') %} - {{ proxy.project }} + {{ proxy.project_name }} {{ proxy.domain }} {% for backend in proxy.backends %} From b5ae61f80815be89be3d5ef49681fc676c860e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 7 May 2025 12:48:35 +0300 Subject: [PATCH 158/174] Remove unused ipv4addr filter --- app.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app.py b/app.py index 2b88eaf..4aedcfd 100644 --- a/app.py +++ b/app.py @@ -30,7 +30,6 @@ from keystone_browser import puppetclasses from keystone_browser import proxies from keystone_browser import stats -from keystone_browser import utils from keystone_browser import cinder from keystone_browser import neutron from keystone_browser import trove @@ -446,11 +445,5 @@ def extract_hostname(backend): return hostname -@app.template_test() -def ipv4addr(s): - """Is the given string an IPv4 address?""" - return utils.is_ipv4(s) - - if __name__ == "__main__": app.run(port=3000, debug=True) From 31b66b4bb9f057f697aa6d49e958c35527cb644e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Fri, 23 May 2025 23:18:46 +0300 Subject: [PATCH 159/174] Fix project link in breadcrumbs for nonexistent serversr Bug: T395161 --- templates/server.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/server.html b/templates/server.html index e64531f..c2011eb 100644 --- a/templates/server.html +++ b/templates/server.html @@ -5,7 +5,7 @@ {% block content %}
              1. Projects
                2. -
              2. {{ project }}
                3. +
              3. {{ project }}
              4. {{ fqdn }}
              From 76af0e67763fa3d4e7cc7fa7bbfd1d406cca7b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Fri, 23 May 2025 23:22:35 +0300 Subject: [PATCH 160/174] Pass project ID to template --- app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app.py b/app.py index 4aedcfd..5fa8fb2 100644 --- a/app.py +++ b/app.py @@ -246,6 +246,7 @@ def server(fqdn): ctx = { "fqdn": fqdn, "project": project_name, + "project_id": project_id, } try: cached = "purge" not in flask.request.args From b9e5bc1a0a21a8982048deb77ba7570675aff698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 24 May 2025 14:31:06 +0300 Subject: [PATCH 161/174] Fix project in FQDNs on server list view --- templates/servers.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/servers.html b/templates/servers.html index 0f483e0..ed94c49 100644 --- a/templates/servers.html +++ b/templates/servers.html @@ -10,7 +10,7 @@
                {% for server in servers %} - {% set fqdn = server.name ~ '.' ~ server.project_id ~ '.eqiad1.wikimedia.cloud' %} + {% set fqdn = server.name ~ '.' ~ server.project_name ~ '.eqiad1.wikimedia.cloud' %}
              • {{ fqdn }}
                • {% endfor %}
              From 515498e1d94610a6acd72f147150c8f4e35f1fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Sat, 24 May 2025 14:32:30 +0300 Subject: [PATCH 162/174] Sort server view --- templates/servers.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/servers.html b/templates/servers.html index ed94c49..90c35bb 100644 --- a/templates/servers.html +++ b/templates/servers.html @@ -9,7 +9,7 @@
                - {% for server in servers %} + {% for server in servers|sort(attribute='name,project_name') %} {% set fqdn = server.name ~ '.' ~ server.project_name ~ '.eqiad1.wikimedia.cloud' %}
              • {{ fqdn }}
                • {% endfor %} From 62469865efa00c040fe11b02794bacae95343fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 4 Jun 2025 15:05:44 +0300 Subject: [PATCH 163/174] Remove enabled filters from endpoint searches Bug: T396011 --- keystone_browser/proxies.py | 4 +--- keystone_browser/puppetclasses.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/keystone_browser/proxies.py b/keystone_browser/proxies.py index 5b7c4f9..4a09c9b 100644 --- a/keystone_browser/proxies.py +++ b/keystone_browser/proxies.py @@ -35,9 +35,7 @@ def url_template(): """Get the url template for accessing the proxy service.""" c = keystone.keystone_client() proxy = c.services.list(type="proxy")[0] - endpoint = c.endpoints.list( - service=proxy.id, interface="public", enabled=True - )[0] + endpoint = c.endpoints.list(service=proxy.id, interface="public")[0] return endpoint.url diff --git a/keystone_browser/puppetclasses.py b/keystone_browser/puppetclasses.py index 43d9b5b..cdedd04 100644 --- a/keystone_browser/puppetclasses.py +++ b/keystone_browser/puppetclasses.py @@ -31,9 +31,7 @@ def url_template(): """Get the url template for accessing the Puppet ENC service.""" c = keystone.keystone_client() proxy = c.services.list(type="puppet-enc")[0] - endpoint = c.endpoints.list( - service=proxy.id, interface="public", enabled=True - )[0] + endpoint = c.endpoints.list(service=proxy.id, interface="public")[0] return endpoint.url.replace("/$(project_id)s", "") From 745308ca63a27db9dd5a98c3bd56a6ce0a8dcfff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Thu, 19 Jun 2025 15:53:56 +0300 Subject: [PATCH 164/174] keystone: Add object_storage role --- keystone_browser/keystone.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index 5c58b0c..59d577c 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -39,13 +39,19 @@ ("designateadmin", "906f1588626d4d0993629ea3928b6fb4"), ("glanceadmin", "1102f4ff63c3435793d0e4340bf4b04e"), ("keystonevalidate", "f3bebf5f4b6f40fa91f3614431f2c283"), + ("object_storage", "db4c840c39d44b059ccb110851226c8d"), # Members ("reader", "f75a3c410bca4e96a1cf6ac103b0ccaf"), ] ) -SERVICE_ACCOUNT_ROLES = ["glanceadmin", "designateadmin", "keystonevalidate"] +SERVICE_ACCOUNT_ROLES = [ + "glanceadmin", + "designateadmin", + "keystonevalidate", + "object_storage", +] @functools.lru_cache(maxsize=None) From 59e812970162bc3101a3579f50429a97e6e8bb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Thu, 11 Sep 2025 12:35:28 +0300 Subject: [PATCH 165/174] Support proxies with IPv6 address backends Bug: T404302 --- app.py | 2 +- keystone_browser/proxies.py | 22 +++++++++------------- keystone_browser/utils.py | 15 ++++++++------- keystone_browser/utils_test.py | 7 +++++++ tox.ini | 8 ++++++-- 5 files changed, 31 insertions(+), 23 deletions(-) create mode 100644 keystone_browser/utils_test.py diff --git a/app.py b/app.py index 5fa8fb2..03552a9 100644 --- a/app.py +++ b/app.py @@ -436,7 +436,7 @@ def contains(haystack, needle): @app.template_filter("extract_hostname") def extract_hostname(backend): """Extract a hostname from a backend description.""" - hostname = proxies.parse_backend(backend).get("hostname") + hostname = proxies.parse_backend(backend) if not hostname: return None if not hostname.endswith(".wikimedia.cloud"): diff --git a/keystone_browser/proxies.py b/keystone_browser/proxies.py index 4a09c9b..9c4db69 100644 --- a/keystone_browser/proxies.py +++ b/keystone_browser/proxies.py @@ -19,17 +19,14 @@ # with this program. If not, see . import functools -import re import socket +from urllib.parse import urlparse from . import cache from . import keystone from . import utils -RE_BACKEND = re.compile(r"^https?://(?P[^:]+):(?P\d+)$") - - @functools.lru_cache(maxsize=1) def url_template(): """Get the url template for accessing the proxy service.""" @@ -92,12 +89,11 @@ def all_proxies(cached=True): @functools.lru_cache(maxsize=1024) def parse_backend(backend): """Parse a proxy backend specification.""" - m = RE_BACKEND.match(backend) - if not m: - return {"hostname": backend} - data = m.groupdict() - if utils.is_ipv4(data["host"]): - data["hostname"] = socket.getfqdn(data["host"]) - else: - data["hostname"] = data["host"] - return data + url = urlparse(backend) + if not url or not url.hostname: + return None + + if not utils.is_ip(url.hostname): + return None + + return socket.getfqdn(url.hostname) diff --git a/keystone_browser/utils.py b/keystone_browser/utils.py index 9e1a4dc..86940ba 100644 --- a/keystone_browser/utils.py +++ b/keystone_browser/utils.py @@ -18,20 +18,21 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +import ipaddress import re from typing import List, Union -RE_IPV4ADDR = re.compile( - r"^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}" - r"(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$" -) RE_DIGIT = re.compile("(\\d+)") -def is_ipv4(s): - """Is the given string an IPv4 address?""" - return RE_IPV4ADDR.match(s) is not None +def is_ip(s): + """Is the given string an IP address?""" + try: + ipaddress.ip_address(s) + return True + except ValueError: + return False def natural_sort_key(element: str) -> List[Union[str, int]]: diff --git a/keystone_browser/utils_test.py b/keystone_browser/utils_test.py new file mode 100644 index 0000000..8185dc6 --- /dev/null +++ b/keystone_browser/utils_test.py @@ -0,0 +1,7 @@ +from . import utils + + +def test_is_ip(): + assert utils.is_ip("192.0.2.1") + assert utils.is_ip("::1") + assert not utils.is_ip("example.com") diff --git a/tox.ini b/tox.ini index 39d2003..5dbe735 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py3 +envlist = py3-{format,pytest} -[testenv] +[testenv:py3-format] commands = black --check --diff . flake8 @@ -12,6 +12,10 @@ deps = black flake8 +[testenv:py3-pytest] +commands = pytest -v {posargs} +deps = pytest + [flake8] exclude = .tox,venv count = 1 From 2977c25a286b2a50e8817d8f10bdf0efb6a734ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Wed, 17 Sep 2025 10:42:33 +0300 Subject: [PATCH 166/174] Add Octavia load balancer display Bug: T404419 --- app.py | 4 +++ keystone_browser/octavia.py | 52 +++++++++++++++++++++++++++++++++++++ keystone_browser/os.py | 27 +++++++++++++++++++ requirements.txt | 1 + templates/project.html | 46 ++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+) create mode 100644 keystone_browser/octavia.py create mode 100644 keystone_browser/os.py diff --git a/app.py b/app.py index 03552a9..76de46c 100644 --- a/app.py +++ b/app.py @@ -33,6 +33,7 @@ from keystone_browser import cinder from keystone_browser import neutron from keystone_browser import trove +from keystone_browser import octavia requests.utils.default_user_agent = lambda *args, **kwargs: ( @@ -147,6 +148,9 @@ def project(project): "neutron_limits": neutron.limits(project_id, cached), "databases": trove.project_instances(project_id, cached), "floating_ips": neutron.floating_ips(project_id, cached), + "load_balancers": octavia.project_load_balancers( + project_id, cached + ), } ) except Exception: diff --git a/keystone_browser/octavia.py b/keystone_browser/octavia.py new file mode 100644 index 0000000..1f98db4 --- /dev/null +++ b/keystone_browser/octavia.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of OpenStack browser +# +# Copyright (c) 2025 Taavi Väänänen for the Wikimedia Foundation +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +from . import cache +from . import os + + +def project_load_balancers(project_id, cached=True): + """Get a list of all database instances in a given project.""" + key = "octavia:project-lbs:{}".format(project_id) + data = None + if cached: + data = cache.CACHE.load(key) + if data is None: + data = [] + for lb in os.session().load_balancer.load_balancers( + project_id=project_id + ): + data.append( + { + "id": lb.id, + "name": lb.name, + "provider": lb.provider, + "provisioning_status": lb.provisioning_status, + "operating_status": lb.operating_status, + "flavor_id": lb.flavor_id, + "listeners": lb.listeners, + "pools": lb.pools, + # TODO: what is the additional_vips property? + "vips": [lb.vip_address], + } + ) + cache.CACHE.save(key, data, 300) + return data diff --git a/keystone_browser/os.py b/keystone_browser/os.py new file mode 100644 index 0000000..bfef036 --- /dev/null +++ b/keystone_browser/os.py @@ -0,0 +1,27 @@ +# This file is part of OpenStack browser +# +# Copyright (c) 2025 Taavi Väänänen for the Wikimedia Foundation +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +import functools + +import openstack + + +@functools.lru_cache() +def session(): + """Get a session for the novaobserver user.""" + return openstack.connect(cloud="novaobserver") diff --git a/requirements.txt b/requirements.txt index 021ca57..1cf88b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ dnspython flask gunicorn ldap3 +openstacksdk python-cinderclient python-glanceclient python-keystoneclient diff --git a/templates/project.html b/templates/project.html index 5afc66f..8828e34 100644 --- a/templates/project.html +++ b/templates/project.html @@ -219,6 +219,52 @@

              {% endif %} + {% if load_balancers %} +
              + + + + + + + + + + + {% for lb in load_balancers|sort(attribute='name') %} + + + + + + {% endfor %} + +
              NameAddressStatus
              {{ lb.name }} +
                + {% for vip in lb.vips|sort %} +
              • {{ vip }}
                • + {% endfor %} +
              +               		+ {% if lb.provisioning_status == "ACTIVE" %} + Active, + {% else %} + {{ lb.provisioning_status }}, + {% endif %} + + {% if lb.operating_status == "ONLINE" %} + Online + {% else %} + {{ lb.operating_status }} + {% endif %} +
              +
              + {% endif %} {% if zones %}
              {% else %} -

              Unknown project '{{ project_name }}'. Are you just guessing?

              +

              Failed to collect information about project '{{ project_name }}'.

              {% endif %} {% endblock %} diff --git a/templates/project_404.html b/templates/project_404.html new file mode 100644 index 0000000..37198d4 --- /dev/null +++ b/templates/project_404.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} + +{% block title %}Project {{ project }} - {{ super() }}{% endblock %} + +{% block content %} +
                +
              1. Projects
                2. +
              2. {{ project }}
                3. +
              + +
              +

              Project: {{ project }}

              +
              + +

              Unknown project '{{ project }}'. Are you just guessing?

              +{% endblock %} From 06dbbaf83d6ca6636444d0c1534dec5e62a290c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Tue, 27 Jan 2026 12:43:52 +0200 Subject: [PATCH 172/174] Format with latest black --- app.py | 1 - keystone_browser/keystone.py | 1 - keystone_browser/utils.py | 1 - 3 files changed, 3 deletions(-) diff --git a/app.py b/app.py index cd5d74f..1b63cfb 100644 --- a/app.py +++ b/app.py @@ -35,7 +35,6 @@ from keystone_browser import trove from keystone_browser import octavia - requests.utils.default_user_agent = lambda *args, **kwargs: ( "openstack-browser (tools.openstack-browser@toolforge.org)" + f" python-requests/{requests.__version__}" diff --git a/keystone_browser/keystone.py b/keystone_browser/keystone.py index b01a5e9..d5f4be1 100644 --- a/keystone_browser/keystone.py +++ b/keystone_browser/keystone.py @@ -29,7 +29,6 @@ from . import cache - ROLES = collections.OrderedDict( [ # Admins diff --git a/keystone_browser/utils.py b/keystone_browser/utils.py index 86940ba..8248eed 100644 --- a/keystone_browser/utils.py +++ b/keystone_browser/utils.py @@ -22,7 +22,6 @@ import re from typing import List, Union - RE_DIGIT = re.compile("(\\d+)") From 12920da47fcd603d0de3eca075c8936ad59bf5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Tue, 27 Jan 2026 12:49:03 +0200 Subject: [PATCH 173/174] Replace Wikitech developer account link Bug: T415582 --- templates/user.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/user.html b/templates/user.html index d5680f2..fb8cb19 100644 --- a/templates/user.html +++ b/templates/user.html @@ -16,9 +16,9 @@

              User: {{ user.cn }}

              -
              Wikitech
              -
              User:{{ user.cn }}
              -
              shell
              +
              Developer account
              +
              {{ user.cn }}
              +
              Shell
              {{ user.uid }}
              From 61b80fdbddf1ba4d6096e6a05ccbdefd15b21e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taavi=20V=C3=A4=C3=A4n=C3=A4nen?= Date: Tue, 27 Jan 2026 16:05:14 +0200 Subject: [PATCH 174/174] Use separate page for flavor information --- app.py | 10 ++++++++- templates/flavors.html | 47 ++++++++++++++++++++++++++++++++++++++++++ templates/home.html | 28 ------------------------- templates/layout.html | 10 ++++++++- 4 files changed, 65 insertions(+), 30 deletions(-) create mode 100644 templates/flavors.html diff --git a/app.py b/app.py index 1b63cfb..9869d0f 100644 --- a/app.py +++ b/app.py @@ -53,7 +53,6 @@ def home(): ctx.update( { "usage": stats.usage(cached), - "flavors": nova.flavors("observer", cached).values(), } ) except Exception: @@ -366,6 +365,15 @@ def networks(): return flask.render_template("networks.html", **ctx) +@app.route("/flavors/") +def flavors(): + cached = "purge" not in flask.request.args + ctx = { + "flavors": nova.flavors("observer", cached).values(), + } + return flask.render_template("flavors.html", **ctx) + + @app.route("/api/projects.json") def api_projects_json(): cached = "purge" not in flask.request.args diff --git a/templates/flavors.html b/templates/flavors.html new file mode 100644 index 0000000..d65cdf6 --- /dev/null +++ b/templates/flavors.html @@ -0,0 +1,47 @@ +{% extends "layout.html" %} + +{% block title %}Flavors - {{ super() }}{% endblock %} + +{% block content %} +
              +
              + + + + + + + + + + + {% for flavor in flavors|sort(attribute="vcpus") %} + + + + + + {% endfor %} + +
              NameCPUsRAM
              {{ flavor.name|default('UNKNOWN') }}{{ flavor.vcpus|default('-') }}{{ flavor.ram|default('-') }}M
              +
              +
              +{% endblock %} + +{% block css %} + +{% endblock %} +{% block js %} +{{ super() }} + + +{% endblock %} diff --git a/templates/home.html b/templates/home.html index c1b808c..07a28a9 100644 --- a/templates/home.html +++ b/templates/home.html @@ -43,34 +43,6 @@

          {% endif %} -{% if flavors %} -
          - - - - - - - - - - - {% for flavor in flavors|sort(attribute="vcpus") %} - - - - - - {% endfor %} - -
          NameCPUsRAM
          {{ flavor.name|default('UNKNOWN') }}{{ flavor.vcpus|default('-') }}{{ flavor.ram|default('-') }}M
          -
          - {% endif %}

          {% endblock %} {% block css %} diff --git a/templates/layout.html b/templates/layout.html index 60e0bcc..164342e 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -41,8 +41,16 @@ {% with classes = url_for('all_puppetclasses') %}
        • Puppet Classes
          • {% endwith %} + {% with flavors = url_for('flavors') %} {% with networks = url_for('networks') %} -
        • Networks
          • +
        • + + +
          • + {% endwith %} {% endwith %}