From c7be1948cc7203ecbf86699e8030876781d9a6af Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Tue, 7 Jan 2025 19:43:50 -0700
Subject: [PATCH 1/7] config get/set

---
 ceph_devstack/__init__.py |  7 +++++++
 ceph_devstack/cli.py      | 29 +++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py
index c2c88d660..fb7653836 100644
--- a/ceph_devstack/__init__.py
+++ b/ceph_devstack/__init__.py
@@ -50,6 +50,13 @@ def parse_args(args: List[str]) -> argparse.Namespace:
         help="Path to the ceph-devstack config file",
     )
     subparsers = parser.add_subparsers(dest="command")
+    parser_config = subparsers.add_parser("config", help="Get or set config items")
+    subparsers_config = parser_config.add_subparsers(dest="config_op")
+    parser_config_get = subparsers_config.add_parser("get")
+    parser_config_get.add_argument("name")
+    parser_config_set = subparsers_config.add_parser("set")
+    parser_config_set.add_argument("name")
+    parser_config_set.add_argument("value")
     parser_doc = subparsers.add_parser(
         "doctor", help="Check that the system meets requirements"
     )
diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py
index b3116f94d..b30f26742 100644
--- a/ceph_devstack/cli.py
+++ b/ceph_devstack/cli.py
@@ -20,6 +20,35 @@ def main():
     if args.command == "show-conf":
         print(yaml.safe_dump(config))
         return
+    if args.command == "config":
+        path = args.name.split(".")
+        sub_obj = obj = config
+        i = 0
+        if args.config_op == "get":
+            while i < len(path):
+                sub_path = path[i]
+                try:
+                    sub_obj = sub_obj[sub_path]
+                except KeyError:
+                    logger.error(f"{args.name} not found in config")
+                    sys.exit(1)
+                i += 1
+            print(yaml.safe_dump(sub_obj))
+            return
+        elif args.config_op == "set":
+            last_index = len(path) - 1
+            while i <= last_index:
+                # print(i)
+                if i < last_index:
+                    sub_obj = sub_obj[path[i]]
+                elif i == last_index:
+                    sub_obj[path[i]] = args.value
+                    print(config)
+                # print(path[i])
+                # print(sub_obj)
+                # print(config)
+                i += 1
+            return
     config["args"] = vars(args)
     data_path = Path(config["data_dir"]).expanduser()
     data_path.mkdir(parents=True, exist_ok=True)

From 74ee1f8804f131405e253e8a26097afa1ce8543c Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Tue, 7 Jan 2025 19:43:59 -0700
Subject: [PATCH 2/7] wip gha sqlite

---
 .github/workflows/ci.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 26d2af055..407db7c88 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Install packages
-        run: sudo apt install podman golang-github-containernetworking-plugin-dnsname
+        run: sudo apt install podman golang-github-containernetworking-plugin-dnsname sqlite3
       - name: Create virtualenv
         run: python3 -m venv venv
       - name: Install
@@ -37,6 +37,11 @@ jobs:
       - name: Dump logs
         if: success() || failure()
         run: podman logs -f teuthology
+      - name: Dump job data
+        if: success() || failure()
+        run: |
+          podman cp paddles:/paddles/dev.db /tmp/
+          sqlite3 /tmp/dev.db ".output stdout" ".mode json" "select * from jobs"'
       - name: Stop
         run: ./venv/bin/ceph-devstack -v stop
       - name: Remove

From bb7b5ba7b08f8a78e81600fddf6d1a214b57c45f Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 9 Jan 2025 11:21:05 -0700
Subject: [PATCH 3/7] config g/s

---
 ceph_devstack/__init__.py | 30 ++++++++++++++++++++++++++++++
 ceph_devstack/cli.py      | 29 +++--------------------------
 2 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py
index fb7653836..b5ae6a3d1 100644
--- a/ceph_devstack/__init__.py
+++ b/ceph_devstack/__init__.py
@@ -136,6 +136,36 @@ def load(self, config_path: Optional[Path] = None):
             elif user_path != DEFAULT_CONFIG_PATH.expanduser():
                 raise OSError(f"Config file at {user_path} not found!")
 
+    def get_value(self, name: str) -> str:
+        path = name.split(".")
+        obj = config
+        i = 0
+        while i < len(path):
+            sub_path = path[i]
+            try:
+                obj = obj[sub_path]
+            except KeyError:
+                logger.error(f"{name} not found in config")
+                raise
+            i += 1
+        if isinstance(obj, (str, int, bool)):
+            return str(obj)
+        return yaml.safe_dump(obj).strip()
+
+    def set_value(self, name: str, value: str):
+        path = name.split(".")
+        obj = config
+        i = 0
+        last_index = len(path) - 1
+        value = yaml.safe_load(value)
+        while i <= last_index:
+            if i < last_index:
+                obj = obj[path[i]]
+            elif i == last_index:
+                obj[path[i]] = value
+                print(yaml.safe_dump(config))
+            i += 1
+
 
 yaml.SafeDumper.add_representer(
     Config,
diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py
index b30f26742..81be1c41f 100644
--- a/ceph_devstack/cli.py
+++ b/ceph_devstack/cli.py
@@ -21,34 +21,11 @@ def main():
         print(yaml.safe_dump(config))
         return
     if args.command == "config":
-        path = args.name.split(".")
-        sub_obj = obj = config
-        i = 0
         if args.config_op == "get":
-            while i < len(path):
-                sub_path = path[i]
-                try:
-                    sub_obj = sub_obj[sub_path]
-                except KeyError:
-                    logger.error(f"{args.name} not found in config")
-                    sys.exit(1)
-                i += 1
-            print(yaml.safe_dump(sub_obj))
-            return
+            print(config.get_value(args.name))
         elif args.config_op == "set":
-            last_index = len(path) - 1
-            while i <= last_index:
-                # print(i)
-                if i < last_index:
-                    sub_obj = sub_obj[path[i]]
-                elif i == last_index:
-                    sub_obj[path[i]] = args.value
-                    print(config)
-                # print(path[i])
-                # print(sub_obj)
-                # print(config)
-                i += 1
-            return
+            config.set_value(args.name, args.value)
+        return
     config["args"] = vars(args)
     data_path = Path(config["data_dir"]).expanduser()
     data_path.mkdir(parents=True, exist_ok=True)

From 1798c4a53a149463f567c97996153c921b37f9e0 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 9 Jan 2025 12:53:32 -0700
Subject: [PATCH 4/7] toml

---
 ceph_devstack/__init__.py | 61 +++++++++++++++++++--------------------
 ceph_devstack/config.toml | 23 +++++++++++++++
 2 files changed, 52 insertions(+), 32 deletions(-)
 create mode 100644 ceph_devstack/config.toml

diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py
index b5ae6a3d1..316189932 100644
--- a/ceph_devstack/__init__.py
+++ b/ceph_devstack/__init__.py
@@ -1,9 +1,11 @@
 import argparse
 import logging.config
-import yaml
+import tomlkit
+import tomlkit.items
+import tomlkit.exceptions
 
-from pathlib import Path, PosixPath
-from typing import List, Optional
+from pathlib import Path
+from typing import List, Optional, Union
 
 
 VERBOSE = 15
@@ -12,17 +14,7 @@
 logger = logging.getLogger("ceph-devstack")
 
 PROJECT_ROOT = Path(__file__).parent
-DEFAULT_CONFIG_PATH = Path("~/.config/ceph-devstack/config.yml")
-
-
-def represent_path(dumper: yaml.dumper.SafeDumper, data: PosixPath) -> yaml.Node:
-    return dumper.represent_scalar("tag:yaml.org,2002:str", str(data))
-
-
-yaml.SafeDumper.add_representer(
-    PosixPath,
-    represent_path,
-)
+DEFAULT_CONFIG_PATH = Path("~/.config/ceph-devstack/config.toml")
 
 
 def parse_args(args: List[str]) -> argparse.Namespace:
@@ -126,15 +118,19 @@ def deep_merge(*maps):
 
 
 class Config(dict):
+    __slots__ = ["user_obj", "user_path"]
+
     def load(self, config_path: Optional[Path] = None):
-        self.update(yaml.safe_load((Path(__file__).parent / "config.yml").read_text()))
+        parsed = tomlkit.parse((Path(__file__).parent / "config.toml").read_text())
+        print(parsed)
+        self.update(parsed)
         if config_path:
-            user_path = config_path.expanduser()
-            if user_path.exists():
-                user_obj = yaml.safe_load(user_path.read_text()) or {}
-                self.update(deep_merge(config, user_obj))
-            elif user_path != DEFAULT_CONFIG_PATH.expanduser():
-                raise OSError(f"Config file at {user_path} not found!")
+            self.user_path = config_path.expanduser()
+            if self.user_path.exists():
+                self.user_obj: dict = tomlkit.parse(self.user_path.read_text()) or {}
+                self.update(deep_merge(config, self.user_obj))
+            elif self.user_path != DEFAULT_CONFIG_PATH.expanduser():
+                raise OSError(f"Config file at {self.user_path} not found!")
 
     def get_value(self, name: str) -> str:
         path = name.split(".")
@@ -150,28 +146,29 @@ def get_value(self, name: str) -> str:
             i += 1
         if isinstance(obj, (str, int, bool)):
             return str(obj)
-        return yaml.safe_dump(obj).strip()
+        return tomlkit.dumps(obj).strip()
 
     def set_value(self, name: str, value: str):
         path = name.split(".")
-        obj = config
+        obj = self.user_obj
         i = 0
         last_index = len(path) - 1
-        value = yaml.safe_load(value)
+        item: Union[tomlkit.items.Item, str] = value
+        try:
+            item = tomlkit.value(item)
+        except tomlkit.exceptions.UnexpectedCharError:
+            pass
+        except tomlkit.exceptions.InternalParserError:
+            pass
         while i <= last_index:
             if i < last_index:
                 obj = obj[path[i]]
             elif i == last_index:
-                obj[path[i]] = value
-                print(yaml.safe_dump(config))
+                obj[path[i]] = item
+                self.update(self.user_obj)
+                self.user_path.write_text(tomlkit.dumps(self.user_obj).strip())
             i += 1
 
 
-yaml.SafeDumper.add_representer(
-    Config,
-    yaml.representer.SafeRepresenter.represent_dict,
-)
-
-
 config = Config()
 config.load()
diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml
new file mode 100644
index 000000000..af07ab673
--- /dev/null
+++ b/ceph_devstack/config.toml
@@ -0,0 +1,23 @@
+data_dir = "~/.local/share/ceph-devstack"
+
+[containers.archive]
+image = "python:alpine"
+
+[containers.beanstalk]
+image = "quay.io/ceph-infra/teuthology-beanstalkd:latest"
+
+[containers.paddles]
+image = "quay.io/ceph-infra/paddles:latest"
+
+[containers.postgres]
+image = "quay.io/ceph-infra/teuthology-postgresql:latest"
+
+[containers.pulpito]
+image = "quay.io/ceph-infra/pulpito:latest"
+
+[containers.testnode]
+count = 3
+image = "quay.io/ceph-infra/teuthology-testnode:latest"
+
+[containers.teuthology]
+image = "quay.io/ceph-infra/teuthology-dev:latest"

From 1fb83076e3c50c2add86f04f19a3aa508ebd3b59 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 9 Jan 2025 12:53:40 -0700
Subject: [PATCH 5/7] dump

---
 ceph_devstack/__init__.py | 4 ++++
 ceph_devstack/cli.py      | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py
index 316189932..cad491ad7 100644
--- a/ceph_devstack/__init__.py
+++ b/ceph_devstack/__init__.py
@@ -44,6 +44,7 @@ def parse_args(args: List[str]) -> argparse.Namespace:
     subparsers = parser.add_subparsers(dest="command")
     parser_config = subparsers.add_parser("config", help="Get or set config items")
     subparsers_config = parser_config.add_subparsers(dest="config_op")
+    subparsers_config.add_parser("dump")
     parser_config_get = subparsers_config.add_parser("get")
     parser_config_get.add_argument("name")
     parser_config_set = subparsers_config.add_parser("set")
@@ -132,6 +133,9 @@ def load(self, config_path: Optional[Path] = None):
             elif self.user_path != DEFAULT_CONFIG_PATH.expanduser():
                 raise OSError(f"Config file at {self.user_path} not found!")
 
+    def dump(self):
+        return tomlkit.dumps(self)
+
     def get_value(self, name: str) -> str:
         path = name.split(".")
         obj = config
diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py
index 81be1c41f..e8a45f7ac 100644
--- a/ceph_devstack/cli.py
+++ b/ceph_devstack/cli.py
@@ -21,6 +21,8 @@ def main():
         print(yaml.safe_dump(config))
         return
     if args.command == "config":
+        if args.config_op == "dump":
+            print(config.dump())
         if args.config_op == "get":
             print(config.get_value(args.name))
         elif args.config_op == "set":

From 07f712a59614dde373b7ebc66b31ddd3d8bd1a2c Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 9 Jan 2025 18:43:48 -0700
Subject: [PATCH 6/7] conf user obj

---
 ceph_devstack/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py
index cad491ad7..1fa22544b 100644
--- a/ceph_devstack/__init__.py
+++ b/ceph_devstack/__init__.py
@@ -132,6 +132,8 @@ def load(self, config_path: Optional[Path] = None):
                 self.update(deep_merge(config, self.user_obj))
             elif self.user_path != DEFAULT_CONFIG_PATH.expanduser():
                 raise OSError(f"Config file at {self.user_path} not found!")
+            else:
+                self.user_obj = {}
 
     def dump(self):
         return tomlkit.dumps(self)
@@ -166,7 +168,7 @@ def set_value(self, name: str, value: str):
             pass
         while i <= last_index:
             if i < last_index:
-                obj = obj[path[i]]
+                obj = obj.setdefault(path[i], {})
             elif i == last_index:
                 obj[path[i]] = item
                 self.update(self.user_obj)

From e0f676c69bc9c9fbdb5310493a377e7280027c21 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 9 Jan 2025 18:47:55 -0700
Subject: [PATCH 7/7] conf mkdir

---
 ceph_devstack/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py
index 1fa22544b..a682d38a5 100644
--- a/ceph_devstack/__init__.py
+++ b/ceph_devstack/__init__.py
@@ -172,6 +172,7 @@ def set_value(self, name: str, value: str):
             elif i == last_index:
                 obj[path[i]] = item
                 self.update(self.user_obj)
+                self.user_path.parent.mkdir(exist_ok=True)
                 self.user_path.write_text(tomlkit.dumps(self.user_obj).strip())
             i += 1