diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26d2af05..407db7c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install packages - run: sudo apt install podman golang-github-containernetworking-plugin-dnsname + run: sudo apt install podman golang-github-containernetworking-plugin-dnsname sqlite3 - name: Create virtualenv run: python3 -m venv venv - name: Install @@ -37,6 +37,11 @@ jobs: - name: Dump logs if: success() || failure() run: podman logs -f teuthology + - name: Dump job data + if: success() || failure() + run: | + podman cp paddles:/paddles/dev.db /tmp/ + sqlite3 /tmp/dev.db ".output stdout" ".mode json" "select * from jobs"' - name: Stop run: ./venv/bin/ceph-devstack -v stop - name: Remove diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index c2c88d66..a682d38a 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -1,9 +1,11 @@ import argparse import logging.config -import yaml +import tomlkit +import tomlkit.items +import tomlkit.exceptions -from pathlib import Path, PosixPath -from typing import List, Optional +from pathlib import Path +from typing import List, Optional, Union VERBOSE = 15 @@ -12,17 +14,7 @@ logger = logging.getLogger("ceph-devstack") PROJECT_ROOT = Path(__file__).parent -DEFAULT_CONFIG_PATH = Path("~/.config/ceph-devstack/config.yml") - - -def represent_path(dumper: yaml.dumper.SafeDumper, data: PosixPath) -> yaml.Node: - return dumper.represent_scalar("tag:yaml.org,2002:str", str(data)) - - -yaml.SafeDumper.add_representer( - PosixPath, - represent_path, -) +DEFAULT_CONFIG_PATH = Path("~/.config/ceph-devstack/config.toml") def parse_args(args: List[str]) -> argparse.Namespace: @@ -50,6 +42,14 @@ def parse_args(args: List[str]) -> argparse.Namespace: help="Path to the ceph-devstack config file", ) subparsers = parser.add_subparsers(dest="command") + parser_config = subparsers.add_parser("config", help="Get or set config items") + subparsers_config = parser_config.add_subparsers(dest="config_op") + subparsers_config.add_parser("dump") + parser_config_get = subparsers_config.add_parser("get") + parser_config_get.add_argument("name") + parser_config_set = subparsers_config.add_parser("set") + parser_config_set.add_argument("name") + parser_config_set.add_argument("value") parser_doc = subparsers.add_parser( "doctor", help="Check that the system meets requirements" ) @@ -119,21 +119,62 @@ def deep_merge(*maps): class Config(dict): + __slots__ = ["user_obj", "user_path"] + def load(self, config_path: Optional[Path] = None): - self.update(yaml.safe_load((Path(__file__).parent / "config.yml").read_text())) + parsed = tomlkit.parse((Path(__file__).parent / "config.toml").read_text()) + print(parsed) + self.update(parsed) if config_path: - user_path = config_path.expanduser() - if user_path.exists(): - user_obj = yaml.safe_load(user_path.read_text()) or {} - self.update(deep_merge(config, user_obj)) - elif user_path != DEFAULT_CONFIG_PATH.expanduser(): - raise OSError(f"Config file at {user_path} not found!") - - -yaml.SafeDumper.add_representer( - Config, - yaml.representer.SafeRepresenter.represent_dict, -) + self.user_path = config_path.expanduser() + if self.user_path.exists(): + self.user_obj: dict = tomlkit.parse(self.user_path.read_text()) or {} + self.update(deep_merge(config, self.user_obj)) + elif self.user_path != DEFAULT_CONFIG_PATH.expanduser(): + raise OSError(f"Config file at {self.user_path} not found!") + else: + self.user_obj = {} + + def dump(self): + return tomlkit.dumps(self) + + def get_value(self, name: str) -> str: + path = name.split(".") + obj = config + i = 0 + while i < len(path): + sub_path = path[i] + try: + obj = obj[sub_path] + except KeyError: + logger.error(f"{name} not found in config") + raise + i += 1 + if isinstance(obj, (str, int, bool)): + return str(obj) + return tomlkit.dumps(obj).strip() + + def set_value(self, name: str, value: str): + path = name.split(".") + obj = self.user_obj + i = 0 + last_index = len(path) - 1 + item: Union[tomlkit.items.Item, str] = value + try: + item = tomlkit.value(item) + except tomlkit.exceptions.UnexpectedCharError: + pass + except tomlkit.exceptions.InternalParserError: + pass + while i <= last_index: + if i < last_index: + obj = obj.setdefault(path[i], {}) + elif i == last_index: + obj[path[i]] = item + self.update(self.user_obj) + self.user_path.parent.mkdir(exist_ok=True) + self.user_path.write_text(tomlkit.dumps(self.user_obj).strip()) + i += 1 config = Config() diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index b3116f94..e8a45f7a 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -20,6 +20,14 @@ def main(): if args.command == "show-conf": print(yaml.safe_dump(config)) return + if args.command == "config": + if args.config_op == "dump": + print(config.dump()) + if args.config_op == "get": + print(config.get_value(args.name)) + elif args.config_op == "set": + config.set_value(args.name, args.value) + return config["args"] = vars(args) data_path = Path(config["data_dir"]).expanduser() data_path.mkdir(parents=True, exist_ok=True) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml new file mode 100644 index 00000000..af07ab67 --- /dev/null +++ b/ceph_devstack/config.toml @@ -0,0 +1,23 @@ +data_dir = "~/.local/share/ceph-devstack" + +[containers.archive] +image = "python:alpine" + +[containers.beanstalk] +image = "quay.io/ceph-infra/teuthology-beanstalkd:latest" + +[containers.paddles] +image = "quay.io/ceph-infra/paddles:latest" + +[containers.postgres] +image = "quay.io/ceph-infra/teuthology-postgresql:latest" + +[containers.pulpito] +image = "quay.io/ceph-infra/pulpito:latest" + +[containers.testnode] +count = 3 +image = "quay.io/ceph-infra/teuthology-testnode:latest" + +[containers.teuthology] +image = "quay.io/ceph-infra/teuthology-dev:latest"