From 89a11e6336136143a56e3eb2e533521451bb39cb Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 7 Jul 2026 12:51:12 -0600 Subject: [PATCH 1/5] Alternative stacks This change makes the "stacks" deployed by ceph-devstack configurable. The existing behavior of ceph-devstack now belongs to the "teuthology" stack. This commit also adds a "ceph" stack, which deploys a minimal Ceph cluster in a rootless container. It can optionally compile Ceph and build a container to use for the deployment. The compile phase uses configurable persistent stores for sccache, dnf cache, and npm cache. --- .github/workflows/ci.yml | 21 + ceph_devstack/__init__.py | 33 +- ceph_devstack/ceph_devstack.te | 1 + ceph_devstack/cli.py | 88 ++- ceph_devstack/config.toml | 49 ++ ceph_devstack/exec.py | 21 +- ceph_devstack/resources/__init__.py | 10 +- ceph_devstack/resources/ceph/__init__.py | 120 ++- .../resources/ceph/ceph-node-entrypoint.sh | 466 +++++++++++ ceph_devstack/resources/ceph/ceph_node.py | 740 ++++++++++++++++++ ceph_devstack/resources/ceph/containers.py | 6 +- ceph_devstack/resources/container.py | 28 +- ceph_devstack/sccache-s3.conf | 8 + ceph_devstack/sccache.conf | 2 + pyproject.toml | 6 + tests/integration/test_ceph_stack.py | 24 + tests/integration/test_ceph_stack.sh | 69 ++ tests/resources/ceph/test_ceph_node.py | 435 ++++++++++ .../resources/ceph/test_cephdevstack_core.py | 54 +- .../resources/ceph/test_requirements_ceph.py | 2 +- tests/resources/test_podmanresource.py | 13 +- tests/test_config.py | 25 + tests/test_stacks.py | 62 ++ 23 files changed, 2188 insertions(+), 95 deletions(-) create mode 100755 ceph_devstack/resources/ceph/ceph-node-entrypoint.sh create mode 100644 ceph_devstack/resources/ceph/ceph_node.py create mode 100644 ceph_devstack/sccache-s3.conf create mode 100644 ceph_devstack/sccache.conf create mode 100644 tests/integration/test_ceph_stack.py create mode 100755 tests/integration/test_ceph_stack.sh create mode 100644 tests/resources/ceph/test_ceph_node.py create mode 100644 tests/test_stacks.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d09cdbca..ef686ab6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,3 +90,24 @@ jobs: run: uv run ceph-devstack -v stop - name: Remove run: uv run ceph-devstack -v remove + ceph-stack: + name: Ceph stack integration on ${{ matrix.os }} (python ${{ matrix.python }}) + runs-on: ${{ matrix.os }} + needs: lint + strategy: + matrix: + include: + - os: ubuntu-24.04 + python: "3.12" + steps: + - uses: actions/checkout@v7 + - name: Install packages + run: sudo apt-get update && sudo apt-get install podman golang-github-containernetworking-plugin-dnsname curl + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Set owner for /dev/loop-control + run: sudo chown $(whoami) /dev/loop-control + - name: Ceph stack integration test + env: + CEPH_DEVSTACK_INTEGRATION: "1" + run: bash tests/integration/test_ceph_stack.sh diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index d71474cd..3d1a06ed 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -41,6 +41,11 @@ def parse_args(args: List[str]) -> argparse.Namespace: default=DEFAULT_CONFIG_PATH, help="Path to the ceph-devstack config file", ) + parser.add_argument( + "--stack", + default=None, + help="Stack to deploy (default: value of 'stack' in config)", + ) subparsers = parser.add_subparsers(dest="command") parser_config = subparsers.add_parser("config", help="Get or set config items") subparsers_config = parser_config.add_subparsers(dest="config_op") @@ -128,7 +133,9 @@ def deep_merge(*maps): class Config(dict): - __slots__ = ["user_obj", "user_path"] + __slots__ = ["user_obj", "user_path", "active_stack", "active_services"] + active_stack: str | None + active_services: list[str] def load(self, config_path: Path | None = None): args = self.get("args") @@ -146,6 +153,30 @@ def load(self, config_path: Path | None = None): self.user_obj = {} if args: self["args"] = args + self.active_stack = None + self.active_services: list[str] = [] + + def apply_stack(self, stack_name: str | None = None) -> list[str]: + if stack_name is None: + default = self.get("stack", "teuthology") + stack_name = default if isinstance(default, str) else "teuthology" + stacks = self.get("stacks", {}) + if stack_name not in stacks: + known = ", ".join(sorted(stacks)) or "(none)" + raise ValueError(f"Unknown stack {stack_name!r}; known stacks: {known}") + stack = stacks[stack_name] + self.active_stack = stack_name + self.active_services = list(stack.get("services", [])) + + overrides = {} + for key in ("data_dir", "containers", "env"): + if key in stack: + overrides[key] = stack[key] + if overrides: + merged = deep_merge(self, overrides) + for key in overrides: + self[key] = merged[key] + return self.active_services def dump(self): return tomlkit.dumps(self) diff --git a/ceph_devstack/ceph_devstack.te b/ceph_devstack/ceph_devstack.te index f661669b..fdb9afc4 100644 --- a/ceph_devstack/ceph_devstack.te +++ b/ceph_devstack/ceph_devstack.te @@ -125,3 +125,4 @@ allow container_init_t tun_tap_device_t:chr_file mounton; allow container_t user_tmp_t:sock_file write; allow container_t unconfined_t:unix_stream_socket connectto; +allow container_t fixed_disk_device_t:blk_file setattr; diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index 8658bcbc..1fba02a2 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -2,7 +2,9 @@ import logging import sys +from argparse import Namespace from pathlib import Path +from subprocess import CalledProcessError from ceph_devstack import config, logger, parse_args, VERBOSE from ceph_devstack.requirements import check_requirements @@ -15,53 +17,65 @@ "unset": lambda config, args: config.unset_value(args.name), } -COMMAND_HANDLERS = { - "doctor": None, - "apply": lambda args, obj: obj.apply(args.command), - "pull": lambda _, obj: obj.pull(), - "build": lambda _, obj: obj.build(), - "create": lambda _, obj: obj.create(), - "remove": lambda _, obj: obj.remove(), - "start": lambda _, obj: obj.start(), - "stop": lambda _, obj: obj.stop(), - "watch": lambda _, obj: obj.watch(), - "wait": lambda args, obj: obj.wait(container_name=args.container), - "logs": lambda args, obj: obj.logs( - run_name=args.run_name, job_id=args.job_id, locate=args.locate - ), -} + +def _configure_console_logging(verbose: bool) -> None: + if not verbose: + return + for handler in logging.getLogger("root").handlers: + if not isinstance(handler, logging.FileHandler): + handler.setLevel(VERBOSE) + + +def _action_kwargs(args: Namespace) -> dict: + match args.command: + case "wait": + return {"container_name": args.container} + case "logs": + return { + "run_name": args.run_name, + "job_id": args.job_id, + "locate": args.locate, + } + case _: + return {} + + +async def _apply(stack: CephDevStack, action: str, **kwargs) -> int: + """Map a CLI command to a stack action and apply it.""" + if not all( + await asyncio.gather( + check_requirements(), + stack.check_requirements(), + ) + ): + logger.error("Requirements not met!") + return 1 + if action == "doctor": + return 0 + result = await stack.apply(action, **kwargs) + return result if result is not None else 0 def main() -> int: args = parse_args(sys.argv[1:]) config.load(args.config_file) - if args.verbose: - for handler in logging.getLogger("root").handlers: - if not isinstance(handler, logging.FileHandler): - handler.setLevel(VERBOSE) + try: + config.apply_stack(args.stack) + except ValueError as exc: + logger.error(str(exc)) + return 1 + _configure_console_logging(args.verbose) if args.command == "config": CONFIG_HANDLERS[args.config_op](config, args) return 0 config["args"] = vars(args) - data_path = Path(config["data_dir"]).expanduser() - data_path.mkdir(parents=True, exist_ok=True) - obj = CephDevStack() - - async def run(): - if not all( - await asyncio.gather( - check_requirements(), - obj.check_requirements(), - ) - ): - logger.error("Requirements not met!") - return 1 - handler = COMMAND_HANDLERS.get(args.command) - if handler: - return await handler(args, obj) - + Path(config["data_dir"]).expanduser().mkdir(parents=True, exist_ok=True) + stack = CephDevStack(stack_name=config.active_stack) + action = args.command try: - sys.exit(asyncio.run(run())) + return asyncio.run(_apply(stack, action, **_action_kwargs(args))) + except CalledProcessError as exc: + return exc.returncode if exc.returncode is not None else 1 except KeyboardInterrupt: logger.debug("Exiting!") return 130 # 128 + SIGINT diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 147a729e..786b1223 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -1,5 +1,54 @@ +stack = "teuthology" data_dir = "~/.local/share/ceph-devstack" +[stacks.teuthology] +services = [ + "postgres", + "paddles", + "beanstalk", + "pulpito", + "teuthology", + "testnode", + "archive", +] +secrets = ["ssh_keypair"] + +[stacks.ceph] +services = ["ceph_node"] +secrets = [] +data_dir = "~/.local/share/ceph-devstack/ceph" + +[containers.ceph_node] +image = "quay.io/ceph-ci/ceph:main" +loop_device_count = 3 +loop_device_size = "5G" +dashboard_port = 8080 +dashboard_ssl = false +# dashboard_user = "admin" +# dashboard_password = "admin" +# dashboard = false # disable mgr dashboard +# image_builder = "cpatch" # default; patch local binaries into runtime image +# image_builder = "container" # future: container/build.sh + local packages +# Build from a local ceph.git checkout: +# repo = "~/dev/ceph" +# build_dir = "build" # relative to repo; passed to build-with-container.py -b +# build_distro = "centos9" # build-with-container.py -d +# build_steps = ["build"] # default for cpatch; ["packages"] for container builder +# sccache = true # default; local disk cache (see sccache.conf) +# sccache = false # disable sccache +# sccache_mode = "local" # local (default) or s3 for shared remote cache +# sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" +# sccache_cache_size = "100G" # local cache size limit +# sccache_debug = false # enable SCCACHE_LOG=debug on build stdout +# sccache_conf = "/path/to/sccache.conf" # override bundled config entirely +# npm_cache = true # default; persist dashboard npm downloads +# npm_cache_path = "~/.local/share/ceph-devstack/cache/npm" # override npm cache location +# dnf_cache = false # persist dnf downloads when rebuilding ceph-build images +# dnf_cache_path = "~/.local/share/ceph-devstack/cache/dnf" # override dnf cache base dir +# repo may be a git worktree; ceph-devstack mounts git metadata for builds at /ceph +# base_image = "quay.io/ceph-ci/ceph:main" +# image = "localhost/ceph-devstack:main" + [containers.archive] image = "python:alpine" diff --git a/ceph_devstack/exec.py b/ceph_devstack/exec.py index 9cc27def..1bddb991 100644 --- a/ceph_devstack/exec.py +++ b/ceph_devstack/exec.py @@ -8,7 +8,7 @@ import signal import subprocess -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from ceph_devstack import logger, VERBOSE @@ -70,6 +70,25 @@ def signal_children(self, signal: signal.Signals, recursive=True): with contextlib.suppress(ProcessLookupError): os.killpg(pid, signal) + async def collect_output(self) -> Tuple[str, str]: + stdout = b"" + stderr = b"" + if self.stdout is not None: + stdout = await self.stdout.read() + if self.stderr is not None: + stderr = await self.stderr.read() + return stdout.decode(), stderr.decode() + + async def log_failure(self, cmd: List[str]) -> Tuple[str, str]: + stdout, stderr = await self.collect_output() + returncode = self.returncode if self.returncode is not None else -1 + logger.error(f"Command failed ({returncode}): {' '.join(cmd)}") + for line in stderr.rstrip("\n").splitlines(): + logger.error(line) + for line in stdout.rstrip("\n").splitlines(): + logger.error(line) + return stdout, stderr + class Command: def __init__( diff --git a/ceph_devstack/resources/__init__.py b/ceph_devstack/resources/__init__.py index 0a3c2a10..b9607191 100644 --- a/ceph_devstack/resources/__init__.py +++ b/ceph_devstack/resources/__init__.py @@ -65,9 +65,13 @@ async def cmd( ) returncode = await proc.wait() if check and returncode != 0: - # out = await proc.stderr.read() - # logger.error(out.decode()) - raise CalledProcessError(cmd=args, returncode=returncode) + stdout, stderr = await proc.log_failure(args) + raise CalledProcessError( + returncode, + args, + output=stdout or None, + stderr=stderr or None, + ) return proc def format_cmd(self, args: List): diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 17cc5641..11dea1b7 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -18,6 +18,7 @@ Teuthology, Archive, ) +from ceph_devstack.resources.ceph.ceph_node import CephNode, CONTAINER_CLUSTER_DIR from ceph_devstack.resources.ceph.requirements import ( HasSudo, LoopControlDeviceExists, @@ -85,23 +86,39 @@ class CephDevStackNetwork(Network): _name = "ceph-devstack" +SERVICES = { + "postgres": Postgres, + "paddles": Paddles, + "beanstalk": Beanstalk, + "pulpito": Pulpito, + "teuthology": Teuthology, + "testnode": TestNode, + "archive": Archive, + "ceph_node": CephNode, +} + +SECRETS = { + "ssh_keypair": SSHKeyPair, +} + + class CephDevStack: networks = [CephDevStackNetwork] secrets = [SSHKeyPair] - def __init__(self): - services = [ - Postgres, - Paddles, - Beanstalk, - Pulpito, - Teuthology, - TestNode, - Archive, - ] + def __init__(self, stack_name: str | None = None): + if ( + stack_name and stack_name != config.active_stack + ) or config.active_stack is None: + config.apply_stack(stack_name) + + self.stack_name = config.active_stack self.service_specs = {} - for service in services: - name = service.__name__.lower() + for name in config.active_services: + service = SERVICES.get(name) + if service is None: + logger.warning(f"Unknown service {name!r} in stack {self.stack_name!r}") + continue count = config["containers"][name].get("count", 1) if count == 0: continue @@ -115,9 +132,20 @@ def __init__(self): self.service_specs[name]["objects"] = [ service(name=f"{name}_{i}") for i in range(count) ] - if postgres_spec := self.service_specs.get("postgres"): + self._wire_services() + stack = config.get("stacks", {}).get(self.stack_name, {}) + self.secrets = [ + SECRETS[secret_name] + for secret_name in stack.get("secrets", []) + if secret_name in SECRETS + ] + + def _wire_services(self): + if (postgres_spec := self.service_specs.get("postgres")) and ( + paddles_spec := self.service_specs.get("paddles") + ): postgres_obj = postgres_spec["objects"][0] - paddles_obj = self.service_specs["paddles"]["objects"][0] + paddles_obj = paddles_spec["objects"][0] paddles_obj.env_vars["PADDLES_SQLALCHEMY_URL"] = ( postgres_obj.paddles_sqla_url ) @@ -126,8 +154,9 @@ async def check_requirements(self): result = True result = has_sudo = await HasSudo().evaluate() - result = result and await LoopControlDeviceExists().evaluate() - result = result and await LoopControlDeviceWriteable().evaluate() + if "testnode" in self.service_specs or "ceph_node" in self.service_specs: + result = result and await LoopControlDeviceExists().evaluate() + result = result and await LoopControlDeviceWriteable().evaluate() # Check for SELinux being enabled and Enforcing; then check for the # presence of our module. If necessary, inform the user and instruct @@ -135,14 +164,24 @@ async def check_requirements(self): if has_sudo and await host.selinux_enforcing(): result = result and await SELinuxModule().evaluate() - for name, obj in config["containers"].items(): - if (repo := obj.get("repo")) and not host.path_exists(repo): + for name in self.service_specs: + obj = config["containers"][name] + if (repo := obj.get("repo")) and not host.path_exists( + os.path.expanduser(str(repo)) + ): result = False logger.error(f"Repo for {name} not found at {repo}") return result - async def apply(self, action): - return await getattr(self, action)() + async def apply(self, action: str, **kwargs) -> int | None: + if action == "wait": + return await self.wait(**kwargs) + if action == "logs": + return await self.logs(**kwargs) + method = getattr(self, action, None) + if method is None: + raise AttributeError(f"Unknown action {action!r}") + return await method() async def pull(self): logger.info("Pulling images...") @@ -155,14 +194,18 @@ async def build(self): await spec["objects"][0].build() async def create(self): + args = config.get("args", {}) + if args.get("build"): + await self.build() logger.info("Creating containers...") await CephDevStackNetwork().create() - await SSHKeyPair().create() - containers = [] + for secret in self.secrets: + await secret().create() + tasks = [] for spec in self.service_specs.values(): for object in spec["objects"]: - containers.append(object.create()) - await asyncio.gather(*containers) + tasks.append(object.create()) + await asyncio.gather(*tasks) async def start(self): await self.create() @@ -170,12 +213,21 @@ async def start(self): for spec in self.service_specs.values(): for object in spec["objects"]: await object.start() - logger.info( - "All containers are running. To monitor teuthology, try running: podman " - "logs -f teuthology" - ) - hostname = host.hostname() - logger.info(f"View test results at http://{hostname}:8081/") + if "teuthology" in self.service_specs: + logger.info( + "All containers are running. To monitor teuthology, try running: " + "podman logs -f teuthology" + ) + else: + logger.info("All containers are running.") + if "pulpito" in self.service_specs: + hostname = host.hostname() + logger.info(f"View test results at http://{hostname}:8081/") + if "ceph_node" in self.service_specs: + logger.info( + "Monitor the cluster with: podman exec ceph_node ceph " + f"-c {CONTAINER_CLUSTER_DIR}/ceph.conf -s" + ) async def stop(self): logger.info("Stopping containers...") @@ -193,10 +245,14 @@ async def remove(self): containers.append(object.remove()) await asyncio.gather(*containers) await CephDevStackNetwork().remove() - await SSHKeyPair().remove() + for secret in self.secrets: + await secret().remove() async def watch(self): - logger.info("Watching containers; will replace any that are stopped") + logger.info( + "Entering watch mode: while waiting for teuthology to " + "exit, other containers will be replaced as they are stopped." + ) containers = [] for spec in self.service_specs.values(): if not spec["count"] > 0: diff --git a/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh new file mode 100755 index 00000000..ed6dab58 --- /dev/null +++ b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh @@ -0,0 +1,466 @@ +#!/bin/bash +set -euo pipefail + +CLUSTER_DIR="${CLUSTER_DIR:?CLUSTER_DIR is required}" +CEPH_CONF="${CLUSTER_DIR}/ceph.conf" +ADMIN_KEYRING="${CLUSTER_DIR}/ceph.client.admin.keyring" +MON_ID="${MON_ID:-a}" +MGR_ID="${MGR_ID:-x}" +OSD_DEVICES="${OSD_DEVICES:-}" +DASHBOARD_ENABLED="${DASHBOARD_ENABLED:-true}" +DASHBOARD_PORT="${DASHBOARD_PORT:-8080}" +DASHBOARD_SSL="${DASHBOARD_SSL:-false}" +DASHBOARD_USER="${DASHBOARD_USER:-admin}" +DASHBOARD_PASSWORD="${DASHBOARD_PASSWORD:-admin}" +BOOTSTRAP_DIR="${CLUSTER_DIR}/bootstrap" +BOOTSTRAP_OSD_KEYRING="/var/lib/ceph/bootstrap-osd/ceph.keyring" + +export CEPH_VOLUME_ALLOW_LOOP_DEVICES=true + +mkdir -p "${CLUSTER_DIR}/var/lib/ceph" + +export_ceph_conf() { + if [[ -f "${CEPH_CONF}" ]]; then + export CEPH_CONF + else + unset CEPH_CONF || true + fi +} + +chown_cluster() { + chown -R ceph:ceph /var/lib/ceph + for path in "${CEPH_CONF}" "${ADMIN_KEYRING}" "${CLUSTER_DIR}/fsid"; do + if [[ -e "${path}" ]]; then + chown ceph:ceph "${path}" + fi + done +} + +detect_mon_ip() { + local ip + + ip="$(ip -4 route get 1.1.1.1 2>/dev/null | \ + awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}')" + if [[ -n "${ip}" ]]; then + echo "${ip}" + return + fi + ip="$(hostname -I | awk '{print $1}')" + if [[ -n "${ip}" ]]; then + echo "${ip}" + return + fi + echo "127.0.0.1" +} + +detect_public_network() { + local mon_ip="$1" + local net + + net="$(ip route list 2>/dev/null | grep -w "${mon_ip}" | grep -v default | \ + grep -E '/[0-9]+' | awk '{print $1; exit}')" + if [[ -n "${net}" ]]; then + echo "${net}" + return + fi + echo "${mon_ip}/32" +} + +bootstrap_cluster() { + local fsid mon_ip public_network mon_keyring monmap mon_host + fsid="$(cat "${CLUSTER_DIR}/fsid" 2>/dev/null || uuidgen | tee "${CLUSTER_DIR}/fsid")" + mon_ip="$(detect_mon_ip)" + public_network="$(detect_public_network "${mon_ip}")" + mon_host="v2:${mon_ip}:3300,v1:${mon_ip}:6789" + mon_keyring="${BOOTSTRAP_DIR}/ceph.mon.keyring" + monmap="${BOOTSTRAP_DIR}/ceph.monmap" + + mkdir -p "${BOOTSTRAP_DIR}" + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/mon/ceph-${MON_ID}" + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/mgr/ceph-${MGR_ID}" + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/bootstrap-osd" + + ceph-authtool --create-keyring "${mon_keyring}" --gen-key -n mon. --cap mon 'allow *' + ceph-authtool "${mon_keyring}" --gen-key -n client.admin \ + --cap mon 'allow *' \ + --cap osd 'allow *' \ + --cap mds 'allow *' \ + --cap mgr 'allow *' + ceph-authtool --create-keyring \ + "${CLUSTER_DIR}/var/lib/ceph/bootstrap-osd/ceph.keyring" \ + --gen-key -n client.bootstrap-osd \ + --cap mon 'profile bootstrap-osd' + ceph-authtool "${mon_keyring}" --import-keyring \ + "${CLUSTER_DIR}/var/lib/ceph/bootstrap-osd/ceph.keyring" + + monmaptool --create --clobber --fsid "${fsid}" \ + --addv "${MON_ID}" "[${mon_host}]" \ + "${monmap}" + + cat > "${CEPH_CONF}" </dev/null || hostname) +public bind addr = + +[osd] +osd numa auto affinity = false +EOF + export_ceph_conf + + ceph-mon --mkfs -i "${MON_ID}" --monmap "${monmap}" --keyring "${mon_keyring}" \ + --conf "${CEPH_CONF}" + + ceph-authtool --create-keyring "${ADMIN_KEYRING}" + ceph-authtool "${ADMIN_KEYRING}" \ + --import-keyring "${mon_keyring}" + + chown_cluster + rm -rf "${BOOTSTRAP_DIR}" +} + +start_mon() { + if pgrep -f "ceph-mon -i ${MON_ID}" >/dev/null 2>&1; then + return + fi + ceph-mon -i "${MON_ID}" --conf "${CEPH_CONF}" --setuser ceph --setgroup ceph -f & +} + +start_mgr() { + if pgrep -f "ceph-mgr -i ${MGR_ID}" >/dev/null 2>&1; then + return + fi + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/mgr/ceph-${MGR_ID}" + mgr_keyring="${CLUSTER_DIR}/var/lib/ceph/mgr/ceph-${MGR_ID}/keyring" + for attempt in $(seq 1 30); do + if ceph --conf "${CEPH_CONF}" auth get-or-create "mgr.${MGR_ID}" \ + mon 'allow profile mgr' osd 'allow *' mds 'allow *' \ + -o "${mgr_keyring}"; then + break + fi + sleep 2 + done + if [[ ! -f "${mgr_keyring}" ]]; then + echo "Failed to create mgr.${MGR_ID} keyring" >&2 + return 1 + fi + chown ceph:ceph "${mgr_keyring}" + ceph-mgr -i "${MGR_ID}" --conf "${CEPH_CONF}" --no-mon-config \ + --keyring "${mgr_keyring}" \ + --setuser ceph --setgroup ceph -f & +} + +wait_for_mgr() { + local attempt available + + for attempt in $(seq 1 60); do + available="$(ceph --conf "${CEPH_CONF}" mgr stat -f json 2>/dev/null | python3 -c \ + 'import json,sys; print(json.load(sys.stdin).get("available", False))' \ + 2>/dev/null || echo False)" + if [[ "${available}" == "True" ]]; then + return 0 + fi + sleep 2 + done + echo "Timed out waiting for mgr" >&2 + return 1 +} + +setup_dashboard() { + local mon_ip scheme url port_option password_file + + if [[ "${DASHBOARD_ENABLED}" != "true" ]]; then + return 0 + fi + + password_file="${CLUSTER_DIR}/dashboard-admin-secret.txt" + mon_ip="$(detect_mon_ip)" + + ceph --conf "${CEPH_CONF}" mgr module enable dashboard + + if [[ "${DASHBOARD_SSL}" == "true" ]]; then + port_option="ssl_server_port" + scheme="https" + ceph --conf "${CEPH_CONF}" config set mgr mgr/dashboard/ssl true --force + ceph --conf "${CEPH_CONF}" config set mgr \ + "mgr/dashboard/${MGR_ID}/${port_option}" "${DASHBOARD_PORT}" --force + ceph --conf "${CEPH_CONF}" dashboard create-self-signed-cert 2>/dev/null || true + else + port_option="server_port" + scheme="http" + ceph --conf "${CEPH_CONF}" config set mgr mgr/dashboard/ssl false --force + ceph --conf "${CEPH_CONF}" config set mgr \ + "mgr/dashboard/${MGR_ID}/${port_option}" "${DASHBOARD_PORT}" --force + fi + ceph --conf "${CEPH_CONF}" config set mgr \ + "mgr/dashboard/${MGR_ID}/server_addr" "0.0.0.0" --force + + for attempt in $(seq 1 60); do + if ceph --conf "${CEPH_CONF}" -h 2>/dev/null | grep -q '^dashboard '; then + break + fi + sleep 2 + done + + printf '%s' "${DASHBOARD_PASSWORD}" > "${password_file}" + chown ceph:ceph "${password_file}" + chmod 600 "${password_file}" + + if ceph --conf "${CEPH_CONF}" dashboard ac-user-show "${DASHBOARD_USER}" >/dev/null 2>&1; then + ceph --conf "${CEPH_CONF}" dashboard ac-user-set-password "${DASHBOARD_USER}" \ + -i "${password_file}" + else + ceph --conf "${CEPH_CONF}" dashboard ac-user-create "${DASHBOARD_USER}" \ + -i "${password_file}" administrator --force-password + fi + + url="${scheme}://${mon_ip}:${DASHBOARD_PORT}/" + echo "Ceph dashboard: ${url}" + echo " user: ${DASHBOARD_USER}" + if [[ "${DASHBOARD_SHOW_PASSWORD:-false}" == "true" ]]; then + echo " password: ${DASHBOARD_PASSWORD}" + else + echo " password: (see dashboard_password in config; set DASHBOARD_SHOW_PASSWORD=true to log it)" + fi +} + +wait_for_mon() { + local attempt + for attempt in $(seq 1 60); do + if ceph --conf "${CEPH_CONF}" -s >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "Timed out waiting for monitor" >&2 + return 1 +} + +bluestore_show_label() { + local dev="$1" + ceph-bluestore-tool show-label --dev "${dev}" --no-mon-config +} + +get_osd_id_from_device() { + local dev="$1" + bluestore_show_label "${dev}" | python3 -c ' +import json +import sys + +data = json.load(sys.stdin) +labels = data.get("devices", data) +for label in labels.values(): + if label.get("description") == "main": + print(label["whoami"]) + break +' +} + +device_has_osd_label() { + local dev="$1" + get_osd_id_from_device "${dev}" >/dev/null 2>&1 +} + +chown_block_device() { + local dev="$1" + chown ceph:ceph "${dev}" +} + +mkfs_raw_osd() { + local osd_id="$1" osd_dir="$2" osd_fsid="$3" osd_secret="$4" + local attempt + + for attempt in $(seq 1 5); do + if printf '%s' "${osd_secret}" | ceph-osd --cluster ceph --conf "${CEPH_CONF}" \ + --osd-objectstore bluestore --mkfs \ + --no-mon-config \ + -i "${osd_id}" \ + --monmap "${osd_dir}/activate.monmap" \ + --osd-data "${osd_dir}" \ + --osd-uuid "${osd_fsid}" \ + --setuser ceph --setgroup ceph \ + --keyfile -; then + return 0 + fi + echo "ceph-osd --mkfs failed for osd.${osd_id} (attempt ${attempt}/5)" >&2 + sleep 1 + done + return 1 +} + +prepare_raw_osd() { + local dev="$1" + local osd_id osd_fsid osd_dir osd_secret + + if device_has_osd_label "${dev}"; then + return 0 + fi + + # Clear any stale label/header from a previous failed prepare. + dd if=/dev/zero of="${dev}" bs=1M count=1 conv=notrunc status=none 2>/dev/null || true + + osd_fsid="$(uuidgen)" + osd_secret="$(ceph-authtool --gen-print-key)" + osd_id="$(printf '{"cephx_secret":"%s"}' "${osd_secret}" | \ + ceph --conf "${CEPH_CONF}" --cluster ceph --name client.bootstrap-osd \ + --keyring "${BOOTSTRAP_OSD_KEYRING}" -i - osd new "${osd_fsid}")" + + osd_dir="/var/lib/ceph/osd/ceph-${osd_id}" + rm -rf "${osd_dir}" + mkdir -p "${osd_dir}" + + ceph-authtool "${osd_dir}/keyring" --create-keyring --name "osd.${osd_id}" \ + --add-key "${osd_secret}" \ + --cap mon 'allow profile osd' \ + --cap mgr 'allow profile osd' \ + --cap osd 'allow *' + + ceph --conf "${CEPH_CONF}" --cluster ceph --name client.bootstrap-osd \ + --keyring "${BOOTSTRAP_OSD_KEYRING}" \ + mon getmap -o "${osd_dir}/activate.monmap" + + chown_block_device "${dev}" + ln -snf "${dev}" "${osd_dir}/block" + chown -R ceph:ceph "${osd_dir}" + + mkfs_raw_osd "${osd_id}" "${osd_dir}" "${osd_fsid}" "${osd_secret}" + chown -R ceph:ceph "${osd_dir}" +} + +start_raw_osd() { + local dev="$1" + local osd_id osd_dir + + if ! device_has_osd_label "${dev}"; then + echo "No BlueStore label on ${dev}; skipping osd start" >&2 + return 1 + fi + + osd_id="$(get_osd_id_from_device "${dev}")" + osd_dir="/var/lib/ceph/osd/ceph-${osd_id}" + mkdir -p "${osd_dir}" + + chown_block_device "${dev}" + + # After mkfs the osd dir is already populated; prime-osd-dir is only needed + # when re-activating from a labeled device with an empty/missing data dir. + if [[ ! -f "${osd_dir}/ready" ]]; then + rm -f "${osd_dir}/block" "${osd_dir}/block.wal" "${osd_dir}/block.db" + chown -R ceph:ceph "${osd_dir}" + ceph-bluestore-tool prime-osd-dir \ + --path "${osd_dir}" \ + --no-mon-config \ + --dev "${dev}" + fi + + ln -snf "${dev}" "${osd_dir}/block" + chown -R ceph:ceph "${osd_dir}" + + if ! pgrep -f "ceph-osd -i ${osd_id}" >/dev/null 2>&1; then + ceph-osd -i "${osd_id}" --conf "${CEPH_CONF}" --no-mon-config \ + --keyring "${osd_dir}/keyring" \ + --setuser ceph --setgroup ceph -f & + fi +} + +wait_for_osds() { + local attempt expected up + local -a devices=() + + if [[ -z "${OSD_DEVICES}" ]]; then + return 0 + fi + IFS=',' read -ra devices <<< "${OSD_DEVICES}" + expected=0 + for dev in "${devices[@]}"; do + [[ -b "${dev}" ]] && expected=$((expected + 1)) + done + [[ "${expected}" -gt 0 ]] || return 0 + + for attempt in $(seq 1 60); do + up="$(ceph --conf "${CEPH_CONF}" osd stat -f json 2>/dev/null | python3 -c \ + 'import json,sys; print(json.load(sys.stdin).get("num_osds_up", 0))' 2>/dev/null || echo 0)" + if [[ "${up}" -ge "${expected}" ]]; then + return 0 + fi + sleep 2 + done + echo "Timed out waiting for ${expected} OSD(s)" >&2 + return 1 +} + +wait_for_health() { + local attempt health + + for attempt in $(seq 1 90); do + health="$(ceph --conf "${CEPH_CONF}" health 2>/dev/null || true)" + case "${health}" in + HEALTH_OK) + echo "${health}" + return 0 + ;; + HEALTH_ERR) + echo "${health} ($(ceph --conf "${CEPH_CONF}" health detail 2>/dev/null | tr '\n' ' '))" >&2 + return 1 + ;; + esac + sleep 2 + done + echo "Timed out waiting for cluster health" >&2 + return 1 +} + +prepare_osds() { + local dev + if [[ -z "${OSD_DEVICES}" ]]; then + return + fi + IFS=',' read -ra devices <<< "${OSD_DEVICES}" + for dev in "${devices[@]}"; do + [[ -b "${dev}" ]] || continue + prepare_raw_osd "${dev}" + done + for dev in "${devices[@]}"; do + [[ -b "${dev}" ]] || continue + start_raw_osd "${dev}" + done +} + +if [[ ! -f "${CEPH_CONF}" ]]; then + echo "Bootstrapping ceph cluster in ${CLUSTER_DIR}" + bootstrap_cluster +else + export_ceph_conf +fi + +start_mon +wait_for_mon +start_mgr +wait_for_mgr +setup_dashboard +prepare_osds +wait_for_osds +wait_for_health + +echo "Ceph cluster is running; use: podman exec ${HOSTNAME:-ceph_node} ceph -s" +exec tail -f /dev/null diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py new file mode 100644 index 00000000..55318636 --- /dev/null +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -0,0 +1,740 @@ +import asyncio +import os +import shlex +import shutil +import subprocess +import sys +import uuid + +from pathlib import Path +from subprocess import CalledProcessError +from typing import List + +from ceph_devstack import PROJECT_ROOT, config, logger +from ceph_devstack.host import host +from ceph_devstack.resources.container import Container + + +DEFAULT_CEPH_IMAGE = "quay.io/ceph-ci/ceph:main" +DEFAULT_BASE_IMAGE = "quay.io/ceph-ci/ceph:main" +PACKAGE_SCCACHE_CONF = PROJECT_ROOT / "sccache.conf" +PACKAGE_SCCACHE_S3_CONF = PROJECT_ROOT / "sccache-s3.conf" +CONTAINER_SCCACHE_DIR = "/sccache" +CONTAINER_GIT_METADATA_DIR = "/git-metadata" +BWC_HOMEDIR = "/ceph" +REPO_DEVSTACK_DIR = ".ceph-devstack" +BUILD_ENV_NAME = "build.env" +ENTRYPOINT_SCRIPT = Path(__file__).with_name("ceph-node-entrypoint.sh") +CLUSTER_ENTRYPOINT_NAME = "ceph-node-entrypoint.sh" +CLUSTER_DATA_NAMES = ("var", "fsid", CLUSTER_ENTRYPOINT_NAME) +CONTAINER_CLUSTER_DIR = "/var/lib/ceph-devstack/cluster" + +DEFAULT_COMPILE_STEPS = { + "cpatch": ["build"], + "container": ["packages"], +} + +CEPH_NODE_CAPABILITIES = [ + "SYS_ADMIN", + "NET_ADMIN", + "SYS_TIME", + "SYS_RAWIO", + "MKNOD", + "NET_RAW", + "SETUID", + "SETGID", + "CHOWN", + "SYS_PTRACE", +] + + +def expand_path(path: str | Path) -> Path: + return Path(os.path.expanduser(str(path))) + + +def git_worktree_info(repo: Path) -> tuple[Path, str] | None: + """Return the main .git directory and worktree name for a linked worktree.""" + git_path = repo.resolve() / ".git" + if not git_path.is_file(): + return None + text = git_path.read_text(encoding="utf-8").strip() + if not text.startswith("gitdir:"): + return None + admin_dir = Path(text.split(":", 1)[1].strip()) + if admin_dir.parent.name != "worktrees": + raise ValueError(f"unexpected git worktree gitdir: {admin_dir}") + main_git_dir = admin_dir.parent.parent + if not main_git_dir.is_dir(): + raise FileNotFoundError(f"git metadata dir not found: {main_git_dir}") + return main_git_dir.resolve(), admin_dir.name + + +def worktree_submodule_git_mounts( + repo: Path, + worktree_name: str, + git_overlay_dir: Path, +) -> list[str]: + """Return podman mounts that rewrite submodule ``.git`` files for ``/ceph``.""" + proc = subprocess.run( + ["git", "-C", str(repo), "submodule", "foreach", "--quiet", "echo $sm_path"], + check=False, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + return [] + + overlay_dir = git_overlay_dir / "submodules" + overlay_dir.mkdir(parents=True, exist_ok=True) + mounts: list[str] = [] + for sm_path in proc.stdout.splitlines(): + sm_path = sm_path.strip() + if not sm_path: + continue + gitdir = ( + f"{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/modules/{sm_path}" + ) + overlay = overlay_dir / f"{sm_path.replace('/', '__')}.git" + overlay.write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + mounts.append(f"--volume={overlay}:{BWC_HOMEDIR}/{sm_path}/.git:Z,ro") + return mounts + + +def worktree_container_mounts( + repo: Path, + main_git_dir: Path, + worktree_name: str, +) -> list[str]: + """Return podman mounts that make a linked worktree usable at ``/ceph``.""" + repo = repo.resolve() + main_git_dir = main_git_dir.resolve() + git_overlay_dir = repo / REPO_DEVSTACK_DIR / "git" + git_overlay_dir.mkdir(parents=True, exist_ok=True) + + dot_git = git_overlay_dir / "dot-git" + dot_git.write_text( + f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}\n", + encoding="utf-8", + ) + + admin_gitdir = git_overlay_dir / "gitdir" + admin_gitdir.write_text(f"{BWC_HOMEDIR}/.git\n", encoding="utf-8") + + mounts = [ + f"--volume={main_git_dir}:{CONTAINER_GIT_METADATA_DIR}:Z,ro", + f"--volume={dot_git}:{BWC_HOMEDIR}/.git:Z,ro", + f"--volume={admin_gitdir}:{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/gitdir:Z,ro", + ] + mounts.extend(worktree_submodule_git_mounts(repo, worktree_name, git_overlay_dir)) + return mounts + + +class CephNode(Container): + """Single-container Ceph cluster without cephadm. + + Builds the runtime image when ``repo`` is configured, creates host loop + devices, and runs mon/mgr/osd inside one capability-scoped container via + ``ceph-node-entrypoint.sh``. + """ + + _name = "ceph_node" + stop_cmd = ["podman", "container", "stop", "{name}"] + remove_cmd = ["podman", "container", "rm", "-f", "{name}"] + start_cmd = ["podman", "container", "start", "{name}"] + exists_cmd = ["podman", "container", "inspect", "{name}"] + + def __init__(self, name: str = ""): + super().__init__(name) + self.loop_device_count = self.config.get("loop_device_count", 3) + self.devices = [self._device_name(i) for i in range(self.loop_device_count)] + + @property + def loop_device_base(self) -> int: + if "loop_device_base" in self.config: + return int(self.config["loop_device_base"]) + testnode = config.get("containers", {}).get("testnode", {}) + count = int(testnode.get("count", 0) or 0) + per_node = int(testnode.get("loop_device_count", 1) or 1) + return count * per_node + + @property + def config_key(self) -> str: + return "ceph_node" + + @property + def cluster_dir(self) -> Path: + return expand_path(self.config.get("output_dir", config["data_dir"])) + + @property + def persistent_cache_dir(self) -> Path: + # Lives beside cluster_dir so ``remove()`` can tear down cluster state + # without wiping npm/sccache/dnf caches. + return self.cluster_dir.parent / "cache" + + @property + def loop_img_dir(self) -> Path: + # Keep loop backing files outside cluster_dir so podman volume relabeling + # does not touch them on container start (same layout as TestNode). + return self.cluster_dir.parent / "disk_images" + + @property + def legacy_loop_img_dir(self) -> Path: + return self.cluster_dir / "disk_images" + + @property + def container_entrypoint(self) -> Path: + return self.cluster_dir / CLUSTER_ENTRYPOINT_NAME + + @property + def container_entrypoint_script(self) -> str: + return f"{self.container_cluster_dir}/{CLUSTER_ENTRYPOINT_NAME}" + + @property + def container_cluster_dir(self) -> str: + return CONTAINER_CLUSTER_DIR + + @property + def mon_id(self) -> str: + return self.config.get("mon_id", "a") + + @property + def mgr_id(self) -> str: + return self.config.get("mgr_id", "x") + + @property + def dashboard_enabled(self) -> bool: + return self.config.get("dashboard", True) is not False + + @property + def dashboard_port(self) -> int: + return int(self.config.get("dashboard_port", 8080)) + + @property + def dashboard_ssl(self) -> bool: + return bool(self.config.get("dashboard_ssl", False)) + + @property + def dashboard_user(self) -> str: + return self.config.get("dashboard_user", "admin") + + @property + def dashboard_password(self) -> str: + return self.config.get("dashboard_password", "admin") + + @property + def dashboard_show_password(self) -> bool: + return self.config.get("dashboard_show_password", False) is True + + @property + def base_image(self) -> str: + return self.config.get("base_image", DEFAULT_BASE_IMAGE) + + @property + def build_subdir(self) -> str: + build_dir = self.config.get("build_dir", "build") + if not build_dir: + return "build" + path = Path(os.path.expanduser(str(build_dir))) + if path.is_absolute() and self.repo: + repo = Path(self.repo).resolve() + try: + return str(path.resolve().relative_to(repo)) + except ValueError: + return path.name + return str(build_dir).strip("/") + + @property + def build_path(self) -> Path: + if not self.repo: + return Path() + return Path(self.repo) / self.build_subdir + + @property + def image_builder(self) -> str: + return self.config.get("image_builder", "cpatch") + + @property + def compile_steps(self) -> List[str]: + default = DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) + return list(self.config.get("build_steps", default)) + + @property + def sccache_enabled(self) -> bool: + return self.config.get("sccache", True) is not False + + @property + def sccache_mode(self) -> str: + return str(self.config.get("sccache_mode", "local")).lower() + + @property + def sccache_debug(self) -> bool: + return self.config.get("sccache_debug", False) is True + + @property + def sccache_cache_path(self) -> Path: + if custom := self.config.get("sccache_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "sccache" + + @property + def npm_cache_enabled(self) -> bool: + return self.config.get("npm_cache", True) is not False + + @property + def npm_cache_path(self) -> Path | None: + if not self.npm_cache_enabled: + return None + if custom := self.config.get("npm_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "npm" + + @property + def dnf_cache_path(self) -> Path | None: + if self.config.get("dnf_cache", False) is not True: + return None + if custom := self.config.get("dnf_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "dnf" + + def _build_cache_args(self) -> list[str]: + """Return build-with-container.py cache flags for persistent build caches.""" + args: list[str] = [] + + npm_cache = self.npm_cache_path + if npm_cache is not None: + npm_cache.mkdir(parents=True, exist_ok=True) + args.extend(["--npm-cache-path", str(npm_cache)]) + + dnf_cache = self.dnf_cache_path + if dnf_cache is not None: + dnf_cache.mkdir(parents=True, exist_ok=True) + args.extend(["--dnf-cache-path", str(dnf_cache)]) + + return args + + @property + def create_cmd(self): + host_cluster_dir = str(self.cluster_dir.resolve()) + container_cluster_dir = self.container_cluster_dir + entrypoint = self.container_entrypoint_script + return [ + "podman", + "container", + "create", + "-i", + "--network", + "host", + "--cap-add", + ",".join(CEPH_NODE_CAPABILITIES), + "--security-opt", + "unmask=/sys/dev/block", + "-v", + f"{host_cluster_dir}:{container_cluster_dir}", + "-v", + f"{host_cluster_dir}/var/lib/ceph:/var/lib/ceph", + "-v", + "/run/udev:/run/udev", + "-v", + "/sys/dev/block:/sys/dev/block", + "-v", + "/dev/fuse:/dev/fuse", + "-v", + "/dev/disk:/dev/disk", + "-v", + "/dev/null:/sys/class/dmi/id/board_serial", + "-v", + "/dev/null:/sys/class/dmi/id/chassis_serial", + "-v", + "/dev/null:/sys/class/dmi/id/product_serial", + "--device", + "/dev/net/tun", + *[f"--device={device}" for device in self.devices], + "-e", + f"CLUSTER_DIR={container_cluster_dir}", + "-e", + f"MON_ID={self.mon_id}", + "-e", + f"MGR_ID={self.mgr_id}", + "-e", + f"OSD_DEVICES={','.join(self.devices)}", + "-e", + f"DASHBOARD_ENABLED={'true' if self.dashboard_enabled else 'false'}", + "-e", + f"DASHBOARD_PORT={self.dashboard_port}", + "-e", + f"DASHBOARD_SSL={'true' if self.dashboard_ssl else 'false'}", + "-e", + f"DASHBOARD_USER={self.dashboard_user}", + "-e", + f"DASHBOARD_PASSWORD={self.dashboard_password}", + "-e", + f"DASHBOARD_SHOW_PASSWORD={'true' if self.dashboard_show_password else 'false'}", + "--entrypoint", + "/bin/bash", + "--name", + "{name}", + "{image}", + "-c", + f". {shlex.quote(entrypoint)}", + ] + + def _device_name(self, index: int) -> str: + return f"/dev/loop{self.loop_device_base + index}" + + def _device_image(self, device: str) -> str: + return f"{self.name}-{device.removeprefix('/dev/loop')}" + + def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: + if not self.sccache_enabled: + return [], [] + + homedir = BWC_HOMEDIR + if custom_conf := self.config.get("sccache_conf"): + conf_src = expand_path(custom_conf) + use_local_cache = self.sccache_mode == "local" + elif self.sccache_mode == "s3": + conf_src = PACKAGE_SCCACHE_S3_CONF + use_local_cache = False + else: + conf_src = PACKAGE_SCCACHE_CONF + use_local_cache = True + if not conf_src.is_file(): + raise FileNotFoundError(f"sccache config not found: {conf_src}") + shutil.copy2(conf_src, repo / "sccache.conf") + + lines = [ + "SCCACHE=true", + f"SCCACHE_CONF={homedir}/sccache.conf", + f"SCCACHE_ERROR_LOG={homedir}/.ceph-devstack/sccache_log.txt", + "CEPH_BUILD_NORMALIZE_PATHS=true", + ] + if self.sccache_debug: + lines.append("SCCACHE_LOG=debug") + + extra_args: list[str] = [] + if use_local_cache: + cache_path = self.sccache_cache_path + cache_path.mkdir(parents=True, exist_ok=True) + extra_args.append(f"--volume={cache_path}:{CONTAINER_SCCACHE_DIR}:Z") + lines.append(f"SCCACHE_DIR={CONTAINER_SCCACHE_DIR}") + cache_size = self.config.get("sccache_cache_size", "100G") + lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") + elif self.sccache_mode == "s3": + lines.extend( + [ + "SCCACHE_S3_NO_CREDENTIALS=true", + "SCCACHE_S3_RW_MODE=READ_ONLY", + ] + ) + return lines, extra_args + + def _prepare_build_env(self) -> tuple[Path | None, list[str]]: + if not self.repo: + return None, [] + repo = Path(self.repo) + extra_args: list[str] = [] + + worktree = git_worktree_info(repo) + if worktree is not None: + main_git_dir, worktree_name = worktree + extra_args.extend( + worktree_container_mounts(repo, main_git_dir, worktree_name) + ) + + lines, sccache_extra = self._sccache_build_env(repo) + extra_args.extend(sccache_extra) + + if not lines: + return None, extra_args + + devstack_dir = repo / REPO_DEVSTACK_DIR + devstack_dir.mkdir(exist_ok=True) + env_path = devstack_dir / BUILD_ENV_NAME + env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return env_path, extra_args + + def _compile_cmd( + self, + env_file: Path | None = None, + extra_args: List[str] | None = None, + ) -> List[str]: + distro = self.config.get("build_distro", "centos9") + script = str(Path(self.repo) / "src/script/build-with-container.py") + cmd = [ + sys.executable, + script, + "-d", + distro, + "-b", + self.build_subdir, + "--homedir", + BWC_HOMEDIR, + ] + for step in self.compile_steps: + cmd.extend(["-e", step]) + if env_file is not None: + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(self._build_cache_args()) + for extra in extra_args or []: + cmd.append(f"--extra={extra}") + return cmd + + def _cpatch_cmd(self) -> List[str]: + return [ + "sudo", + "../src/script/cpatch", + "--base", + self.base_image, + "--target", + self.image, + "--core", + ] + + def _git_value(self, args: str) -> str: + return subprocess.check_output( + f"git {args}".split(), + cwd=self.repo, + text=True, + ).strip() + + async def _run_cmd(self, cmd: List[str], cwd: str): + proc = await host.arun( + cmd, + cwd=Path(cwd).expanduser(), + stream_output=True, + ) + returncode = await proc.wait() + if returncode != 0: + stdout, stderr = await proc.log_failure(cmd) + raise CalledProcessError( + returncode, + cmd, + output=stdout or None, + stderr=stderr or None, + ) + + async def _compile(self): + logger.info( + f"{self.name}: compiling ceph via build-with-container.py in {self.repo}" + ) + env_file, extra_args = self._prepare_build_env() + await self._run_cmd( + self._compile_cmd(env_file=env_file, extra_args=extra_args), + cwd=self.repo, + ) + + def _verify_build_tree(self): + build_path = self.build_path + if (build_path / "build.ninja").exists() or (build_path / "Makefile").exists(): + return + raise FileNotFoundError( + f"Ceph build dir {build_path} missing Makefile or build.ninja " + "after build-with-container.py" + ) + + async def _build_image_cpatch(self): + self._verify_build_tree() + build_path = self.build_path + logger.info(f"{self.name}: building {self.image} via cpatch in {build_path}") + await self._run_cmd(self._cpatch_cmd(), cwd=str(build_path)) + + async def _build_image_container(self): + raise NotImplementedError( + "image_builder='container' requires ceph container/build.sh to consume " + "locally-built packages; enable this once that support lands upstream" + ) + + async def _build_image(self): + builders = { + "cpatch": self._build_image_cpatch, + "container": self._build_image_container, + } + try: + builder = builders[self.image_builder] + except KeyError as exc: + known = ", ".join(sorted(builders)) + raise ValueError( + f"Unknown image_builder {self.image_builder!r}; known: {known}" + ) from exc + await builder() + + def install_entrypoint(self): + if not ENTRYPOINT_SCRIPT.is_file(): + raise FileNotFoundError(f"Entrypoint script not found: {ENTRYPOINT_SCRIPT}") + shutil.copy2(ENTRYPOINT_SCRIPT, self.container_entrypoint) + self.container_entrypoint.chmod(0o755) + + async def label_cluster_dir(self): + if sys.platform == "darwin": + return + await self.cmd( + ["chcon", "-Rt", "container_file_t", str(self.cluster_dir)], + check=False, + ) + + async def remove_legacy_loop_img_dir(self): + legacy = self.legacy_loop_img_dir + if not legacy.is_dir(): + return + logger.info( + f"{self.name}: removing legacy loop image dir at {legacy} " + "(loop backing files now live outside the cluster mount)" + ) + for device in self.devices: + if not host.path_exists(device): + continue + proc = await self.cmd(["losetup", device], check=False) + if proc and await proc.wait() == 0: + await self.cmd(["sudo", "losetup", "-d", device], check=False) + shutil.rmtree(legacy) + + async def remove_cluster_data(self): + cluster_dir = self.cluster_dir.resolve() + if not cluster_dir.exists(): + return + paths = [ + cluster_dir / name + for name in CLUSTER_DATA_NAMES + if (cluster_dir / name).exists() + ] + if not paths: + return + logger.info(f"{self.name}: removing cluster data at {cluster_dir}") + for path in paths: + await self.cmd( + ["podman", "unshare", "rm", "-rf", str(path)], + check=False, + ) + + async def build(self): + if not self.should_build: + return + await self._compile() + await self._build_image() + + async def create(self): + logger.info(f"{self.name}: preparing cluster at {self.cluster_dir}") + self.cluster_dir.mkdir(parents=True, exist_ok=True) + (self.cluster_dir / "var/lib/ceph").mkdir(parents=True, exist_ok=True) + (self.cluster_dir / "var/lib/ceph/mon" / f"ceph-{self.mon_id}").mkdir( + parents=True, exist_ok=True + ) + (self.cluster_dir / "var/lib/ceph/mgr" / f"ceph-{self.mgr_id}").mkdir( + parents=True, exist_ok=True + ) + if not (self.cluster_dir / "fsid").exists(): + (self.cluster_dir / "fsid").write_text(f"{uuid.uuid4()}\n") + self.install_entrypoint() + await self.remove_legacy_loop_img_dir() + await self.label_cluster_dir() + logger.info( + f"{self.name}: creating {self.loop_device_count} loop devices " + f"({self.config.get('loop_device_size', '5G')} each)" + ) + await self.create_loop_devices() + await super().create() + + async def remove(self): + await super().remove() + await self.remove_loop_devices() + await self.remove_legacy_loop_img_dir() + await self.remove_cluster_data() + + async def is_running(self): + if not await self.exists(): + return False + proc = await self.cmd( + [ + "podman", + "exec", + self.name, + "ceph", + "--conf", + f"{self.container_cluster_dir}/ceph.conf", + "-s", + ], + check=False, + ) + return proc is not None and await proc.wait() == 0 + + async def wait(self): + if not await self.exists(): + return 1 + for _ in range(60): + if await self.is_running(): + return 0 + await asyncio.sleep(5) + return 1 + + async def create_loop_devices(self): + for device in self.devices: + await self.create_loop_device(device) + + async def remove_loop_devices(self): + for device in self.devices: + await self.remove_loop_device(device) + + async def create_loop_device(self, device: str): + size = self.config.get("loop_device_size", "5G") + os.makedirs(self.loop_img_dir, exist_ok=True) + proc = await self.cmd( + ["bash", "-c", "lsmod | grep -q loop"], + check=False, + ) + if proc and await proc.wait() != 0: + await self.cmd(["sudo", "modprobe", "loop"]) + loop_img_name = self.loop_img_dir / self._device_image(device) + await self.remove_loop_device(device) + device_pos = device.removeprefix("/dev/loop") + await self.cmd( + [ + "sudo", + "mknod", + "-m700", + device, + "b", + "7", + device_pos, + ], + check=True, + ) + await self.cmd( + ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], + check=True, + ) + await self.cmd( + [ + "sudo", + "dd", + "if=/dev/null", + f"of={loop_img_name}", + "bs=1", + "count=0", + f"seek={size}", + ], + check=True, + ) + await self.cmd(["sudo", "losetup", device, str(loop_img_name)], check=True) + await self.cmd(["chcon", "-t", "fixed_disk_device_t", device]) + device_pos = device.removeprefix("/dev/loop") + await self.cmd( + [ + "sudo", + "udevadm", + "trigger", + "--action=add", + f"--name=block/loop{device_pos}", + ], + check=False, + ) + await self.cmd(["sudo", "udevadm", "settle"], check=False) + + async def remove_loop_device(self, device: str): + loop_img_name = self.loop_img_dir / self._device_image(device) + if os.path.ismount(device): + await self.cmd(["umount", device], check=True) + if host.path_exists(device): + await self.cmd(["sudo", "losetup", "-d", device]) + await self.cmd(["sudo", "rm", "-f", device], check=True) + if loop_img_name.exists(): + loop_img_name.unlink() diff --git a/ceph_devstack/resources/ceph/containers.py b/ceph_devstack/resources/ceph/containers.py index 9e3ea1c7..b90dd391 100644 --- a/ceph_devstack/resources/ceph/containers.py +++ b/ceph_devstack/resources/ceph/containers.py @@ -181,9 +181,7 @@ def __init__(self, name: str = ""): self.index = 0 if "_" in self.name: self.index = int(self.name.split("_")[-1]) - self.loop_device_count = config["containers"]["testnode"].get( - "loop_device_count", 1 - ) + self.loop_device_count = self.config.get("loop_device_count", 1) self.devices = [self.device_name(i) for i in range(self.loop_device_count)] @property @@ -274,7 +272,7 @@ async def remove_loop_devices(self): await self.remove_loop_device(device) async def create_loop_device(self, device: str): - size = config["containers"]["testnode"]["loop_device_size"] + size = self.config.get("loop_device_size", "5G") os.makedirs(self.loop_img_dir, exist_ok=True) proc = await self.cmd(["lsmod", "|", "grep", "loop"]) if proc and await proc.wait() != 0: diff --git a/ceph_devstack/resources/container.py b/ceph_devstack/resources/container.py index 49662b3c..e7e547a1 100644 --- a/ceph_devstack/resources/container.py +++ b/ceph_devstack/resources/container.py @@ -45,9 +45,13 @@ def add_env_to_args(self, args: List): args.insert(-1, f"{key}={value}") return args + @property + def config_key(self) -> str: + return self.__class__.__name__.lower() + @property def config(self): - return config["containers"].get(self.__class__.__name__.lower(), {}) + return config["containers"].get(self.config_key, {}) @property def image_name(self) -> str: @@ -70,15 +74,33 @@ def image_tag(self): @property def repo(self): repo = self.config.get("repo", "") + if not repo: + return "" try: return repo.expanduser() except AttributeError: return os.path.expanduser(repo) + @property + def build_dir(self): + build_dir = self.config.get("build_dir", "") + if not build_dir: + return "" + try: + return build_dir.expanduser() + except AttributeError: + return os.path.expanduser(build_dir) + @property def cwd(self): + if self.build_dir: + return self.build_dir return self.repo or "." + @property + def should_build(self): + return bool(self.repo) + async def pull(self): if not getattr(self, "pull_cmd", None): return @@ -92,9 +114,9 @@ async def pull(self): ) async def build(self): - if not getattr(self, "repo", None): + if not self.should_build: return - logger.debug(f"{self.name}: building from repo: {self.repo}") + logger.debug(f"{self.name}: building from {self.cwd}") await self.cmd( self.format_cmd(self.build_cmd), check=True, diff --git a/ceph_devstack/sccache-s3.conf b/ceph_devstack/sccache-s3.conf new file mode 100644 index 00000000..01db75c7 --- /dev/null +++ b/ceph_devstack/sccache-s3.conf @@ -0,0 +1,8 @@ +[cache.s3] +bucket = "ceph-sccache" +endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud" +use_ssl = true +key_prefix = "" +server_side_encryption = false +no_credentials = true +region = "auto" diff --git a/ceph_devstack/sccache.conf b/ceph_devstack/sccache.conf new file mode 100644 index 00000000..1a9194df --- /dev/null +++ b/ceph_devstack/sccache.conf @@ -0,0 +1,2 @@ +[cache.disk] +dir = "/sccache" diff --git a/pyproject.toml b/pyproject.toml index 95472fb6..0b97e02d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ ceph-devstack = "ceph_devstack.cli:main" [tool.setuptools] include-package-data = true +[tool.setuptools.package-data] +"ceph_devstack.resources.ceph" = ["*.sh"] + [tool.setuptools.packages.find] namespaces = false @@ -56,3 +59,6 @@ requires = [ testpaths = ["tests"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" +markers = [ + "integration: integration tests requiring podman (deselect with '-m \"not integration\"')", +] diff --git a/tests/integration/test_ceph_stack.py b/tests/integration/test_ceph_stack.py new file mode 100644 index 00000000..c7aaae0c --- /dev/null +++ b/tests/integration/test_ceph_stack.py @@ -0,0 +1,24 @@ +"""Ceph stack integration test (podman + live cluster).""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = Path(__file__).with_name("test_ceph_stack.sh") + +pytestmark = pytest.mark.skipif( + os.environ.get("CEPH_DEVSTACK_INTEGRATION") != "1" + or shutil.which("podman") is None, + reason="requires CEPH_DEVSTACK_INTEGRATION=1 and podman", +) + + +@pytest.mark.integration +def test_ceph_stack_reaches_health_ok_and_dashboard() -> None: + subprocess.run(["bash", str(_SCRIPT)], check=True, cwd=_REPO_ROOT) diff --git a/tests/integration/test_ceph_stack.sh b/tests/integration/test_ceph_stack.sh new file mode 100755 index 00000000..92807bb6 --- /dev/null +++ b/tests/integration/test_ceph_stack.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Integration test for the ceph stack: bootstrap a single-node cluster and +# verify HEALTH_OK plus a live dashboard API endpoint. +set -euo pipefail + +STACK=ceph +CONTAINER=ceph_node +CEPH_CONF=/var/lib/ceph-devstack/cluster/ceph.conf +DASHBOARD_PORT=8080 +DASHBOARD_USER=admin +DASHBOARD_PASSWORD=admin + +cd "$(dirname "$0")/../.." + +cleanup() { + local status=$? + if [[ $status -ne 0 ]]; then + echo "=== ${CONTAINER} logs (last 200 lines) ===" >&2 + podman logs "$CONTAINER" 2>&1 | tail -200 >&2 || true + fi + uv run ceph-devstack --stack "$STACK" -v stop || true + uv run ceph-devstack --stack "$STACK" -v remove || true + exit "$status" +} +trap cleanup EXIT + +uv run ceph-devstack --stack "$STACK" -v doctor --fix +uv run ceph-devstack --stack "$STACK" -v pull +uv run ceph-devstack --stack "$STACK" -v create +uv run ceph-devstack --stack "$STACK" -v start + +health="" +for _ in $(seq 1 90); do + health="$( + podman exec "$CONTAINER" ceph --conf "$CEPH_CONF" health 2>/dev/null \ + | awk '{print $1}' || true + )" + if [[ "$health" == "HEALTH_OK" ]]; then + echo "Cluster reached HEALTH_OK" + break + fi + sleep 2 +done + +if [[ "$health" != "HEALTH_OK" ]]; then + echo "Expected HEALTH_OK, got: ${health:-}" >&2 + exit 1 +fi + +dashboard_ok=false +for _ in $(seq 1 30); do + if curl -sf -X POST "http://127.0.0.1:${DASHBOARD_PORT}/api/auth" \ + -H 'Accept: application/vnd.ceph.api.v1.0+json' \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"${DASHBOARD_USER}\",\"password\":\"${DASHBOARD_PASSWORD}\"}" \ + >/dev/null; then + dashboard_ok=true + break + fi + sleep 2 +done + +if [[ "$dashboard_ok" != "true" ]]; then + echo "Dashboard API did not become ready on port ${DASHBOARD_PORT}" >&2 + exit 1 +fi + +echo "Dashboard API is live on port ${DASHBOARD_PORT}" +echo "Ceph stack integration test passed" diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py new file mode 100644 index 00000000..63560e39 --- /dev/null +++ b/tests/resources/ceph/test_ceph_node.py @@ -0,0 +1,435 @@ +from unittest.mock import AsyncMock, patch +import sys + + +from ceph_devstack import config +from ceph_devstack.resources.container import Container +from ceph_devstack.resources.ceph.ceph_node import ( + BUILD_ENV_NAME, + CLUSTER_ENTRYPOINT_NAME, + CONTAINER_CLUSTER_DIR, + CONTAINER_GIT_METADATA_DIR, + CONTAINER_SCCACHE_DIR, + CephNode, + ENTRYPOINT_SCRIPT, + PACKAGE_SCCACHE_CONF, + PACKAGE_SCCACHE_S3_CONF, + REPO_DEVSTACK_DIR, + git_worktree_info, + worktree_container_mounts, +) + + +class TestCephNodeBuild: + def test_image_uses_configured_tag(self): + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + assert CephNode().image == "quay.io/ceph-ci/ceph:main" + + def test_should_build_requires_repo(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + assert CephNode().should_build is True + + def test_should_build_skips_without_repo(self): + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + assert CephNode().should_build is False + + def test_compile_steps_default_for_cpatch(self): + config["containers"]["ceph_node"]["image"] = "example:test" + assert CephNode().compile_steps == ["build"] + + def test_compile_cmd_uses_build_with_container(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["build_distro"] = "centos9" + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["npm_cache"] = False + cmd = CephNode()._compile_cmd() + assert cmd[0] == sys.executable + assert "build-with-container.py" in cmd[1] + assert "-d" in cmd and "centos9" in cmd + assert "-b" in cmd and "build" in cmd + assert cmd[cmd.index("--homedir") + 1] == "/ceph" + assert "--env-file" not in cmd + assert "--npm-cache-path" not in cmd + + def test_compile_cmd_passes_npm_cache_path(self, tmp_path): + npm_cache = tmp_path / "npm-cache" + config["data_dir"] = str(tmp_path / "data") + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["npm_cache_path"] = str(npm_cache) + config["containers"]["ceph_node"]["sccache"] = False + cmd = CephNode()._compile_cmd() + assert "--npm-cache-path" in cmd + assert str(npm_cache.resolve()) in cmd + assert npm_cache.is_dir() + + def test_compile_cmd_uses_default_npm_cache_under_data_dir(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["sccache"] = False + cmd = CephNode()._compile_cmd() + expected = (data_dir.parent / "cache" / "npm").resolve() + assert "--npm-cache-path" in cmd + assert str(expected) in cmd + assert expected.is_dir() + + def test_compile_cmd_skips_ccache_dir(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["npm_cache"] = False + cmd = CephNode()._compile_cmd() + assert "--ccache-dir" not in cmd + + def test_compile_cmd_passes_dnf_cache_path_when_enabled(self, tmp_path): + dnf_cache = tmp_path / "dnf-cache" + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["npm_cache"] = False + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["dnf_cache"] = True + config["containers"]["ceph_node"]["dnf_cache_path"] = str(dnf_cache) + cmd = CephNode()._compile_cmd() + assert "--dnf-cache-path" in cmd + assert str(dnf_cache.resolve()) in cmd + assert dnf_cache.is_dir() + + def test_prepare_build_env_uses_local_sccache_by_default(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + env_file, extra_args = CephNode()._prepare_build_env() + assert env_file == repo / REPO_DEVSTACK_DIR / BUILD_ENV_NAME + assert (repo / "sccache.conf").read_text() == PACKAGE_SCCACHE_CONF.read_text() + contents = env_file.read_text() + assert "SCCACHE=true" in contents + assert "SCCACHE_CONF=/ceph/sccache.conf" in contents + assert "SCCACHE_DIR=/sccache" in contents + assert "SCCACHE_CACHE_SIZE=100G" in contents + assert "SCCACHE_S3_NO_CREDENTIALS" not in contents + assert "SCCACHE_LOG=" not in contents + expected_cache = (data_dir.parent / "cache" / "sccache").resolve() + assert f"--volume={expected_cache}:{CONTAINER_SCCACHE_DIR}:Z" in extra_args + assert expected_cache.is_dir() + + def test_prepare_build_env_enables_sccache_debug_when_configured(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_debug"] = True + env_file, _extra_args = CephNode()._prepare_build_env() + assert "SCCACHE_LOG=debug" in env_file.read_text() + + def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + env_file, extra_args = CephNode()._prepare_build_env() + assert ( + repo / "sccache.conf" + ).read_text() == PACKAGE_SCCACHE_S3_CONF.read_text() + contents = env_file.read_text() + assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents + assert "SCCACHE_S3_RW_MODE=READ_ONLY" in contents + assert "SCCACHE_DIR=" not in contents + assert extra_args == [] + + def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): + custom_conf = tmp_path / "custom-sccache.conf" + custom_conf.write_text("[cache.s3]\nbucket = test\n") + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_conf"] = str(custom_conf) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + env_file, extra_args = CephNode()._prepare_build_env() + assert (repo / "sccache.conf").read_text() == custom_conf.read_text() + contents = env_file.read_text() + assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents + assert extra_args == [] + + def test_prepare_build_env_skips_when_nothing_to_configure(self, tmp_path): + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache"] = False + assert CephNode()._prepare_build_env() == (None, []) + + def test_git_worktree_info_detects_linked_worktree(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + (admin_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + assert git_worktree_info(worktree) == (main_repo / ".git", "ceph_main") + + def test_prepare_build_env_mounts_git_metadata_for_worktree(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(worktree) + config["containers"]["ceph_node"]["sccache"] = False + env_file, extra_args = CephNode()._prepare_build_env() + assert env_file is None + expected_mounts = worktree_container_mounts( + worktree, main_repo / ".git", "ceph_main" + ) + assert extra_args == expected_mounts + dot_git = worktree / REPO_DEVSTACK_DIR / "git" / "dot-git" + assert dot_git.read_text() == ( + f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/ceph_main\n" + ) + + def test_worktree_container_mounts_do_not_set_git_env(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["data_dir"] = str(tmp_path / "data") + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(worktree) + config["containers"]["ceph_node"]["sccache"] = True + env_file, extra_args = CephNode()._prepare_build_env() + assert env_file is not None + contents = env_file.read_text() + assert "GIT_DIR=" not in contents + assert "GIT_WORK_TREE=" not in contents + assert len(extra_args) == 4 + + def test_compile_cmd_passes_worktree_mount_and_env(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(worktree) + config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["build_distro"] = "centos9" + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["npm_cache"] = False + env_file = worktree / REPO_DEVSTACK_DIR / BUILD_ENV_NAME + extra_args = worktree_container_mounts( + worktree, main_repo / ".git", "ceph_main" + ) + cmd = CephNode()._compile_cmd(env_file=env_file, extra_args=extra_args) + assert cmd[cmd.index("--homedir") + 1] == "/ceph" + assert "--env-file" in cmd + assert str(env_file) in cmd + for extra in extra_args: + assert f"--extra={extra}" in cmd + + def test_bundled_sccache_conf_uses_local_disk(self): + contents = PACKAGE_SCCACHE_CONF.read_text() + assert "[cache.disk]" in contents + assert 'dir = "/sccache"' in contents + assert "rw_mode" not in contents + + def test_bundled_sccache_s3_conf_keeps_anonymous_read_settings(self): + contents = PACKAGE_SCCACHE_S3_CONF.read_text() + assert "[cache.s3]" in contents + assert "no_credentials = true" in contents + assert "rw_mode" not in contents + + def test_cpatch_cmd_uses_upstream_script(self): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["base_image"] = "quay.io/ceph-ci/ceph:main" + cmd = CephNode()._cpatch_cmd() + assert cmd[0:2] == ["sudo", "../src/script/cpatch"] + assert "localhost/ceph-devstack:main" in cmd + + async def test_build_runs_compile_then_cpatch(self, tmp_path): + build_path = tmp_path / "build" + build_path.mkdir() + (build_path / "build.ninja").write_text("") + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"]["build_dir"] = "build" + node = CephNode() + with ( + patch.object(node, "_compile", new=AsyncMock()) as mock_compile, + patch.object(node, "_build_image_cpatch", new=AsyncMock()) as mock_cpatch, + ): + await node.build() + mock_compile.assert_awaited_once() + mock_cpatch.assert_awaited_once() + + +class TestCephNodeRuntime: + def test_cluster_dir_defaults_to_stack_data_dir(self, tmp_path): + config["data_dir"] = str(tmp_path) + assert CephNode().cluster_dir == tmp_path + + def test_loop_img_dir_lives_outside_cluster_dir(self, tmp_path): + cluster_dir = tmp_path / "ceph" + config["data_dir"] = str(cluster_dir) + node = CephNode() + assert node.loop_img_dir == tmp_path / "disk_images" + assert node.loop_img_dir != node.cluster_dir / "disk_images" + + def test_create_cmd_uses_host_network_and_entrypoint(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + cmd = CephNode().create_cmd + assert "podman" in cmd + assert "create" in cmd + assert "--network" in cmd + assert "host" in cmd + entrypoint_idx = cmd.index("--entrypoint") + assert cmd[entrypoint_idx + 1] == "/bin/bash" + entrypoint = f"{CONTAINER_CLUSTER_DIR}/{CLUSTER_ENTRYPOINT_NAME}" + assert cmd[cmd.index("-c") + 1] == f". {entrypoint}" + assert f"{tmp_path}:{CONTAINER_CLUSTER_DIR}" in cmd + assert f"{tmp_path}/var/lib/ceph:/var/lib/ceph" in cmd + assert "/run/udev:/run/udev" in cmd + assert f"CLUSTER_DIR={CONTAINER_CLUSTER_DIR}" in cmd + assert f"{tmp_path}:{tmp_path}" not in cmd + assert "CEPH_VOLUME_ALLOW_LOOP_DEVICES" not in cmd + assert "--device=/dev/loop0" in cmd + assert "--device=/dev/loop1" in cmd + assert "--device=/dev/loop2" in cmd + assert "DASHBOARD_PORT=8080" in cmd + assert "DASHBOARD_SSL=false" in cmd + assert "DASHBOARD_SHOW_PASSWORD=false" in cmd + assert "CONTAINER_NAME=ceph_node" in cmd + + def test_dashboard_show_password_when_enabled(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + config["containers"]["ceph_node"]["dashboard_show_password"] = True + assert "DASHBOARD_SHOW_PASSWORD=true" in CephNode().create_cmd + + async def test_create_installs_entrypoint_in_cluster_dir(self, tmp_path): + cluster_dir = tmp_path / "ceph" + config["data_dir"] = str(cluster_dir) + node = CephNode() + with ( + patch.object(node, "remove_legacy_loop_img_dir", new=AsyncMock()), + patch.object(node, "label_cluster_dir", new=AsyncMock()), + patch.object(node, "create_loop_devices", new=AsyncMock()), + patch.object(Container, "create", new=AsyncMock()), + ): + await node.create() + installed = cluster_dir / CLUSTER_ENTRYPOINT_NAME + assert installed.is_file() + assert installed.read_text() == ENTRYPOINT_SCRIPT.read_text() + + async def test_create_removes_legacy_loop_img_dir(self, tmp_path): + cluster_dir = tmp_path / "ceph" + cluster_dir.mkdir() + legacy = cluster_dir / "disk_images" + legacy.mkdir() + (legacy / "ceph_node-0").write_bytes(b"") + config["data_dir"] = str(cluster_dir) + config["containers"]["ceph_node"]["loop_device_count"] = 1 + node = CephNode() + with ( + patch.object(node, "create_loop_devices", new=AsyncMock()), + patch.object(node, "label_cluster_dir", new=AsyncMock()), + patch.object(Container, "create", new=AsyncMock()), + ): + await node.create() + assert not legacy.exists() + + async def test_create_sets_up_loop_devices_and_container(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"]["loop_device_count"] = 2 + node = CephNode() + with ( + patch.object(node, "create_loop_devices", new=AsyncMock()) as mock_loop, + patch.object(Container, "create", new=AsyncMock()) as mock_super_create, + ): + await node.create() + mock_loop.assert_awaited_once() + mock_super_create.assert_awaited_once() + assert (tmp_path / "fsid").exists() + + async def test_remove_tears_down_container_and_loop_devices(self, tmp_path): + cluster_dir = tmp_path / "ceph" + cluster_dir.mkdir() + (cluster_dir / "var" / "lib" / "ceph").mkdir(parents=True) + (cluster_dir / "fsid").write_text("test\n") + npm_cache = tmp_path / "cache" / "npm" + npm_cache.mkdir(parents=True) + (npm_cache / "marker").write_bytes(b"x" * 100) + config["data_dir"] = str(cluster_dir) + node = CephNode() + with ( + patch.object(Container, "remove", new=AsyncMock()) as mock_super_remove, + patch.object(node, "remove_loop_devices", new=AsyncMock()) as mock_loop, + patch.object(node, "remove_legacy_loop_img_dir", new=AsyncMock()), + patch.object(node, "cmd", new=AsyncMock()) as mock_cmd, + ): + await node.remove() + mock_super_remove.assert_awaited_once() + mock_loop.assert_awaited_once() + mock_cmd.assert_any_await( + [ + "podman", + "unshare", + "rm", + "-rf", + str((cluster_dir / "var").resolve()), + ], + check=False, + ) + mock_cmd.assert_any_await( + [ + "podman", + "unshare", + "rm", + "-rf", + str((cluster_dir / "fsid").resolve()), + ], + check=False, + ) + assert npm_cache.exists() + + async def test_is_running_checks_ceph_status_in_container(self, tmp_path): + config["data_dir"] = str(tmp_path) + node = CephNode() + with ( + patch.object(node, "exists", new=AsyncMock(return_value=True)), + patch.object(node, "cmd", new=AsyncMock()) as mock_cmd, + ): + mock_cmd.return_value.wait = AsyncMock(return_value=0) + assert await node.is_running() is True + assert mock_cmd.await_args is not None + exec_cmd = mock_cmd.await_args.args[0] + assert exec_cmd[:3] == ["podman", "exec", "ceph_node"] + assert "ceph" in exec_cmd diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index d30a1625..cf6148ce 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -19,6 +19,7 @@ class TestCephDevStackServiceSpecs: def test_service_specs_includes_all_services(self): devstack = CephDevStack() + assert devstack.stack_name == "teuthology" assert "postgres" in devstack.service_specs assert "paddles" in devstack.service_specs assert "beanstalk" in devstack.service_specs @@ -260,16 +261,15 @@ async def test_remove_calls_remove_on_all_containers(self): mock_network_instance = MagicMock() mock_network_instance.remove = AsyncMock() MockNetwork.return_value = mock_network_instance - with patch("ceph_devstack.resources.ceph.SSHKeyPair") as MockSecret: - mock_secret_instance = MagicMock() - mock_secret_instance.remove = AsyncMock() - MockSecret.return_value = mock_secret_instance - with patch("ceph_devstack.logger.info"): - await devstack.remove() - mock_postgres.remove.assert_called_once() - mock_paddles.remove.assert_called_once() - mock_network_instance.remove.assert_called_once() - mock_secret_instance.remove.assert_called_once() + mock_secret = MagicMock() + mock_secret.remove = AsyncMock() + devstack.secrets = [MagicMock(return_value=mock_secret)] + with patch("ceph_devstack.logger.info"): + await devstack.remove() + mock_postgres.remove.assert_called_once() + mock_paddles.remove.assert_called_once() + mock_network_instance.remove.assert_called_once() + mock_secret.remove.assert_called_once() class TestCephDevStackStop: @@ -422,3 +422,37 @@ def test_init_without_postgres(self): devstack = CephDevStack() assert "archive" in devstack.service_specs assert "postgres" not in devstack.service_specs + + +class TestCephDevStackStacks: + def test_custom_stack_limits_services(self, tmp_path): + config_file = tmp_path / "config.toml" + config_file.write_text( + """ +[stacks.minimal] +services = ["postgres", "paddles"] +""" + ) + config.load(config_file) + devstack = CephDevStack(stack_name="minimal") + assert devstack.stack_name == "minimal" + assert set(devstack.service_specs) == {"postgres", "paddles"} + + def test_ceph_stack_has_expected_services(self): + devstack = CephDevStack(stack_name="ceph") + assert devstack.stack_name == "ceph" + assert set(devstack.service_specs) == {"ceph_node"} + assert devstack.secrets == [] + + async def test_ceph_stack_create_prepares_node(self): + devstack = CephDevStack(stack_name="ceph") + mock_node = AsyncMock() + devstack.service_specs = { + "ceph_node": {"count": 1, "objects": [mock_node]}, + } + with patch("ceph_devstack.resources.ceph.CephDevStackNetwork") as MockNetwork: + mock_network = MagicMock() + mock_network.create = AsyncMock() + MockNetwork.return_value = mock_network + await devstack.create() + mock_node.create.assert_awaited_once() diff --git a/tests/resources/ceph/test_requirements_ceph.py b/tests/resources/ceph/test_requirements_ceph.py index cc4b58a7..92d9f24c 100644 --- a/tests/resources/ceph/test_requirements_ceph.py +++ b/tests/resources/ceph/test_requirements_ceph.py @@ -227,10 +227,10 @@ async def test_check_requirements_returns_true_when_all_pass(self): async def test_check_requirements_returns_false_when_repo_missing(self): devstack = CephDevStack() - devstack.service_specs = {} config["containers"] = { "custom": {"repo": "/nonexistent/path"}, } + devstack.service_specs = {"custom": {}} with ( patch("ceph_devstack.resources.ceph.HasSudo") as MockHasSudo, diff --git a/tests/resources/test_podmanresource.py b/tests/resources/test_podmanresource.py index 3ee8b6e4..2a0d7fcb 100644 --- a/tests/resources/test_podmanresource.py +++ b/tests/resources/test_podmanresource.py @@ -1,3 +1,4 @@ +import logging import pytest from pathlib import Path @@ -54,7 +55,13 @@ async def test_cmd(self, cls): print(m_arun.await_args_list) m_arun.assert_awaited_once_with(["0"], cwd=Path("."), stream_output=False) - async def test_cmd_failed(self, cls): + async def test_cmd_failed(self, cls, caplog): obj = cls() - with pytest.raises(CalledProcessError): - await obj.cmd(["false"], check=True) + with caplog.at_level(logging.ERROR), pytest.raises(CalledProcessError): + await obj.cmd( + ["/bin/sh", "-c", "echo podman-failure >&2; exit 1"], + check=True, + force_local=True, + ) + assert "Command failed (1)" in caplog.text + assert "podman-failure" in caplog.text diff --git a/tests/test_config.py b/tests/test_config.py index 5d11c1dc..83697422 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -103,7 +103,27 @@ def test_unset_value_simple_key(self, test_config): class TestConfigDefaults: def test_config_defaults(self): assert config == { + "stack": "teuthology", "data_dir": "~/.local/share/ceph-devstack", + "stacks": { + "teuthology": { + "services": [ + "postgres", + "paddles", + "beanstalk", + "pulpito", + "teuthology", + "testnode", + "archive", + ], + "secrets": ["ssh_keypair"], + }, + "ceph": { + "services": ["ceph_node"], + "secrets": [], + "data_dir": "~/.local/share/ceph-devstack/ceph", + }, + }, "containers": { "archive": {"image": "python:alpine"}, "beanstalk": {"image": "quay.io/ceph-infra/teuthology-beanstalkd:main"}, @@ -118,5 +138,10 @@ def test_config_defaults(self): "image": "quay.io/ceph-infra/teuthology-testnode:main", }, "teuthology": {"image": "quay.io/ceph-infra/teuthology-dev:main"}, + "ceph_node": { + "image": "quay.io/ceph-ci/ceph:main", + "loop_device_count": 3, + "loop_device_size": "5G", + }, }, } diff --git a/tests/test_stacks.py b/tests/test_stacks.py new file mode 100644 index 00000000..a173f799 --- /dev/null +++ b/tests/test_stacks.py @@ -0,0 +1,62 @@ +import pytest + +from ceph_devstack import config, parse_args + + +class TestApplyStack: + def test_apply_stack_sets_active_stack(self): + config.apply_stack("teuthology") + assert config.active_stack == "teuthology" + + def test_apply_stack_sets_active_services(self): + config.apply_stack("teuthology") + assert "postgres" in config.active_services + assert "teuthology" in config.active_services + + def test_apply_stack_uses_config_default(self): + config.apply_stack() + assert config.active_stack == "teuthology" + + def test_apply_stack_unknown_raises(self): + with pytest.raises(ValueError, match="Unknown stack"): + config.apply_stack("nonexistent") + + def test_apply_stack_merges_container_overrides(self, tmp_path): + config_file = tmp_path / "config.toml" + config_file.write_text( + """ +[stacks.minimal] +services = ["teuthology", "testnode"] + +[stacks.minimal.containers.testnode] +count = 1 +""" + ) + config.load(config_file) + config.apply_stack("minimal") + assert config["containers"]["testnode"]["count"] == 1 + assert config.active_services == ["teuthology", "testnode"] + + def test_apply_stack_merges_data_dir_override(self, tmp_path): + config_file = tmp_path / "config.toml" + config_file.write_text( + """ +[stacks.ceph] +services = ["ceph_node"] +data_dir = "~/.local/share/ceph-devstack/custom-ceph" +""" + ) + config.load(config_file) + config.apply_stack("ceph") + assert config["data_dir"] == "~/.local/share/ceph-devstack/custom-ceph" + + +class TestParseArgsStack: + def test_parse_args_stack(self): + args = parse_args(["--stack", "ceph", "start"]) + assert args.stack == "ceph" + assert args.command == "start" + + def test_parse_args_stack_default_none(self): + args = parse_args(["start"]) + assert args.stack is None From 6148def65e5afc96c8c15671db873202478c73eb Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 7 Jul 2026 23:35:45 -0600 Subject: [PATCH 2/5] Shared block pool --- README.md | 55 +- ceph_devstack/__init__.py | 5 + ceph_devstack/block_pool.py | 496 ++++++++++++++++++ ceph_devstack/cli.py | 7 + ceph_devstack/config.toml | 16 + ceph_devstack/requirements.py | 8 + ceph_devstack/resources/ceph/__init__.py | 4 + ceph_devstack/resources/ceph/block_devices.py | 150 ++++++ .../resources/ceph/ceph-node-entrypoint.sh | 2 +- ceph_devstack/resources/ceph/ceph_node.py | 117 ++--- ceph_devstack/resources/ceph/containers.py | 97 ++-- ceph_devstack/resources/ceph/host_loops.py | 102 ++++ ceph_devstack/resources/ceph/requirements.py | 36 +- tests/conftest.py | 69 ++- tests/resources/ceph/conftest.py | 77 +++ tests/resources/ceph/test_block_devices.py | 97 ++++ tests/resources/ceph/test_ceph_node.py | 14 + .../resources/ceph/test_cephdevstack_core.py | 6 +- tests/resources/ceph/test_host_loops.py | 32 ++ .../resources/ceph/test_requirements_ceph.py | 101 +++- tests/resources/ceph/test_testnode.py | 34 +- tests/test_block_pool.py | 445 ++++++++++++++++ tests/test_cli.py | 25 + tests/test_config.py | 6 + 24 files changed, 1837 insertions(+), 164 deletions(-) create mode 100644 ceph_devstack/block_pool.py create mode 100644 ceph_devstack/resources/ceph/block_devices.py create mode 100644 ceph_devstack/resources/ceph/host_loops.py create mode 100644 tests/resources/ceph/conftest.py create mode 100644 tests/resources/ceph/test_block_devices.py create mode 100644 tests/resources/ceph/test_host_loops.py create mode 100644 tests/test_block_pool.py create mode 100644 tests/test_cli.py diff --git a/README.md b/README.md index bcd0302e..e4dd8e5e 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,59 @@ python3 -m pip install git+https://github.com/zmc/ceph-devstack.git `ceph-devstack config dump` will output the current configuration. +### Shared block pool (optional) + +To back loop devices with slices of a real NVMe device instead of sparse files, +configure `[block_pool]` in your config. The pool is shared across `ceph_node` +and all `testnode_*` containers. Each loop device uses that container's +`loop_device_size` for its backing region on the pool parent. + +```toml +[block_pool] +parent = "/dev/nvme0n1p1" # or a dedicated whole disk +allow_enroll = true # first run only; unset afterward + +[containers.ceph_node] +loop_device_size = "5G" + +[containers.testnode] +loop_device_size = "5G" +``` + +Requirements: + +* Your user must be in the `disk` group and able to read/write the parent + device directly (enrollment writes a tail marker to the raw device). +* The parent must be empty, or already enrolled by ceph-devstack (tail marker + at the end of the device). + +`ceph-devstack block-pool status` shows current allocations. + +Each container picks host loop devices at create time: it needs +`loop_device_count` devices of `loop_device_size`, reuses any existing backing +files for that container name, then takes the lowest free loop numbers after +scanning what is already attached or claimed under `data_dir/disk_images`. + +If `block_pool.json` is lost but the on-disk marker remains, delete the state +file and start again; the pool reclaims the marker without reformatting the +device. + +### Ceph stack + +To run a single-container local Ceph cluster instead of teuthology: + +```bash +ceph-devstack --stack ceph pull +ceph-devstack --stack ceph create +ceph-devstack --stack ceph start +podman logs -f ceph_node +podman exec ceph_node ceph -c /var/lib/ceph-devstack/cluster/ceph.conf -s +``` + +The dashboard listens on port 8080 by default (`admin` / `admin`). Set +`dashboard_show_password = true` under `[containers.ceph_node]` to print the +password in the container log. + As an example, the following configuration will use a local image for paddles with the tag `TEST`; it will also create ten testnode containers; and will build its teuthology container from the git repo at `~/src/teuthology`: ``` containers: @@ -192,4 +245,4 @@ ceph-devstack -v start ```bash ceph-devstack wait teuthology podman logs -f teuthology -``` \ No newline at end of file +``` diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index 3d1a06ed..778c4b59 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -66,6 +66,11 @@ def parse_args(args: List[str]) -> argparse.Namespace: default=False, help="Apply suggested fixes for issues found", ) + parser_block_pool = subparsers.add_parser( + "block-pool", help="Inspect the shared block device pool" + ) + block_pool_subparsers = parser_block_pool.add_subparsers(dest="block_pool_op") + block_pool_subparsers.add_parser("status", help="Show pool allocations and state") parser_pull = subparsers.add_parser("pull", help="Pull container images") parser_pull.add_argument( "image", diff --git a/ceph_devstack/block_pool.py b/ceph_devstack/block_pool.py new file mode 100644 index 00000000..5c25fbcb --- /dev/null +++ b/ceph_devstack/block_pool.py @@ -0,0 +1,496 @@ +import contextlib +import fcntl +import json +import os +import re +import subprocess +import uuid +from pathlib import Path + +from ceph_devstack import logger + + +_SIZE_RE = re.compile(r"^(\d+(?:\.\d+)?)([KMGTP]?)$", re.IGNORECASE) +_UNITS = { + "": 1, + "K": 1024, + "M": 1024**2, + "G": 1024**3, + "T": 1024**4, + "P": 1024**5, +} + +# Written to the last 4 KiB of an enrolled parent so we can recognize our own +# partitions even if block_pool.json is removed. +_POOL_TAIL_MAGIC = b"CEPH-DEVSTACK-BLOCK-POOL-TAIL-v1\x00" +_TAIL_MARKER_SIZE = 4096 +_PROBE_SIZE = 4096 + +# Whole disks and partitions are allowed; device-mapper/loop types are not. +_WHOLE_DISK_NAME = re.compile( + r"^(?:" + r"nvme\d+n\d+|" + r"mmcblk\d+|" + r"(?:sd|[xv]d)[a-z]+" + r")$" +) +_PARTITION_NAME = re.compile( + r"^(?:" + r"nvme\d+n\d+p\d+|" + r"mmcblk\d+p\d+|" + r"(?:sd|[xv]d)[a-z]+\d+" + r")$" +) +_UNSAFE_NAME = re.compile(r"^(?:dm-\d+|md\d+|loop\d+|ram\d+|fd\d+)$") + + +class BlockPoolError(ValueError): + """Raised when a block pool operation would be unsafe.""" + + +def parse_size(value: str | int) -> int: + if isinstance(value, int): + return value + text = str(value).strip().upper() + match = _SIZE_RE.match(text) + if not match: + raise BlockPoolError(f"invalid size: {value!r}") + number, unit = match.groups() + return int(float(number) * _UNITS[unit.upper()]) + + +def format_size(size_bytes: int) -> str: + for unit, multiplier in ( + ("P", 1024**5), + ("T", 1024**4), + ("G", 1024**3), + ("M", 1024**2), + ("K", 1024), + ): + if size_bytes >= multiplier and size_bytes % multiplier == 0: + return f"{size_bytes // multiplier}{unit}" + return f"{size_bytes}B" + + +def canonical_device_path(path: str) -> str: + return os.path.realpath(os.path.expanduser(path)) + + +def validate_parent_name(parent: str) -> str: + """Return canonical parent path or raise if the device is not a safe block device.""" + parent = canonical_device_path(parent) + if not parent.startswith("/dev/"): + raise BlockPoolError(f"block pool parent must be a /dev path, not {parent!r}") + name = os.path.basename(parent) + if _UNSAFE_NAME.match(name): + raise BlockPoolError( + f"block pool parent {parent!r} is not allowed " + "(device-mapper, md, loop, and ram devices are rejected)" + ) + if not (_PARTITION_NAME.match(name) or _WHOLE_DISK_NAME.match(name)): + raise BlockPoolError( + f"block pool parent {parent!r} must be a block device or partition " + "(e.g. /dev/nvme0n1, /dev/nvme0n1p1, or /dev/sda1)" + ) + if not os.path.exists(parent): + raise BlockPoolError(f"block pool parent {parent!r} does not exist") + if not os.path.exists(f"/sys/class/block/{name}"): + raise BlockPoolError(f"block pool parent {parent!r} is not a block device") + return parent + + +def _is_whole_disk(name: str) -> bool: + return bool(_WHOLE_DISK_NAME.match(name)) + + +def _has_partition_siblings(name: str) -> bool: + """True when a whole-disk parent already has partition devices present.""" + if not _is_whole_disk(name): + return False + block_dir = Path("/sys/class/block") + if not block_dir.is_dir(): + return False + for entry in block_dir.iterdir(): + if not entry.is_dir() or entry.name == name: + continue + if name.startswith("nvme") or name.startswith("mmcblk"): + if entry.name.startswith(f"{name}p"): + return True + elif re.match(r"^(?:sd|[xv]d)[a-z]+$", name): + suffix = entry.name.removeprefix(name) + if suffix.isdigit(): + return True + return False + + +def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +def _device_size_bytes(parent: str) -> int: + name = os.path.basename(parent) + sectors_path = f"/sys/class/block/{name}/size" + try: + sectors = int(Path(sectors_path).read_text().strip()) + return sectors * 512 + except OSError: + pass + try: + return os.path.getsize(parent) + except OSError as exc: + raise BlockPoolError(f"cannot read size of {parent!r}: {exc}") from exc + + +def _read_device(parent: str, offset: int, length: int) -> bytes: + with open(parent, "rb") as handle: + handle.seek(offset) + return handle.read(length) + + +def _write_device(parent: str, offset: int, data: bytes) -> None: + with open(parent, "r+b") as handle: + handle.seek(offset) + handle.write(data) + + +def _is_region_empty(data: bytes) -> bool: + return not data or data == b"\x00" * len(data) + + +def _device_mounted(parent: str) -> bool: + proc = _run(["findmnt", "-n", "-o", "TARGET", "--source", parent]) + return proc.returncode == 0 and bool(proc.stdout.strip()) + + +def _device_has_blkid_signature(parent: str) -> bool: + proc = _run(["blkid", "-p", "-o", "export", parent]) + if proc.returncode != 0: + return False + for line in proc.stdout.splitlines(): + key, _, value = line.partition("=") + if key in {"TYPE", "PTTYPE"} and value: + return True + return False + + +def _tail_marker_offset(parent: str) -> int: + size = _device_size_bytes(parent) + if size < _TAIL_MARKER_SIZE: + raise BlockPoolError(f"block pool parent {parent!r} is too small") + return size - _TAIL_MARKER_SIZE + + +def _read_tail_marker(parent: str) -> dict | None: + offset = _tail_marker_offset(parent) + raw = _read_device(parent, offset, _TAIL_MARKER_SIZE) + if not raw.startswith(_POOL_TAIL_MAGIC): + return None + payload = raw[len(_POOL_TAIL_MAGIC) :].split(b"\x00", 1)[0] + if not payload: + return None + try: + return json.loads(payload.decode()) + except json.JSONDecodeError: + return None + + +def _write_tail_marker(parent: str, payload: dict) -> None: + offset = _tail_marker_offset(parent) + encoded = json.dumps(payload, sort_keys=True).encode() + if len(_POOL_TAIL_MAGIC) + len(encoded) + 1 > _TAIL_MARKER_SIZE: + raise BlockPoolError("block pool tail marker payload is too large") + raw = _POOL_TAIL_MAGIC + encoded + b"\x00" + raw = raw.ljust(_TAIL_MARKER_SIZE, b"\x00") + _write_device(parent, offset, raw) + + +def _probe_parent_empty(parent: str) -> None: + size = _device_size_bytes(parent) + offsets = [0] + if size > 512: + offsets.append(512) + if size > _PROBE_SIZE: + offsets.append(size // 2) + tail = _tail_marker_offset(parent) + if tail not in offsets and tail >= _PROBE_SIZE: + offsets.append(tail) + for offset in offsets: + data = _read_device(parent, offset, _PROBE_SIZE) + if not _is_region_empty(data): + raise BlockPoolError( + f"block pool parent {parent!r} is not empty at offset {offset}; " + "refusing to enroll a partition that may contain data" + ) + + +class BlockPool: + """Allocate sized regions from a shared parent block device.""" + + def __init__( + self, + state_path: Path, + parent: str, + *, + allow_enroll: bool = False, + ): + self.state_path = state_path + self.parent = validate_parent_name(parent) + self.allow_enroll = allow_enroll + self._state = self._load() + + @classmethod + def from_config(cls, cfg: dict) -> "BlockPool | None": + pool_cfg = cfg.get("block_pool") or {} + parent = pool_cfg.get("parent") + if not parent: + return None + state_dir = Path( + os.path.expanduser( + pool_cfg.get("state_dir", "~/.local/share/ceph-devstack") + ) + ) + state_path = state_dir / "block_pool.json" + allow_enroll = pool_cfg.get("allow_enroll", False) is True + return cls(state_path, parent, allow_enroll=allow_enroll) + + @classmethod + def status_from_config(cls, cfg: dict) -> int: + pool = cls.from_config(cfg) + if pool is None: + logger.info("block pool: disabled (no parent configured)") + return 0 + logger.info(f"block pool parent: {pool.parent}") + logger.info(f"enrolled: {pool.enrolled}") + logger.info(f"pool id: {pool._state.get('pool_id')}") + logger.info(f"state file: {pool.state_path}") + allocations = pool._state.get("allocations", {}) + if allocations: + logger.info("allocations:") + for key, region in sorted(allocations.items()): + owner = region.get("owner", "?") + logger.info( + f" {key}: offset {region['offset']}, " + f"size {format_size(region['size'])}, last owner {owner}" + ) + else: + logger.info("allocations: (none)") + free_regions = pool._state.get("free_regions", []) + if free_regions: + logger.info("free regions:") + for region in free_regions: + logger.info( + f" offset {region['offset']}, size {format_size(region['size'])}" + ) + else: + logger.info("free regions: (none)") + logger.info(f"next offset: {pool._state.get('next_offset', 0)}") + return 0 + + @property + def enabled(self) -> bool: + return bool(self.parent) + + @property + def enrolled(self) -> bool: + return bool(self._state.get("enrolled")) + + def _load(self) -> dict: + if self.state_path.exists(): + state = json.loads(self.state_path.read_text()) + if state.get("parent") != self.parent: + raise BlockPoolError( + f"block pool parent changed ({state.get('parent')!r} -> " + f"{self.parent!r}); remove {self.state_path} to reconfigure" + ) + if "slice_size" in state: + raise BlockPoolError( + f"legacy block pool state at {self.state_path} uses slice_size; " + "remove it to migrate to per-device allocations" + ) + return state + return self._fresh_state() + + def _fresh_state(self) -> dict: + return { + "parent": self.parent, + "pool_id": str(uuid.uuid4()), + "enrolled": False, + "next_offset": 0, + "free_regions": [], + "allocations": {}, + } + + @contextlib.contextmanager + def _state_lock(self): + self.state_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = self.state_path.with_suffix(".lock") + with open(lock_path, "w") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + if self.state_path.exists(): + self._state = self._load() + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + def _atomic_save(self) -> None: + tmp_path = self.state_path.with_suffix(".tmp") + tmp_path.write_text(json.dumps(self._state, indent=2, sort_keys=True) + "\n") + os.replace(tmp_path, self.state_path) + + def _save(self) -> None: + self._atomic_save() + + def ensure_ready(self) -> None: + """Verify the parent partition is safe to use before any slice I/O.""" + with self._state_lock(): + if self._ensure_ready_unlocked(): + self._atomic_save() + + def _ensure_ready_unlocked(self) -> bool: + if self.enrolled: + self._verify_enrolled_parent() + return False + self._enroll_new_parent() + return True + + def _verify_enrolled_parent(self) -> None: + marker = _read_tail_marker(self.parent) + if marker is None: + raise BlockPoolError( + f"block pool parent {self.parent!r} is missing the devstack " + f"enrollment marker; refusing to use it. If {self.state_path} " + "was removed accidentally, start again to reclaim the marker. " + "If you wiped the device, set block_pool.allow_enroll = true " + "for a fresh enrollment." + ) + if marker.get("pool_id") != self._state.get("pool_id"): + raise BlockPoolError( + f"block pool parent {self.parent!r} belongs to another pool " + f"({marker.get('pool_id')!r}); remove {self.state_path} only " + "if you intentionally reset the pool" + ) + if marker.get("parent") != self.parent: + raise BlockPoolError( + f"block pool parent path mismatch for enrolled partition " + f"({marker.get('parent')!r} != {self.parent!r})" + ) + + def _enroll_new_parent(self) -> None: + marker = _read_tail_marker(self.parent) + if marker is not None: + if marker.get("parent") != self.parent: + raise BlockPoolError( + f"block pool parent {self.parent!r} has a devstack marker " + f"for a different path ({marker.get('parent')!r})" + ) + marker_pool_id = marker.get("pool_id") + if marker_pool_id != self._state.get("pool_id"): + self._state["pool_id"] = marker_pool_id + self._state["enrolled"] = True + return + + if not self.allow_enroll: + raise BlockPoolError( + f"block pool parent {self.parent!r} is not enrolled. " + "Use a dedicated empty partition and set " + "block_pool.allow_enroll = true for the first run only." + ) + if _device_mounted(self.parent): + raise BlockPoolError( + f"block pool parent {self.parent!r} is mounted; refusing to enroll" + ) + if _device_has_blkid_signature(self.parent): + raise BlockPoolError( + f"block pool parent {self.parent!r} has a filesystem or partition " + "table signature; refusing to enroll" + ) + + parent_name = os.path.basename(self.parent) + if _is_whole_disk(parent_name) and _has_partition_siblings(parent_name): + raise BlockPoolError( + f"block pool parent {self.parent!r} is a whole disk with " + "existing partitions; use a dedicated empty disk or a single " + "partition instead" + ) + _probe_parent_empty(self.parent) + + payload = { + "pool_id": self._state["pool_id"], + "parent": self.parent, + } + _write_tail_marker(self.parent, payload) + self._state["enrolled"] = True + + def _validate_region(self, offset: int, size: int, owner: str) -> None: + parent_size = _device_size_bytes(self.parent) + if offset + size > parent_size: + raise BlockPoolError( + f"block pool region for {owner!r} at offset {offset} with size " + f"{size} exceeds parent size {parent_size} for {self.parent!r}" + ) + if offset >= _tail_marker_offset(self.parent): + raise BlockPoolError( + f"block pool region for {owner!r} would overlap the enrollment " + f"marker on {self.parent!r}" + ) + probe = min(_PROBE_SIZE, size) + data = _read_device(self.parent, offset, probe) + if not _is_region_empty(data): + raise BlockPoolError( + f"block pool region for {owner!r} on {self.parent!r} is not empty " + f"at offset {offset}; refusing to allocate over existing data" + ) + + def _pop_free_region(self, size: int) -> dict | None: + free_regions = self._state.setdefault("free_regions", []) + for index, region in enumerate(free_regions): + if region["size"] == size: + return free_regions.pop(index) + return None + + def get_or_allocate(self, owner: str, index: int, size: int) -> tuple[int, int]: + with self._state_lock(): + changed = self._ensure_ready_unlocked() + key = f"{owner}:{index}" + allocations = self._state.setdefault("allocations", {}) + if key in allocations: + region = allocations[key] + if changed: + self._atomic_save() + return region["offset"], region["size"] + + if region := self._pop_free_region(size): + offset = region["offset"] + else: + offset = self._state.setdefault("next_offset", 0) + self._state["next_offset"] = offset + size + + self._validate_region(offset, size, owner) + allocations[key] = {"offset": offset, "size": size, "owner": owner} + self._atomic_save() + return offset, size + + def release_owner(self, owner: str) -> None: + with self._state_lock(): + prefix = f"{owner}:" + allocations = self._state.get("allocations", {}) + free_regions = self._state.setdefault("free_regions", []) + released = [] + for key, region in list(allocations.items()): + if key.startswith(prefix): + released.append( + {"offset": region["offset"], "size": region["size"]} + ) + del allocations[key] + free_regions.extend(released) + free_regions.sort(key=lambda region: region["offset"]) + self._state["free_regions"] = free_regions + self._atomic_save() + + def allocation_for(self, owner: str, index: int) -> tuple[int, int, int] | None: + key = f"{owner}:{index}" + region = self._state.get("allocations", {}).get(key) + if region is None: + return None + return region["offset"], region["size"], region["size"] diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index 1fba02a2..adb751bc 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -68,6 +68,13 @@ def main() -> int: if args.command == "config": CONFIG_HANDLERS[args.config_op](config, args) return 0 + if args.command == "block-pool": + from ceph_devstack.block_pool import BlockPool + + if args.block_pool_op != "status": + logger.error("Usage: ceph-devstack block-pool status") + return 2 + return BlockPool.status_from_config(config) config["args"] = vars(args) Path(config["data_dir"]).expanduser().mkdir(parents=True, exist_ok=True) stack = CephDevStack(stack_name=config.active_stack) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 786b1223..06da229b 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -1,6 +1,20 @@ stack = "teuthology" data_dir = "~/.local/share/ceph-devstack" +[block_pool] +state_dir = "~/.local/share/ceph-devstack" +# Dedicated empty block device or partition. Whole disks are allowed when +# empty; disks with existing partitions must use a partition path instead. +# Enrollment reads/writes the parent directly: your user must be able to +# open the device for read/write (typically membership in the disk group). +# First run requires allow_enroll = true; the tool writes a tail marker and +# refuses non-empty or foreign devices. If block_pool.json is lost but the +# marker remains, delete the state file and start again; the pool reclaims +# the marker automatically. +# parent = "/dev/nvme0n1" +# parent = "/dev/nvme0n1p1" +# allow_enroll = true + [stacks.teuthology] services = [ "postgres", @@ -26,6 +40,7 @@ dashboard_port = 8080 dashboard_ssl = false # dashboard_user = "admin" # dashboard_password = "admin" +# dashboard_show_password = false # log dashboard password on container start # dashboard = false # disable mgr dashboard # image_builder = "cpatch" # default; patch local binaries into runtime image # image_builder = "container" # future: container/build.sh + local packages @@ -66,6 +81,7 @@ image = "quay.io/ceph-infra/pulpito:main" [containers.testnode] count = 3 +loop_device_count = 1 loop_device_size = "5G" image = "quay.io/ceph-infra/teuthology-testnode:main" diff --git a/ceph_devstack/requirements.py b/ceph_devstack/requirements.py index 1291dbea..8024c8d9 100644 --- a/ceph_devstack/requirements.py +++ b/ceph_devstack/requirements.py @@ -297,4 +297,12 @@ async def check_requirements(): result = result and await SysctlValue("fs.aio-max-nr", 2097152).evaluate() result = result and await SysctlValue("kernel.pid_max", 4194304).evaluate() + from ceph_devstack.resources.ceph.requirements import ( + BlockPoolDiskGroup, + BlockPoolParentAccessible, + ) + + result = result and await BlockPoolDiskGroup().evaluate() + result = result and await BlockPoolParentAccessible().evaluate() + return result diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 11dea1b7..2e20a5d0 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -23,6 +23,8 @@ HasSudo, LoopControlDeviceExists, LoopControlDeviceWriteable, + BlockPoolDiskGroup, + BlockPoolParentAccessible, SELinuxModule, ) from ceph_devstack.resources.ceph.utils import get_runs, get_jobs @@ -157,6 +159,8 @@ async def check_requirements(self): if "testnode" in self.service_specs or "ceph_node" in self.service_specs: result = result and await LoopControlDeviceExists().evaluate() result = result and await LoopControlDeviceWriteable().evaluate() + result = result and await BlockPoolDiskGroup().evaluate() + result = result and await BlockPoolParentAccessible().evaluate() # Check for SELinux being enabled and Enforcing; then check for the # presence of our module. If necessary, inform the user and instruct diff --git a/ceph_devstack/resources/ceph/block_devices.py b/ceph_devstack/resources/ceph/block_devices.py new file mode 100644 index 00000000..c83a3243 --- /dev/null +++ b/ceph_devstack/resources/ceph/block_devices.py @@ -0,0 +1,150 @@ +import os +from pathlib import Path +from typing import Protocol + +from ceph_devstack import config, logger +from ceph_devstack.block_pool import BlockPool, BlockPoolError, format_size, parse_size +from ceph_devstack.exec import Subprocess +from ceph_devstack.host import host + + +class CommandRunner(Protocol): + async def __call__( + self, + args: list[str], + *, + check: bool = False, + force_local: bool = False, + stream_output: bool = False, + ) -> Subprocess: ... + + +class BlockDeviceProvisioner: + """Create loop devices backed by sparse files or NVMe pool slices.""" + + def __init__( + self, + owner: str, + *, + image_dir: Path, + file_size: str, + cmd: CommandRunner, + trigger_udev: bool = False, + pool: BlockPool | None = None, + ): + self.owner = owner + self.image_dir = image_dir + self.file_size = file_size + self.device_size = parse_size(file_size) + self.trigger_udev = trigger_udev + self._cmd = cmd + self.pool = pool if pool is not None else BlockPool.from_config(config) + + @property + def pool_enabled(self) -> bool: + return self.pool is not None + + def _require_pool(self) -> BlockPool: + assert self.pool is not None + return self.pool + + def _image_path(self, device: str) -> Path: + loop_id = device.removeprefix("/dev/loop") + return self.image_dir / f"{self.owner}-{loop_id}" + + async def create_devices(self, devices: list[str]) -> None: + for index, device in enumerate(devices): + await self.create_device(device, index) + + async def remove_devices(self, devices: list[str]) -> None: + for device in devices: + await self.remove_device(device) + if self.pool_enabled: + self._require_pool().release_owner(self.owner) + + async def create_device(self, device: str, index: int) -> None: + await self._ensure_loop_module() + await self.remove_device(device) + device_pos = device.removeprefix("/dev/loop") + await self._cmd( + ["sudo", "mknod", "-m700", device, "b", "7", device_pos], + check=True, + ) + await self._cmd( + ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], + check=True, + ) + if self.pool_enabled: + pool = self._require_pool() + try: + offset, _size = pool.get_or_allocate( + self.owner, index, self.device_size + ) + except BlockPoolError: + logger.error( + f"{self.owner}: refusing to use block pool parent {pool.parent!r}" + ) + raise + logger.info( + f"{self.owner}: region on {pool.parent} " + f"(offset={offset}, size={format_size(self.device_size)}) -> {device}" + ) + await self._cmd( + [ + "sudo", + "losetup", + "--offset", + str(offset), + "--sizelimit", + str(self.device_size), + device, + pool.parent, + ], + check=True, + ) + else: + image_path = self._image_path(device) + os.makedirs(self.image_dir, exist_ok=True) + await self._cmd( + [ + "sudo", + "dd", + "if=/dev/null", + f"of={image_path}", + "bs=1", + "count=0", + f"seek={self.file_size}", + ], + check=True, + ) + await self._cmd(["sudo", "losetup", device, str(image_path)], check=True) + + await self._cmd(["chcon", "-t", "fixed_disk_device_t", device], check=False) + if self.trigger_udev: + await self._cmd( + [ + "sudo", + "udevadm", + "trigger", + "--action=add", + f"--name=block/loop{device_pos}", + ], + check=False, + ) + await self._cmd(["sudo", "udevadm", "settle"], check=False) + + async def remove_device(self, device: str) -> None: + if os.path.ismount(device): + await self._cmd(["umount", device], check=True) + if host.path_exists(device): + await self._cmd(["sudo", "losetup", "-d", device], check=False) + await self._cmd(["sudo", "rm", "-f", device], check=False) + if not self.pool_enabled: + image_path = self._image_path(device) + if image_path.exists(): + image_path.unlink() + + async def _ensure_loop_module(self) -> None: + proc = await self._cmd(["bash", "-c", "lsmod | grep -q loop"], check=False) + if await proc.wait() != 0: + await self._cmd(["sudo", "modprobe", "loop"]) diff --git a/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh index ed6dab58..b9e7e3d5 100755 --- a/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh +++ b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh @@ -462,5 +462,5 @@ prepare_osds wait_for_osds wait_for_health -echo "Ceph cluster is running; use: podman exec ${HOSTNAME:-ceph_node} ceph -s" +echo "Ceph cluster is running; use: podman exec ${CONTAINER_NAME:-ceph_node} ceph -s" exec tail -f /dev/null diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index 55318636..75609e6e 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -12,6 +12,8 @@ from ceph_devstack import PROJECT_ROOT, config, logger from ceph_devstack.host import host +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner +from ceph_devstack.resources.ceph.host_loops import allocate_loop_devices from ceph_devstack.resources.container import Container @@ -145,17 +147,17 @@ class CephNode(Container): def __init__(self, name: str = ""): super().__init__(name) - self.loop_device_count = self.config.get("loop_device_count", 3) - self.devices = [self._device_name(i) for i in range(self.loop_device_count)] + self.loop_device_count = self.config["loop_device_count"] + self._devices: list[str] | None = None + self._block_device_provisioner: BlockDeviceProvisioner | None = None @property - def loop_device_base(self) -> int: - if "loop_device_base" in self.config: - return int(self.config["loop_device_base"]) - testnode = config.get("containers", {}).get("testnode", {}) - count = int(testnode.get("count", 0) or 0) - per_node = int(testnode.get("loop_device_count", 1) or 1) - return count * per_node + def devices(self) -> list[str]: + if self._devices is None: + self._devices = allocate_loop_devices( + self.name, self.loop_device_count, self.loop_img_dir + ) + return self._devices @property def config_key(self) -> str: @@ -369,6 +371,8 @@ def create_cmd(self): f"DASHBOARD_PASSWORD={self.dashboard_password}", "-e", f"DASHBOARD_SHOW_PASSWORD={'true' if self.dashboard_show_password else 'false'}", + "-e", + f"CONTAINER_NAME={self.name}", "--entrypoint", "/bin/bash", "--name", @@ -378,9 +382,6 @@ def create_cmd(self): f". {shlex.quote(entrypoint)}", ] - def _device_name(self, index: int) -> str: - return f"/dev/loop{self.loop_device_base + index}" - def _device_image(self, device: str) -> str: return f"{self.name}-{device.removeprefix('/dev/loop')}" @@ -629,7 +630,7 @@ async def create(self): await self.label_cluster_dir() logger.info( f"{self.name}: creating {self.loop_device_count} loop devices " - f"({self.config.get('loop_device_size', '5G')} each)" + f"({self.config['loop_device_size']} each)" ) await self.create_loop_devices() await super().create() @@ -666,75 +667,27 @@ async def wait(self): await asyncio.sleep(5) return 1 + def _block_provisioner(self) -> BlockDeviceProvisioner: + if self._block_device_provisioner is None: + self._block_device_provisioner = BlockDeviceProvisioner( + self.name, + image_dir=self.loop_img_dir, + file_size=self.config["loop_device_size"], + cmd=self.cmd, + trigger_udev=True, + ) + return self._block_device_provisioner + async def create_loop_devices(self): - for device in self.devices: - await self.create_loop_device(device) + if self.devices: + numbers = [int(device.removeprefix("/dev/loop")) for device in self.devices] + logger.info( + f"{self.name}: host loop devices " + f"{numbers[0]}-{numbers[-1]} " + f"({self.config['loop_device_size']} each)" + ) + await self._block_provisioner().create_devices(self.devices) async def remove_loop_devices(self): - for device in self.devices: - await self.remove_loop_device(device) - - async def create_loop_device(self, device: str): - size = self.config.get("loop_device_size", "5G") - os.makedirs(self.loop_img_dir, exist_ok=True) - proc = await self.cmd( - ["bash", "-c", "lsmod | grep -q loop"], - check=False, - ) - if proc and await proc.wait() != 0: - await self.cmd(["sudo", "modprobe", "loop"]) - loop_img_name = self.loop_img_dir / self._device_image(device) - await self.remove_loop_device(device) - device_pos = device.removeprefix("/dev/loop") - await self.cmd( - [ - "sudo", - "mknod", - "-m700", - device, - "b", - "7", - device_pos, - ], - check=True, - ) - await self.cmd( - ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], - check=True, - ) - await self.cmd( - [ - "sudo", - "dd", - "if=/dev/null", - f"of={loop_img_name}", - "bs=1", - "count=0", - f"seek={size}", - ], - check=True, - ) - await self.cmd(["sudo", "losetup", device, str(loop_img_name)], check=True) - await self.cmd(["chcon", "-t", "fixed_disk_device_t", device]) - device_pos = device.removeprefix("/dev/loop") - await self.cmd( - [ - "sudo", - "udevadm", - "trigger", - "--action=add", - f"--name=block/loop{device_pos}", - ], - check=False, - ) - await self.cmd(["sudo", "udevadm", "settle"], check=False) - - async def remove_loop_device(self, device: str): - loop_img_name = self.loop_img_dir / self._device_image(device) - if os.path.ismount(device): - await self.cmd(["umount", device], check=True) - if host.path_exists(device): - await self.cmd(["sudo", "losetup", "-d", device]) - await self.cmd(["sudo", "rm", "-f", device], check=True) - if loop_img_name.exists(): - loop_img_name.unlink() + await self._block_provisioner().remove_devices(self.devices) + self._devices = None diff --git a/ceph_devstack/resources/ceph/containers.py b/ceph_devstack/resources/ceph/containers.py index b90dd391..51c71958 100644 --- a/ceph_devstack/resources/ceph/containers.py +++ b/ceph_devstack/resources/ceph/containers.py @@ -4,8 +4,10 @@ from pathlib import Path from typing import List -from ceph_devstack import config, DEFAULT_CONFIG_PATH +from ceph_devstack import config, DEFAULT_CONFIG_PATH, logger from ceph_devstack.host import host +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner +from ceph_devstack.resources.ceph.host_loops import allocate_loop_devices from ceph_devstack.resources.container import Container @@ -181,8 +183,17 @@ def __init__(self, name: str = ""): self.index = 0 if "_" in self.name: self.index = int(self.name.split("_")[-1]) - self.loop_device_count = self.config.get("loop_device_count", 1) - self.devices = [self.device_name(i) for i in range(self.loop_device_count)] + self.loop_device_count = self.config["loop_device_count"] + self._devices: list[str] | None = None + self._block_device_provisioner: BlockDeviceProvisioner | None = None + + @property + def devices(self) -> list[str]: + if self._devices is None: + self._devices = allocate_loop_devices( + self.name, self.loop_device_count, self.loop_img_dir + ) + return self._devices @property def loop_img_dir(self): @@ -263,69 +274,29 @@ async def remove(self): await super().remove() await self.remove_loop_devices() + def _block_provisioner(self) -> BlockDeviceProvisioner: + if self._block_device_provisioner is None: + self._block_device_provisioner = BlockDeviceProvisioner( + self.name, + image_dir=self.loop_img_dir, + file_size=self.config["loop_device_size"], + cmd=self.cmd, + ) + return self._block_device_provisioner + async def create_loop_devices(self): - for device in self.devices: - await self.create_loop_device(device) + if self.devices: + numbers = [int(device.removeprefix("/dev/loop")) for device in self.devices] + logger.info( + f"{self.name}: host loop devices " + f"{numbers[0]}-{numbers[-1]} " + f"({self.config['loop_device_size']} each)" + ) + await self._block_provisioner().create_devices(self.devices) async def remove_loop_devices(self): - for device in self.devices: - await self.remove_loop_device(device) - - async def create_loop_device(self, device: str): - size = self.config.get("loop_device_size", "5G") - os.makedirs(self.loop_img_dir, exist_ok=True) - proc = await self.cmd(["lsmod", "|", "grep", "loop"]) - if proc and await proc.wait() != 0: - await self.cmd(["sudo", "modprobe", "loop"]) - loop_img_name = os.path.join(self.loop_img_dir, self.device_image(device)) - await self.remove_loop_device(device) - device_pos = device.removeprefix("/dev/loop") - await self.cmd( - [ - "sudo", - "mknod", - "-m700", - device, - "b", - "7", - device_pos, - ], - check=True, - ) - await self.cmd( - ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], - check=True, - ) - await self.cmd( - [ - "sudo", - "dd", - "if=/dev/null", - f"of={loop_img_name}", - "bs=1", - "count=0", - f"seek={size}", - ], - check=True, - ) - await self.cmd(["sudo", "losetup", device, loop_img_name], check=True) - await self.cmd(["chcon", "-t", "fixed_disk_device_t", device]) - - async def remove_loop_device(self, device: str): - loop_img_name = os.path.join(self.loop_img_dir, self.device_image(device)) - if os.path.ismount(device): - await self.cmd(["umount", device], check=True) - if host.path_exists(device): - await self.cmd(["sudo", "losetup", "-d", device]) - await self.cmd(["sudo", "rm", "-f", device], check=True) - if host.path_exists(loop_img_name): - os.remove(loop_img_name) - - def device_name(self, index: int): - return f"/dev/loop{self.loop_device_count * self.index + index}" - - def device_image(self, device: str): - return f"{self.name}-{device.removeprefix('/dev/loop')}" + await self._block_provisioner().remove_devices(self.devices) + self._devices = None class Teuthology(Container): diff --git a/ceph_devstack/resources/ceph/host_loops.py b/ceph_devstack/resources/ceph/host_loops.py new file mode 100644 index 00000000..07bcfe77 --- /dev/null +++ b/ceph_devstack/resources/ceph/host_loops.py @@ -0,0 +1,102 @@ +"""Host loop device discovery and allocation.""" + +from pathlib import Path + + +def host_loop_path(number: int) -> str: + return f"/dev/loop{number}" + + +def _loop_number_from_backing_name(name: str) -> int | None: + if "-" not in name: + return None + loop_id = name.rsplit("-", 1)[-1] + if not loop_id.isdigit(): + return None + return int(loop_id) + + +def _sysfs_loop_in_use(block_name: str) -> bool: + block = Path("/sys/class/block") / block_name + backing = block / "loop" / "backing_file" + if backing.is_file(): + text = backing.read_text().strip() + if text not in ("", "(deleted)"): + return True + size_file = block / "size" + if not size_file.is_file(): + return False + try: + return int(size_file.read_text()) > 0 + except ValueError: + return False + + +def _discover_used_from_sysfs() -> set[int]: + used: set[int] = set() + block_dir = Path("/sys/class/block") + if not block_dir.is_dir(): + return used + for entry in block_dir.iterdir(): + name = entry.name + if not name.startswith("loop"): + continue + suffix = name.removeprefix("loop") + if suffix.isdigit() and _sysfs_loop_in_use(name): + used.add(int(suffix)) + return used + + +def _discover_used_from_backing_files(image_dir: Path) -> set[int]: + used: set[int] = set() + if not image_dir.is_dir(): + return used + for path in image_dir.iterdir(): + if not path.is_file(): + continue + if (number := _loop_number_from_backing_name(path.name)) is not None: + used.add(number) + return used + + +def discover_used_loop_numbers(image_dir: Path) -> set[int]: + """Loop numbers already attached on the host or claimed by backing files.""" + return _discover_used_from_sysfs() | _discover_used_from_backing_files(image_dir) + + +def owner_loop_numbers(owner: str, image_dir: Path) -> list[int]: + """Existing backing-file loop numbers for an owner, lowest first.""" + if not image_dir.is_dir(): + return [] + numbers: list[int] = [] + prefix = f"{owner}-" + for path in image_dir.iterdir(): + if not path.is_file() or not path.name.startswith(prefix): + continue + if (number := _loop_number_from_backing_name(path.name)) is not None: + numbers.append(number) + return sorted(numbers) + + +def allocate_loop_numbers(owner: str, count: int, image_dir: Path) -> list[int]: + """Pick loop numbers for an owner from config count and host availability.""" + used = discover_used_loop_numbers(image_dir) + reclaimed = owner_loop_numbers(owner, image_dir) + numbers: list[int] = [] + for index in range(count): + if index < len(reclaimed): + numbers.append(reclaimed[index]) + continue + candidate = 0 + while candidate in used or candidate in numbers: + candidate += 1 + numbers.append(candidate) + used.add(candidate) + return numbers + + +def allocate_loop_devices(owner: str, count: int, image_dir: Path) -> list[str]: + return [ + host_loop_path(number) + for number in allocate_loop_numbers(owner, count, image_dir) + ] diff --git a/ceph_devstack/resources/ceph/requirements.py b/ceph_devstack/resources/ceph/requirements.py index 00d79678..7bae9982 100644 --- a/ceph_devstack/resources/ceph/requirements.py +++ b/ceph_devstack/resources/ceph/requirements.py @@ -1,5 +1,6 @@ -from ceph_devstack import logger, PROJECT_ROOT +from ceph_devstack import logger, PROJECT_ROOT, config from ceph_devstack.requirements import Requirement, FixableRequirement +import os class HasSudo(Requirement): @@ -42,6 +43,39 @@ async def suggest(self): ) +class BlockPoolDiskGroup(Requirement): + suggest_msg = "block pool parent requires membership in the disk group" + + async def check(self) -> bool: + from ceph_devstack.block_pool import BlockPool + + if BlockPool.from_config(config) is None: + return True + proc = await self.host.arun(["bash", "-c", "id -nG | grep -qw disk"]) + if await proc.wait() == 0: + return True + logger.error(f"{self.suggest_msg}. Try: sudo usermod -a -G disk $USER") + return False + + +class BlockPoolParentAccessible(Requirement): + suggest_msg = "block pool parent must be readable and writable by this user" + + async def check(self) -> bool: + from ceph_devstack.block_pool import BlockPool + + pool = BlockPool.from_config(config) + if pool is None: + return True + if os.access(pool.parent, os.R_OK | os.W_OK): + return True + logger.error( + f"{self.suggest_msg} ({pool.parent}). " + "Join the disk group and re-login, or fix device permissions." + ) + return False + + class SELinuxModule(FixableRequirement): def __init__(self): fix_cmd = self.fix_cmd_prebuilt diff --git a/tests/conftest.py b/tests/conftest.py index 6dfea423..fb5a7288 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,14 @@ import os import pathlib -import pytest import random - +from collections.abc import Callable from datetime import datetime, timedelta +from unittest.mock import AsyncMock, patch + +import pytest from ceph_devstack import config +from ceph_devstack.block_pool import BlockPool @pytest.fixture(autouse=True) @@ -14,6 +17,68 @@ def reset_config(): yield +@pytest.fixture +def mock_cmd() -> AsyncMock: + return AsyncMock() + + +@pytest.fixture +def pool_parent(tmp_path: pathlib.Path) -> pathlib.Path: + parent = tmp_path / "nvme0n1p1" + parent.write_bytes(b"\x00" * (20 * 1024**2)) + return parent + + +@pytest.fixture +def mock_fresh_enrollment(): + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=False), + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", return_value=False + ), + patch("ceph_devstack.block_pool._read_tail_marker", return_value=None), + ): + yield + + +@pytest.fixture +def block_pool_factory( + tmp_path: pathlib.Path, pool_parent: pathlib.Path +) -> Callable[..., BlockPool]: + def factory(*, enrolled: bool = False, allow_enroll: bool = True) -> BlockPool: + state_path = tmp_path / "block_pool.json" + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(pool_parent), + ): + pool = BlockPool( + state_path, + str(pool_parent), + allow_enroll=allow_enroll, + ) + if enrolled: + pool._state["enrolled"] = True + pool._save() + return pool + + return factory + + +@pytest.fixture +def block_pool(block_pool_factory: Callable[..., BlockPool]) -> BlockPool: + return block_pool_factory() + + +@pytest.fixture +def enrolled_pool(block_pool_factory: Callable[..., BlockPool]) -> BlockPool: + return block_pool_factory(enrolled=True) + + +@pytest.fixture +def disallow_enroll_pool(block_pool_factory: Callable[..., BlockPool]) -> BlockPool: + return block_pool_factory(allow_enroll=False) + + @pytest.fixture(scope="function") def create_log_file(): def _create_log_file(data_dir: pathlib.Path, **kwargs) -> pathlib.Path: diff --git a/tests/resources/ceph/conftest.py b/tests/resources/ceph/conftest.py new file mode 100644 index 00000000..3d8e4573 --- /dev/null +++ b/tests/resources/ceph/conftest.py @@ -0,0 +1,77 @@ +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +from ceph_devstack import config +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner + + +@pytest.fixture +def pool_config(tmp_path: Path, pool_parent: Path) -> None: + config["block_pool"] = { + "parent": str(pool_parent), + "state_dir": str(tmp_path), + "allow_enroll": True, + } + config["containers"]["ceph_node"]["loop_device_size"] = "1M" + config["containers"]["testnode"]["loop_device_size"] = "1M" + + +@pytest.fixture +def pool_provisioner( + tmp_path: Path, + pool_parent: Path, + pool_config: None, + mock_cmd: AsyncMock, +) -> BlockDeviceProvisioner: + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(pool_parent), + ): + return BlockDeviceProvisioner( + "ceph_node", + image_dir=tmp_path / "disk_images", + file_size="1M", + cmd=mock_cmd, + ) + + +@pytest.fixture +def ready_pool_provisioner( + pool_provisioner: BlockDeviceProvisioner, + mock_fresh_enrollment: None, +) -> Iterator[BlockDeviceProvisioner]: + with patch.object(pool_provisioner, "remove_device", new=AsyncMock()): + yield pool_provisioner + + +@pytest.fixture +def sparse_provisioner(tmp_path: Path, mock_cmd: AsyncMock) -> BlockDeviceProvisioner: + return BlockDeviceProvisioner( + "testnode_0", + image_dir=tmp_path / "disk_images", + file_size="1M", + cmd=mock_cmd, + pool=None, + ) + + +@pytest.fixture +def large_pool_provisioner( + tmp_path: Path, + pool_parent: Path, + pool_config: None, + mock_cmd: AsyncMock, +) -> BlockDeviceProvisioner: + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(pool_parent), + ): + return BlockDeviceProvisioner( + "ceph_node", + image_dir=tmp_path / "disk_images", + file_size="5M", + cmd=mock_cmd, + ) diff --git a/tests/resources/ceph/test_block_devices.py b/tests/resources/ceph/test_block_devices.py new file mode 100644 index 00000000..91de7db8 --- /dev/null +++ b/tests/resources/ceph/test_block_devices.py @@ -0,0 +1,97 @@ +from unittest.mock import AsyncMock, patch + +import pytest + +from ceph_devstack.block_pool import BlockPoolError +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner + + +class TestBlockDeviceProvisioner: + async def test_create_device_uses_pool_slice( + self, + ready_pool_provisioner: BlockDeviceProvisioner, + mock_cmd: AsyncMock, + pool_parent, + ): + await ready_pool_provisioner.create_device("/dev/loop9", 0) + losetup = [ + call.args[0] + for call in mock_cmd.await_args_list + if "losetup" in call.args[0] + ][0] + assert "--offset" in losetup + assert "0" in losetup + assert "--sizelimit" in losetup + assert str(ready_pool_provisioner.device_size) in losetup + assert str(pool_parent) in losetup + + async def test_remove_devices_releases_pool_owner( + self, + ready_pool_provisioner: BlockDeviceProvisioner, + ): + await ready_pool_provisioner.create_device("/dev/loop9", 0) + await ready_pool_provisioner.remove_devices(["/dev/loop9"]) + assert ( + ready_pool_provisioner._require_pool().allocation_for("ceph_node", 0) + is None + ) + + async def test_create_device_uses_sparse_file_without_pool( + self, + sparse_provisioner: BlockDeviceProvisioner, + mock_cmd: AsyncMock, + tmp_path, + ): + with ( + patch.object(sparse_provisioner, "remove_device", new=AsyncMock()), + patch.object(sparse_provisioner, "_ensure_loop_module", new=AsyncMock()), + ): + await sparse_provisioner.create_device("/dev/loop1", 0) + dd_calls = [ + call.args[0] for call in mock_cmd.await_args_list if "dd" in call.args[0] + ] + assert dd_calls + image_path = str(tmp_path / "disk_images" / "testnode_0-1") + assert any(image_path in arg for arg in dd_calls[0]) + + async def test_rejects_region_larger_than_parent( + self, + large_pool_provisioner: BlockDeviceProvisioner, + mock_fresh_enrollment: None, + ): + with ( + patch( + "ceph_devstack.block_pool._device_size_bytes", + return_value=512 * 1024, + ), + patch.object(large_pool_provisioner, "remove_device", new=AsyncMock()), + pytest.raises(BlockPoolError, match="exceeds"), + ): + await large_pool_provisioner.create_device("/dev/loop9", 0) + + async def test_ensure_loop_module_loads_when_missing( + self, + sparse_provisioner: BlockDeviceProvisioner, + mock_cmd: AsyncMock, + ): + mock_proc = AsyncMock() + mock_proc.wait = AsyncMock(return_value=1) + mock_cmd.return_value = mock_proc + await sparse_provisioner._ensure_loop_module() + mock_cmd.assert_any_await(["sudo", "modprobe", "loop"]) + + async def test_pool_error_is_logged_and_reraised( + self, + pool_provisioner: BlockDeviceProvisioner, + mock_fresh_enrollment: None, + ): + with ( + patch.object( + pool_provisioner._require_pool(), + "get_or_allocate", + side_effect=BlockPoolError("refused"), + ), + patch.object(pool_provisioner, "remove_device", new=AsyncMock()), + pytest.raises(BlockPoolError, match="refused"), + ): + await pool_provisioner.create_device("/dev/loop9", 0) diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index 63560e39..f7fb83ec 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -291,6 +291,20 @@ async def test_build_runs_compile_then_cpatch(self, tmp_path): class TestCephNodeRuntime: + def test_devices_allocate_from_empty_host(self, tmp_path): + config["data_dir"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["loop_device_count"] = 3 + assert CephNode().devices == ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + + def test_devices_skip_loops_already_claimed(self, tmp_path): + config["data_dir"] = str(tmp_path / "ceph") + image_dir = tmp_path / "disk_images" + image_dir.mkdir(parents=True) + (image_dir / "testnode_0-0").write_bytes(b"") + (image_dir / "testnode_0-1").write_bytes(b"") + config["containers"]["ceph_node"]["loop_device_count"] = 2 + assert CephNode().devices == ["/dev/loop2", "/dev/loop3"] + def test_cluster_dir_defaults_to_stack_data_dir(self, tmp_path): config["data_dir"] = str(tmp_path) assert CephNode().cluster_dir == tmp_path diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index cf6148ce..7acf6e77 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -415,7 +415,11 @@ def test_init_without_postgres(self): "paddles": {"image": "paddles:latest", "count": 1}, "beanstalk": {"image": "beanstalk:latest", "count": 1}, "pulpito": {"image": "pulpito:latest", "count": 1}, - "testnode": {"image": "testnode:latest", "count": 3}, + "testnode": { + "image": "testnode:latest", + "count": 3, + "loop_device_count": 1, + }, "teuthology": {"image": "teuthology:latest", "count": 1}, "archive": {"image": "archive:latest", "count": 1}, } diff --git a/tests/resources/ceph/test_host_loops.py b/tests/resources/ceph/test_host_loops.py new file mode 100644 index 00000000..22de595b --- /dev/null +++ b/tests/resources/ceph/test_host_loops.py @@ -0,0 +1,32 @@ +from ceph_devstack.resources.ceph.host_loops import ( + allocate_loop_numbers, + discover_used_loop_numbers, + host_loop_path, + owner_loop_numbers, +) + + +class TestHostLoops: + def test_host_loop_path(self): + assert host_loop_path(21) == "/dev/loop21" + + def test_discover_used_loop_numbers_from_backing_files(self, tmp_path): + (tmp_path / "testnode_0-1").write_bytes(b"") + (tmp_path / "ceph_node-4").write_bytes(b"") + (tmp_path / "ignore-me").write_bytes(b"") + assert discover_used_loop_numbers(tmp_path) == {1, 4} + + def test_owner_loop_numbers(self, tmp_path): + (tmp_path / "testnode_1-2").write_bytes(b"") + (tmp_path / "testnode_1-0").write_bytes(b"") + assert owner_loop_numbers("testnode_1", tmp_path) == [0, 2] + + def test_allocate_loop_numbers_uses_lowest_free(self, tmp_path): + (tmp_path / "other-0").write_bytes(b"") + assert allocate_loop_numbers("testnode_0", 2, tmp_path) == [1, 2] + + def test_allocate_loop_numbers_reuses_owner_backing_files(self, tmp_path): + (tmp_path / "testnode_1-4").write_bytes(b"") + (tmp_path / "testnode_1-1").write_bytes(b"") + (tmp_path / "other-0").write_bytes(b"") + assert allocate_loop_numbers("testnode_1", 3, tmp_path) == [1, 4, 2] diff --git a/tests/resources/ceph/test_requirements_ceph.py b/tests/resources/ceph/test_requirements_ceph.py index 92d9f24c..07388e7f 100644 --- a/tests/resources/ceph/test_requirements_ceph.py +++ b/tests/resources/ceph/test_requirements_ceph.py @@ -7,6 +7,8 @@ HasSudo, LoopControlDeviceExists, LoopControlDeviceWriteable, + BlockPoolDiskGroup, + BlockPoolParentAccessible, SELinuxModule, ) @@ -200,11 +202,56 @@ class MockRemoteHost: assert req.fix_cmd[7].endswith("ceph_devstack.pp") +class TestBlockPoolRequirements: + async def test_disk_group_skipped_when_pool_disabled(self): + config.pop("block_pool", None) + req = BlockPoolDiskGroup() + assert await req.check() is True + + async def test_disk_group_required_when_pool_configured(self): + config["block_pool"] = {"parent": "/dev/nvme0n1p1"} + req = BlockPoolDiskGroup() + mock_proc = AsyncMock() + mock_proc.wait = AsyncMock(return_value=0) + with ( + patch( + "ceph_devstack.block_pool.BlockPool.from_config", + return_value=object(), + ), + patch.object(req.host, "arun", return_value=mock_proc), + ): + assert await req.check() is True + + async def test_parent_accessible_when_readable(self): + config["block_pool"] = {"parent": "/dev/nvme0n1p1"} + req = BlockPoolParentAccessible() + with ( + patch( + "ceph_devstack.block_pool.BlockPool.from_config", + return_value=type("Pool", (), {"parent": "/dev/nvme0n1p1"})(), + ), + patch("os.access", return_value=True), + ): + assert await req.check() is True + + async def test_parent_accessible_fails_when_not_writable(self): + config["block_pool"] = {"parent": "/dev/nvme0n1p1"} + req = BlockPoolParentAccessible() + with ( + patch( + "ceph_devstack.block_pool.BlockPool.from_config", + return_value=type("Pool", (), {"parent": "/dev/nvme0n1p1"})(), + ), + patch("os.access", return_value=False), + ): + assert await req.check() is False + + class TestCephDevStackCheckRequirements: async def test_check_requirements_returns_true_when_all_pass(self): devstack = CephDevStack() - devstack.service_specs = {} - config["containers"] = {} + devstack.service_specs = {"ceph_node": {}} + config["containers"] = {"ceph_node": {}} with ( patch("ceph_devstack.resources.ceph.HasSudo") as MockHasSudo, @@ -214,6 +261,12 @@ async def test_check_requirements_returns_true_when_all_pass(self): patch( "ceph_devstack.resources.ceph.LoopControlDeviceWriteable" ) as MockLoopCtrlWrite, + patch( + "ceph_devstack.resources.ceph.BlockPoolDiskGroup" + ) as MockBlockPoolDiskGroup, + patch( + "ceph_devstack.resources.ceph.BlockPoolParentAccessible" + ) as MockBlockPoolParentAccessible, patch("ceph_devstack.host.host.selinux_enforcing") as mock_selinux, ): MockHasSudo.return_value = AsyncMock(evaluate=AsyncMock(return_value=True)) @@ -221,10 +274,54 @@ async def test_check_requirements_returns_true_when_all_pass(self): MockLoopCtrlWrite.return_value = AsyncMock( evaluate=AsyncMock(return_value=True) ) + MockBlockPoolDiskGroup.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) + MockBlockPoolParentAccessible.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) mock_selinux.return_value = False result = await devstack.check_requirements() assert result is True + async def test_check_requirements_returns_false_when_block_pool_inaccessible( + self, + ): + devstack = CephDevStack() + devstack.service_specs = {"ceph_node": {}} + config["containers"] = {"ceph_node": {}} + + with ( + patch("ceph_devstack.resources.ceph.HasSudo") as MockHasSudo, + patch( + "ceph_devstack.resources.ceph.LoopControlDeviceExists" + ) as MockLoopCtrl, + patch( + "ceph_devstack.resources.ceph.LoopControlDeviceWriteable" + ) as MockLoopCtrlWrite, + patch( + "ceph_devstack.resources.ceph.BlockPoolDiskGroup" + ) as MockBlockPoolDiskGroup, + patch( + "ceph_devstack.resources.ceph.BlockPoolParentAccessible" + ) as MockBlockPoolParentAccessible, + patch("ceph_devstack.host.host.selinux_enforcing") as mock_selinux, + ): + MockHasSudo.return_value = AsyncMock(evaluate=AsyncMock(return_value=True)) + MockLoopCtrl.return_value = AsyncMock(evaluate=AsyncMock(return_value=True)) + MockLoopCtrlWrite.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) + MockBlockPoolDiskGroup.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) + MockBlockPoolParentAccessible.return_value = AsyncMock( + evaluate=AsyncMock(return_value=False) + ) + mock_selinux.return_value = False + result = await devstack.check_requirements() + assert result is False + async def test_check_requirements_returns_false_when_repo_missing(self): devstack = CephDevStack() config["containers"] = { diff --git a/tests/resources/ceph/test_testnode.py b/tests/resources/ceph/test_testnode.py index 9425edab..14237c2e 100644 --- a/tests/resources/ceph/test_testnode.py +++ b/tests/resources/ceph/test_testnode.py @@ -2,8 +2,8 @@ import pytest -from ceph_devstack.resources.ceph import TestNode as _TestNode from ceph_devstack import config +from ceph_devstack.resources.ceph import TestNode as _TestNode class TestTestnode: @@ -16,22 +16,34 @@ def test_testnode_loop_device_count_default_to_one(self, cls): testnode = cls("testnode_1") assert testnode.loop_device_count == 1 - def test_testnode_create_cmd_includes_related_devices(self, cls): + def test_testnode_create_cmd_includes_related_devices(self, cls, tmp_path): config.load(Path(__file__).parent.joinpath("fixtures", "testnode-config.toml")) + config["data_dir"] = str(tmp_path) testnode = cls("testnode_1") create_cmd = testnode.create_cmd - assert "--device=/dev/loop4" in create_cmd - assert "--device=/dev/loop5" in create_cmd - assert "--device=/dev/loop6" in create_cmd - assert "--device=/dev/loop7" in create_cmd + assert "--device=/dev/loop0" in create_cmd + assert "--device=/dev/loop1" in create_cmd + assert "--device=/dev/loop2" in create_cmd + assert "--device=/dev/loop3" in create_cmd - def test_testnode_devices_is_based_on_loop_device_count_config(self, cls): + def test_testnode_devices_is_based_on_loop_device_count_config(self, cls, tmp_path): config.load(Path(__file__).parent.joinpath("fixtures", "testnode-config.toml")) + config["data_dir"] = str(tmp_path) testnode = cls("testnode_1") assert testnode.loop_device_count == 4 assert testnode.devices == [ - "/dev/loop4", - "/dev/loop5", - "/dev/loop6", - "/dev/loop7", + "/dev/loop0", + "/dev/loop1", + "/dev/loop2", + "/dev/loop3", ] + + def test_testnode_reuses_existing_backing_files(self, cls, tmp_path): + config["data_dir"] = str(tmp_path) + image_dir = tmp_path / "disk_images" + image_dir.mkdir() + (image_dir / "testnode_1-4").write_bytes(b"") + (image_dir / "testnode_1-1").write_bytes(b"") + config["containers"]["testnode"]["loop_device_count"] = 3 + testnode = cls("testnode_1") + assert testnode.devices == ["/dev/loop1", "/dev/loop4", "/dev/loop0"] diff --git a/tests/test_block_pool.py b/tests/test_block_pool.py new file mode 100644 index 00000000..8cd8f363 --- /dev/null +++ b/tests/test_block_pool.py @@ -0,0 +1,445 @@ +from pathlib import Path +from unittest.mock import patch + +import pytest + +from ceph_devstack.block_pool import ( + BlockPool, + BlockPoolError, + format_size, + parse_size, + validate_parent_name, + _has_partition_siblings, + _read_tail_marker, + _write_tail_marker, +) + + +class TestParseSize: + def test_gigabytes(self): + assert parse_size("50G") == 50 * 1024**3 + + def test_integer(self): + assert parse_size(1024) == 1024 + + def test_invalid(self): + with pytest.raises(BlockPoolError): + parse_size("not-a-size") + + def test_format_size_human_units(self): + assert format_size(5 * 1024**3) == "5G" + assert format_size(12345) == "12345B" + + +class TestValidateParentName: + def test_accepts_partition(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/nvme0n1p1", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=True), + ): + assert validate_parent_name("/dev/nvme0n1p1") == "/dev/nvme0n1p1" + + def test_accepts_whole_disk(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/nvme0n1", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=True), + ): + assert validate_parent_name("/dev/nvme0n1") == "/dev/nvme0n1" + + def test_rejects_device_mapper(self): + with pytest.raises(BlockPoolError, match="not allowed"): + validate_parent_name("/dev/dm-0") + + def test_rejects_non_dev_path(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/tmp/not-a-device", + ), + pytest.raises(BlockPoolError, match="must be a /dev path"), + ): + validate_parent_name("/tmp/not-a-device") + + def test_rejects_missing_device(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/nvme0n1p1", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=False), + pytest.raises(BlockPoolError, match="does not exist"), + ): + validate_parent_name("/dev/nvme0n1p1") + + def test_rejects_unknown_block_name(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/foo", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=True), + pytest.raises(BlockPoolError, match="block device or partition"), + ): + validate_parent_name("/dev/foo") + + +class TestBlockPool: + def test_allocate_and_reuse_stable_owner(self, enrolled_pool): + size = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + first_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, size) + second_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 1, size) + again_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, size) + assert again_offset == first_offset + assert first_offset == 0 + assert second_offset == size + + def test_release_and_reuse_slice(self, enrolled_pool): + size = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + enrolled_pool.get_or_allocate("testnode_0", 0, size) + enrolled_pool.get_or_allocate("testnode_0", 1, size) + enrolled_pool.release_owner("testnode_0") + + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + reused_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, size) + assert reused_offset in {0, size} + + def test_different_sizes_pack_sequentially(self, enrolled_pool): + small = parse_size("5G") + large = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + first_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, small) + second_offset, _ = enrolled_pool.get_or_allocate("testnode_0", 0, large) + assert first_offset == 0 + assert second_offset == small + + def test_parent_change_rejected(self, tmp_path, enrolled_pool): + size = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, size) + with ( + patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme1n1p1", + ), + pytest.raises(BlockPoolError, match="parent changed"), + ): + BlockPool( + tmp_path / "block_pool.json", + "/dev/nvme1n1p1", + allow_enroll=True, + ) + + def test_enroll_requires_allow_enroll(self, disallow_enroll_pool): + with pytest.raises(BlockPoolError, match="allow_enroll"): + disallow_enroll_pool.ensure_ready() + + def test_from_config_allow_enroll_defaults_false(self, tmp_path): + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme0n1p1", + ): + pool = BlockPool.from_config( + { + "block_pool": { + "parent": "/dev/nvme0n1p1", + "state_dir": str(tmp_path), + } + } + ) + assert pool is not None + assert pool.allow_enroll is False + + def test_from_config_allow_enroll_explicit_false(self, tmp_path): + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme0n1p1", + ): + pool = BlockPool.from_config( + { + "block_pool": { + "parent": "/dev/nvme0n1p1", + "state_dir": str(tmp_path), + "allow_enroll": False, + } + } + ) + assert pool is not None + assert pool.allow_enroll is False + + def test_from_config_allow_enroll_true(self, tmp_path): + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme0n1p1", + ): + pool = BlockPool.from_config( + { + "block_pool": { + "parent": "/dev/nvme0n1p1", + "state_dir": str(tmp_path), + "allow_enroll": True, + } + } + ) + assert pool is not None + assert pool.allow_enroll is True + + def test_enroll_rejects_non_empty_parent(self, block_pool, mock_fresh_enrollment): + with open(block_pool.parent, "r+b") as handle: + handle.write(b"DATA") + with pytest.raises(BlockPoolError, match="not empty"): + block_pool.ensure_ready() + + def test_enroll_writes_marker_and_state(self, block_pool, mock_fresh_enrollment): + block_pool.ensure_ready() + assert block_pool.enrolled + marker = _read_tail_marker(block_pool.parent) + assert marker is not None + assert marker["pool_id"] == block_pool._state["pool_id"] + assert marker["parent"] == block_pool.parent + + def test_new_slice_rejects_existing_data(self, enrolled_pool): + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch( + "ceph_devstack.block_pool._read_device", + return_value=b"NOTZEROS" + b"\x00" * 4088, + ), + pytest.raises(BlockPoolError, match="not empty"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, 1024 * 1024) + + def test_enroll_rejects_whole_disk_with_partitions(self, tmp_path): + state_path = tmp_path / "block_pool.json" + parent = tmp_path / "nvme0n1" + parent.write_bytes(b"\x00" * (20 * 1024**2)) + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(parent), + ): + pool = BlockPool(state_path, str(parent), allow_enroll=True) + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=False), + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", + return_value=False, + ), + patch("ceph_devstack.block_pool._read_tail_marker", return_value=None), + patch( + "ceph_devstack.block_pool._has_partition_siblings", + return_value=True, + ), + pytest.raises(BlockPoolError, match="existing partitions"), + ): + pool.ensure_ready() + + def test_reclaims_marker_after_state_loss(self, disallow_enroll_pool): + payload = { + "pool_id": "saved-pool-id", + "parent": disallow_enroll_pool.parent, + } + _write_tail_marker(disallow_enroll_pool.parent, payload) + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=False), + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", + return_value=False, + ), + ): + disallow_enroll_pool.ensure_ready() + assert disallow_enroll_pool.enrolled + assert disallow_enroll_pool._state["pool_id"] == "saved-pool-id" + + def test_reused_slice_must_be_empty(self, enrolled_pool): + size = 1024 * 1024 + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch("ceph_devstack.block_pool._read_device", return_value=b"\x00" * 4096), + ): + enrolled_pool.get_or_allocate("testnode_0", 0, size) + enrolled_pool.release_owner("testnode_0") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch( + "ceph_devstack.block_pool._read_device", + return_value=b"STALE" + b"\x00" * 4089, + ), + pytest.raises(BlockPoolError, match="not empty"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, size) + + +class TestBlockPoolStatus: + def test_status_disabled(self, caplog): + with caplog.at_level("INFO", logger="ceph-devstack"): + assert BlockPool.status_from_config({}) == 0 + assert "disabled" in caplog.text + + def test_status_shows_allocations(self, tmp_path, enrolled_pool, caplog): + enrolled_pool._state["allocations"] = { + "ceph_node:0": {"offset": 0, "size": 1024 * 1024, "owner": "ceph_node"} + } + enrolled_pool._save() + with ( + patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=enrolled_pool.parent, + ), + caplog.at_level("INFO", logger="ceph-devstack"), + ): + BlockPool.status_from_config( + { + "block_pool": { + "parent": enrolled_pool.parent, + "state_dir": str(tmp_path), + } + } + ) + assert "ceph_node:0" in caplog.text + assert "ceph_node" in caplog.text + + +class TestBlockPoolEnrollmentGuards: + def test_enroll_rejects_mounted_parent(self, block_pool, mock_fresh_enrollment): + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=True), + pytest.raises(BlockPoolError, match="mounted"), + ): + block_pool.ensure_ready() + + def test_enroll_rejects_blkid_signature(self, block_pool, mock_fresh_enrollment): + with ( + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", + return_value=True, + ), + pytest.raises(BlockPoolError, match="filesystem or partition"), + ): + block_pool.ensure_ready() + + def test_verify_rejects_missing_marker(self, enrolled_pool): + with ( + patch("ceph_devstack.block_pool._read_tail_marker", return_value=None), + pytest.raises(BlockPoolError, match="missing the devstack"), + ): + enrolled_pool.ensure_ready() + + def test_verify_rejects_foreign_pool_id(self, enrolled_pool): + with ( + patch( + "ceph_devstack.block_pool._read_tail_marker", + return_value={ + "parent": enrolled_pool.parent, + "pool_id": "other-pool", + }, + ), + pytest.raises(BlockPoolError, match="belongs to another pool"), + ): + enrolled_pool.ensure_ready() + + def test_region_exceeds_parent_size(self, enrolled_pool): + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch( + "ceph_devstack.block_pool._device_size_bytes", + return_value=512 * 1024, + ), + pytest.raises(BlockPoolError, match="exceeds"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, 1024 * 1024) + + def test_legacy_slice_size_state_rejected_on_load(self, enrolled_pool): + enrolled_pool._state["slice_size"] = 1024 * 1024 + enrolled_pool._save() + with ( + patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=enrolled_pool.parent, + ), + pytest.raises(BlockPoolError, match="legacy block pool state"), + ): + BlockPool(enrolled_pool.state_path, enrolled_pool.parent) + + +class TestPartitionSiblings: + def test_whole_disk_with_nvme_partitions(self): + class _Entry: + def __init__(self, name: str): + self.name = name + + def is_dir(self) -> bool: + return True + + class _BlockDir: + def is_dir(self) -> bool: + return True + + def iterdir(self) -> list[_Entry]: + return [_Entry("nvme0n1"), _Entry("nvme0n1p1")] + + mock_block_dir = _BlockDir() + + def path_for(arg: str): + if arg == "/sys/class/block": + return mock_block_dir + return Path(arg) + + with patch("ceph_devstack.block_pool.Path", side_effect=path_for): + assert _has_partition_siblings("nvme0n1") is True + + def test_partition_name_has_no_siblings(self): + assert _has_partition_siblings("nvme0n1p1") is False + + +class TestBlockPoolAllocation: + def test_allocation_for_missing(self, enrolled_pool): + assert enrolled_pool.allocation_for("nobody", 0) is None + + def test_allocation_for_active(self, enrolled_pool): + size = 1024 * 1024 + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch("ceph_devstack.block_pool._read_device", return_value=b"\x00" * 4096), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, size) + assert enrolled_pool.allocation_for("ceph_node", 0) == (0, size, size) + + +class TestTailMarker: + def test_roundtrip(self, tmp_path): + parent = tmp_path / "disk" + parent.write_bytes(b"\x00" * (1024 * 1024)) + payload = {"pool_id": "abc", "parent": str(parent)} + _write_tail_marker(str(parent), payload) + assert _read_tail_marker(str(parent)) == payload + + def test_corrupt_marker_returns_none(self, tmp_path): + parent = tmp_path / "disk" + parent.write_bytes(b"\x00" * (1024 * 1024)) + offset = len(parent.read_bytes()) - 4096 + with open(parent, "r+b") as handle: + handle.seek(offset) + handle.write(b"CEPH-DEVSTACK-BLOCK-POOL-TAIL-v1\x00not-json") + assert _read_tail_marker(str(parent)) is None diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..90c7e4e7 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,25 @@ +import sys +from unittest.mock import patch + +from ceph_devstack import config, parse_args +from ceph_devstack.block_pool import BlockPool + + +class TestBlockPoolCLI: + def test_parse_args_block_pool_status(self): + args = parse_args(["block-pool", "status"]) + assert args.command == "block-pool" + assert args.block_pool_op == "status" + + def test_main_block_pool_status(self): + from ceph_devstack.cli import main + + with ( + patch.object(sys, "argv", ["ceph-devstack", "block-pool", "status"]), + patch.object( + BlockPool, "status_from_config", return_value=0 + ) as mock_status, + ): + rc = main() + assert rc == 0 + mock_status.assert_called_once_with(config) diff --git a/tests/test_config.py b/tests/test_config.py index 83697422..b926fb68 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -105,6 +105,9 @@ def test_config_defaults(self): assert config == { "stack": "teuthology", "data_dir": "~/.local/share/ceph-devstack", + "block_pool": { + "state_dir": "~/.local/share/ceph-devstack", + }, "stacks": { "teuthology": { "services": [ @@ -134,6 +137,7 @@ def test_config_defaults(self): "pulpito": {"image": "quay.io/ceph-infra/pulpito:main"}, "testnode": { "count": 3, + "loop_device_count": 1, "loop_device_size": "5G", "image": "quay.io/ceph-infra/teuthology-testnode:main", }, @@ -142,6 +146,8 @@ def test_config_defaults(self): "image": "quay.io/ceph-ci/ceph:main", "loop_device_count": 3, "loop_device_size": "5G", + "dashboard_port": 8080, + "dashboard_ssl": False, }, }, } From 5d9830ae6496a66f1de2cbec151f016745ff79e3 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 16 Jul 2026 15:28:14 -0600 Subject: [PATCH 3/5] sccache rw --- ceph_devstack/config.toml | 1 + ceph_devstack/resources/ceph/ceph_node.py | 35 ++++++++++++++++++---- ceph_devstack/sccache-s3.conf | 1 - tests/resources/ceph/test_ceph_node.py | 36 +++++++++++++++++++++++ 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 06da229b..cff6f5d0 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -52,6 +52,7 @@ dashboard_ssl = false # sccache = true # default; local disk cache (see sccache.conf) # sccache = false # disable sccache # sccache_mode = "local" # local (default) or s3 for shared remote cache +# sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) # sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" # sccache_cache_size = "100G" # local cache size limit # sccache_debug = false # enable SCCACHE_LOG=debug on build stdout diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index 75609e6e..16a67ca8 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -268,6 +268,11 @@ def sccache_enabled(self) -> bool: def sccache_mode(self) -> str: return str(self.config.get("sccache_mode", "local")).lower() + @property + def sccache_rw_mode(self) -> bool: + """Whether to use sccache in read-write mode (requires credentials).""" + return self.config.get("sccache_rw_mode", False) is True + @property def sccache_debug(self) -> bool: return self.config.get("sccache_debug", False) is True @@ -421,12 +426,30 @@ def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: cache_size = self.config.get("sccache_cache_size", "100G") lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") elif self.sccache_mode == "s3": - lines.extend( - [ - "SCCACHE_S3_NO_CREDENTIALS=true", - "SCCACHE_S3_RW_MODE=READ_ONLY", - ] - ) + if self.sccache_rw_mode: + # Read-write mode: pass AWS credentials from user's environment + aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") + aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + if not aws_access_key or not aws_secret_key: + raise ValueError( + "sccache_s3_rw=true requires AWS_ACCESS_KEY_ID and " + "AWS_SECRET_ACCESS_KEY to be set in the environment" + ) + lines.extend( + [ + f"AWS_ACCESS_KEY_ID={aws_access_key}", + f"AWS_SECRET_ACCESS_KEY={aws_secret_key}", + "SCCACHE_S3_RW_MODE=READ_WRITE", + ] + ) + else: + # Read-only mode (default) + lines.extend( + [ + "SCCACHE_S3_NO_CREDENTIALS=true", + "SCCACHE_S3_RW_MODE=READ_ONLY", + ] + ) return lines, extra_args def _prepare_build_env(self) -> tuple[Path | None, list[str]]: diff --git a/ceph_devstack/sccache-s3.conf b/ceph_devstack/sccache-s3.conf index 01db75c7..a341b223 100644 --- a/ceph_devstack/sccache-s3.conf +++ b/ceph_devstack/sccache-s3.conf @@ -4,5 +4,4 @@ endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud" use_ssl = true key_prefix = "" server_side_encryption = false -no_credentials = true region = "auto" diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index f7fb83ec..b6da8876 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -1,6 +1,7 @@ from unittest.mock import AsyncMock, patch import sys +import pytest from ceph_devstack import config from ceph_devstack.resources.container import Container @@ -146,6 +147,41 @@ def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): assert "SCCACHE_DIR=" not in contents assert extra_args == [] + def test_prepare_build_env_uses_s3_rw_mode_with_credentials( + self, tmp_path, monkeypatch + ): + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key") + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + config["containers"]["ceph_node"]["sccache_rw_mode"] = True + env_file, extra_args = CephNode()._prepare_build_env() + contents = env_file.read_text() + assert "AWS_ACCESS_KEY_ID=test-key-id" in contents + assert "AWS_SECRET_ACCESS_KEY=test-secret-key" in contents + assert "SCCACHE_S3_RW_MODE=READ_WRITE" in contents + assert "SCCACHE_S3_NO_CREDENTIALS=true" not in contents + assert extra_args == [] + + def test_prepare_build_env_raises_error_for_s3_rw_without_credentials( + self, tmp_path, monkeypatch + ): + monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + config["containers"]["ceph_node"]["sccache_rw_mode"] = True + with pytest.raises( + ValueError, match="AWS_ACCESS_KEY_ID.*AWS_SECRET_ACCESS_KEY" + ): + CephNode()._prepare_build_env() + def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): custom_conf = tmp_path / "custom-sccache.conf" custom_conf.write_text("[cache.s3]\nbucket = test\n") From 4563e639e3beb55bd170152fcf44a7ab0c349006 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 16 Jul 2026 15:43:26 -0600 Subject: [PATCH 4/5] fixup! sccache rw --- ceph_devstack/resources/ceph/ceph_node.py | 43 ++++++++++++++--------- tests/resources/ceph/test_ceph_node.py | 8 +++-- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index 16a67ca8..0d70d186 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -10,6 +10,8 @@ from subprocess import CalledProcessError from typing import List +import tomlkit + from ceph_devstack import PROJECT_ROOT, config, logger from ceph_devstack.host import host from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner @@ -390,6 +392,21 @@ def create_cmd(self): def _device_image(self, device: str) -> str: return f"{self.name}-{device.removeprefix('/dev/loop')}" + def _get_s3_credentials_env(self) -> list[str]: + """Get S3 credential environment variables for read-write mode.""" + aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") + aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + if not aws_access_key or not aws_secret_key: + raise ValueError( + "sccache_rw_mode=true requires AWS_ACCESS_KEY_ID and " + "AWS_SECRET_ACCESS_KEY to be set in the environment" + ) + return [ + f"AWS_ACCESS_KEY_ID={aws_access_key}", + f"AWS_SECRET_ACCESS_KEY={aws_secret_key}", + "SCCACHE_S3_RW_MODE=READ_WRITE", + ] + def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: if not self.sccache_enabled: return [], [] @@ -406,7 +423,15 @@ def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: use_local_cache = True if not conf_src.is_file(): raise FileNotFoundError(f"sccache config not found: {conf_src}") - shutil.copy2(conf_src, repo / "sccache.conf") + + # Parse and modify config for S3 mode to set no_credentials correctly + conf_content = conf_src.read_text() + if self.sccache_mode == "s3": + conf_data = tomlkit.parse(conf_content) + # Set no_credentials based on whether we're using read-write mode + conf_data["cache"]["s3"]["no_credentials"] = not self.sccache_rw_mode + conf_content = tomlkit.dumps(conf_data) + (repo / "sccache.conf").write_text(conf_content) lines = [ "SCCACHE=true", @@ -427,21 +452,7 @@ def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") elif self.sccache_mode == "s3": if self.sccache_rw_mode: - # Read-write mode: pass AWS credentials from user's environment - aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") - aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") - if not aws_access_key or not aws_secret_key: - raise ValueError( - "sccache_s3_rw=true requires AWS_ACCESS_KEY_ID and " - "AWS_SECRET_ACCESS_KEY to be set in the environment" - ) - lines.extend( - [ - f"AWS_ACCESS_KEY_ID={aws_access_key}", - f"AWS_SECRET_ACCESS_KEY={aws_secret_key}", - "SCCACHE_S3_RW_MODE=READ_WRITE", - ] - ) + lines.extend(self._get_s3_credentials_env()) else: # Read-only mode (default) lines.extend( diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index b6da8876..5a923a99 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -2,6 +2,7 @@ import sys import pytest +import tomlkit from ceph_devstack import config from ceph_devstack.resources.container import Container @@ -138,9 +139,10 @@ def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): config["containers"]["ceph_node"]["repo"] = str(repo) config["containers"]["ceph_node"]["sccache_mode"] = "s3" env_file, extra_args = CephNode()._prepare_build_env() - assert ( - repo / "sccache.conf" - ).read_text() == PACKAGE_SCCACHE_S3_CONF.read_text() + sccache_conf = repo / "sccache.conf" + assert sccache_conf.exists() + conf_data = tomlkit.parse(sccache_conf.read_text()) + assert conf_data["cache"]["s3"]["no_credentials"] is True contents = env_file.read_text() assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents assert "SCCACHE_S3_RW_MODE=READ_ONLY" in contents From f1093002b037318bad9e1fd9e18857e4f4ce864c Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 16 Jul 2026 15:53:39 -0600 Subject: [PATCH 5/5] fixup! sccache rw --- ceph_devstack/config.toml | 15 +++++++-------- tests/resources/ceph/test_ceph_node.py | 14 +++++++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index cff6f5d0..71d3c1c2 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -49,15 +49,14 @@ dashboard_ssl = false # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # build-with-container.py -d # build_steps = ["build"] # default for cpatch; ["packages"] for container builder -# sccache = true # default; local disk cache (see sccache.conf) -# sccache = false # disable sccache -# sccache_mode = "local" # local (default) or s3 for shared remote cache -# sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) -# sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" -# sccache_cache_size = "100G" # local cache size limit -# sccache_debug = false # enable SCCACHE_LOG=debug on build stdout +sccache = true # local disk cache (see sccache.conf); set to false to disable +sccache_mode = "local" # local (default) or s3 for shared remote cache +sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) +# sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" # override default cache location +sccache_cache_size = "100G" # local cache size limit +sccache_debug = false # enable SCCACHE_LOG=debug on build stdout # sccache_conf = "/path/to/sccache.conf" # override bundled config entirely -# npm_cache = true # default; persist dashboard npm downloads +npm_cache = true # persist dashboard npm downloads # npm_cache_path = "~/.local/share/ceph-devstack/cache/npm" # override npm cache location # dnf_cache = false # persist dnf downloads when rebuilding ceph-build images # dnf_cache_path = "~/.local/share/ceph-devstack/cache/dnf" # override dnf cache base dir diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index 5a923a99..b2af9843 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -186,7 +186,7 @@ def test_prepare_build_env_raises_error_for_s3_rw_without_credentials( def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): custom_conf = tmp_path / "custom-sccache.conf" - custom_conf.write_text("[cache.s3]\nbucket = test\n") + custom_conf.write_text('[cache.s3]\nbucket = "test"\n') repo = tmp_path / "ceph" repo.mkdir() config["containers"]["ceph_node"]["image"] = "example:test" @@ -194,7 +194,10 @@ def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): config["containers"]["ceph_node"]["sccache_conf"] = str(custom_conf) config["containers"]["ceph_node"]["sccache_mode"] = "s3" env_file, extra_args = CephNode()._prepare_build_env() - assert (repo / "sccache.conf").read_text() == custom_conf.read_text() + sccache_conf = repo / "sccache.conf" + conf_data = tomlkit.parse(sccache_conf.read_text()) + assert conf_data["cache"]["s3"]["bucket"] == "test" + assert conf_data["cache"]["s3"]["no_credentials"] is True contents = env_file.read_text() assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents assert extra_args == [] @@ -298,11 +301,12 @@ def test_bundled_sccache_conf_uses_local_disk(self): assert 'dir = "/sccache"' in contents assert "rw_mode" not in contents - def test_bundled_sccache_s3_conf_keeps_anonymous_read_settings(self): + def test_bundled_sccache_s3_conf_has_s3_settings(self): contents = PACKAGE_SCCACHE_S3_CONF.read_text() assert "[cache.s3]" in contents - assert "no_credentials = true" in contents - assert "rw_mode" not in contents + assert "bucket" in contents + assert "endpoint" in contents + # no_credentials is added dynamically by _sccache_build_env def test_cpatch_cmd_uses_upstream_script(self): config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main"