diff --git a/Dockerfile.lab b/Dockerfile.lab index 514f51f..9c56f6f 100644 --- a/Dockerfile.lab +++ b/Dockerfile.lab @@ -36,6 +36,9 @@ RUN pip install --no-cache-dir "/src[rag-lab]" RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')" \ && chmod -R a+rX /opt/hf-cache +# Create the workspace as root and hand it over BEFORE switching user: the classic Docker builder (Ubuntu's docker.io, no +# BuildKit) creates a WORKDIR as root even after USER, which made `labagent rag init` fail with a PermissionError. +RUN mkdir -p /home/coder/lab && chown coder:coder /home/coder/lab USER coder WORKDIR /home/coder/lab RUN labagent rag init --dir /home/coder/lab && python -m ipykernel install --user --name python3 diff --git a/labagent/packs/rag_ide/ec2.py b/labagent/packs/rag_ide/ec2.py new file mode 100644 index 0000000..041f042 --- /dev/null +++ b/labagent/packs/rag_ide/ec2.py @@ -0,0 +1,230 @@ +"""EC2 platform for rag-ide: ONE shared instance per cohort, one container per learner. + +Why not one VM per learner: learners are idle most of the time, so they share CPU, and a single box with a pinned image costs +a small fraction of per-learner serverless sandboxes. What keeps it safe: + - no SSH and no instance profile (learners get no AWS credentials); only ports 80 and 443 are open + - IMDSv2 required with hop limit 1, so containers cannot reach the instance metadata service + - every learner container is unprivileged, memory/CPU/pid limited, with all Linux capabilities dropped + - the box shuts itself down (and terminates) after `hours`, and also when no learner container is left running + - the account must be on the labagent allow list; the image source is a pinned git commit that is verified after checkout +HTTPS comes from Caddy and Let's Encrypt on a .sslip.io name, so no domain is needed. Learner URLs are paths on that name. +""" +from __future__ import annotations + +import math +import re +import shlex +from typing import Any, Dict, List, Optional, Tuple + +from ..base import PackError +from .modal_gateway import slug + +REPO = "https://github.com/becloudready/labagent" +PINNED_REF = "737a284413ae94bb69cd425935df0f31e3919a28" +CANONICAL_OWNER = "099720109477" +AMI_NAME = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" +SSM_UBUNTU = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id" +IMAGE = "labagent-rag-lab:local" +# (type, vCPU, GiB) smallest first. x86_64 only (the image is built on the box). +INSTANCE_TYPES: List[Tuple[str, int, int]] = [ + ("m6i.large", 2, 8), ("m6i.xlarge", 4, 16), ("m6i.2xlarge", 8, 32), ("m6i.4xlarge", 16, 64), ("m6i.8xlarge", 32, 128), +] +OVERSUBSCRIBE_RAM = 0.5 # learners rarely use their whole memory limit at the same time +LEARNERS_PER_VCPU = 3 # CPU is shared more aggressively than memory + + +def pick_instance_type(learners: int, memory_mib: int, override: str = "auto") -> str: + """Smallest type with RAM >= 50% of the sum of learner limits (plus 2 GiB system) and ~3 learners per vCPU.""" + if override != "auto": + if not re.fullmatch(r"[a-z0-9]+\.[a-z0-9]+", override): + raise PackError("instance_type must look like m6i.2xlarge, or be 'auto'") + return override + need_gib = learners * (memory_mib / 1024.0) * OVERSUBSCRIBE_RAM + 2 + need_cpu = math.ceil(learners / LEARNERS_PER_VCPU) + for t, cpu, gib in INSTANCE_TYPES: + if gib >= need_gib and cpu >= need_cpu: + return t + raise PackError("that many learners do not fit one instance: split them into two cohorts") + + +def sslip_host(ip: str) -> str: + return ip.replace(".", "-") + ".sslip.io" + + +def learner_url(host: str, learner: str) -> str: + return f"https://{host}/{learner}/" + + +def build_user_data(learners: Dict[str, str], *, ref: str, hours: int, idle_minutes: int, cpu: float, memory_mib: int, + tls: str = "letsencrypt", env: Optional[Dict[str, str]] = None) -> str: + """cloud-init script. `learners` maps name -> password. Passwords are in user-data, which anyone with EC2 read access in the + account can see: this is why the account guard matters. Learner containers cannot read it (hop limit 1).""" + if not re.fullmatch(r"[0-9a-f]{40}", ref): + raise PackError("ref must be a full 40-character commit SHA") + for n, pw in learners.items(): + if not re.fullmatch(r"[a-z0-9-]+", n) or not re.fullmatch(r"[A-Za-z0-9_-]{8,}", pw): + raise PackError("learner names and passwords must be plain [a-z0-9-] / [A-Za-z0-9_-] tokens") + q = shlex.quote + extra = " ".join(f"-e {q(k)}={q(v)}" for k, v in (env or {}).items()) + runs, routes = [], [] + for n, pw in learners.items(): + runs.append(f""" +docker volume create lab-{n} >/dev/null +docker run --rm -u root -v lab-{n}:/w {IMAGE} chown -R 1000:1000 /w +docker run -d --name lab-{n} --network labnet --cpus {cpu:g} --memory {memory_mib}m --pids-limit 1024 \\ + --cap-drop ALL --security-opt no-new-privileges -e PASSWORD={q(pw)} -e HF_HUB_OFFLINE=1 {extra} \\ + -v lab-{n}:/home/coder/lab/work {IMAGE} \\ + code-server --bind-addr 0.0.0.0:8080 --auth password --idle-timeout-seconds {idle_minutes * 60} /home/coder/lab""") + routes.append(f" redir /{n} /{n}/\n handle_path /{n}/* {{\n reverse_proxy lab-{n}:8080\n }}") + tls_line = " tls internal\n" if tls == "internal" else "" + return f"""#!/bin/bash +set -euxo pipefail +exec > >(tee /var/log/labagent-boot.log) 2>&1 +shutdown -h +{hours * 60} "labagent: hard time cap reached" +export DEBIAN_FRONTEND=noninteractive +apt-get update -y && apt-get install -y docker.io git curl +systemctl enable --now docker +TOKEN=$(curl -sS -X PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60') +IP=$(curl -sS -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/public-ipv4) +HOST=$(echo "$IP" | tr . -).sslip.io +mkdir -p /opt/labagent/src && cd /opt/labagent/src +git init -q && git remote add origin {REPO} && git fetch -q --depth 1 origin {ref} && git checkout -q FETCH_HEAD +test "$(git rev-parse HEAD)" = "{ref}" +docker build -q -f Dockerfile.lab -t {IMAGE} . +docker network create labnet +{''.join(runs)} +mkdir -p /opt/labagent/caddy +cat > /opt/labagent/caddy/Caddyfile < /usr/local/bin/labagent-idle-check <<'IDLE' +#!/bin/bash +# Stop paying when every learner has gone: shut down after {max(5, idle_minutes)} minutes with no running learner container. +n=$(docker ps --filter name=lab- --format '{{{{.Names}}}}' | wc -l) +f=/run/labagent-empty-since +if [ "$n" -gt 0 ]; then rm -f "$f"; exit 0; fi +[ -f "$f" ] || date +%s > "$f" +[ $(( $(date +%s) - $(cat "$f") )) -ge {max(5, idle_minutes) * 60} ] && shutdown -h now +IDLE +chmod +x /usr/local/bin/labagent-idle-check +echo '* * * * * root /usr/local/bin/labagent-idle-check' > /etc/cron.d/labagent-idle +echo READY > /var/log/labagent-ready +""" + + +class Ec2Gateway: + """Real AWS calls, guarded by the allow list. Imported lazily by the pack so tests can substitute a fake.""" + + def __init__(self, region: Optional[str] = None, profile: Optional[str] = None, session: Any = None, config: Any = None): + from ...config import load_config + from ...context import Ctx + import boto3 + self.cfg = config or load_config() + if region: + self.cfg.region = region + self.ctx = Ctx(self.cfg, session or boto3.Session(profile_name=profile, region_name=self.cfg.region)) + + def check_auth(self) -> str: + from ...context import GuardError + try: + self.ctx.guard_account() + except GuardError as e: + raise PackError(f"{e} Example: LABAGENT_ALLOWED_ACCOUNTS=") from e + except Exception as e: + raise PackError(f"could not read AWS credentials ({type(e).__name__}). Configure them or pass --profile") from e + return f"{self.ctx.account_id} in {self.cfg.region}" + + @property + def ec2(self): + return self.ctx.client("ec2") + + def ami(self) -> str: + """Latest Ubuntu 24.04 from Canonical. SSM is the clean source; some roles cannot read it, so fall back to DescribeImages.""" + try: + return self.ctx.client("ssm").get_parameter(Name=SSM_UBUNTU)["Parameter"]["Value"] + except Exception: + imgs = self.ec2.describe_images(Owners=[CANONICAL_OWNER], Filters=[{"Name": "name", "Values": [AMI_NAME]}, {"Name": "state", "Values": ["available"]}])["Images"] + if not imgs: + raise PackError("could not find an Ubuntu 24.04 image: allow ssm:GetParameter or ec2:DescribeImages for this identity") + return max(imgs, key=lambda i: i["CreationDate"])["ImageId"] + + def security_group(self, cohort: str) -> str: + name = f"labagent-rag-ide-{slug(cohort, 30)}" + found = self.ec2.describe_security_groups(Filters=[{"Name": "group-name", "Values": [name]}])["SecurityGroups"] + if found: + return found[0]["GroupId"] + vpc = self.ec2.describe_vpcs(Filters=[{"Name": "isDefault", "Values": ["true"]}])["Vpcs"] + if not vpc: + raise PackError("no default VPC in this region: create one or use another region") + sg = self.ec2.create_security_group(GroupName=name, Description="labagent rag-ide: HTTP/HTTPS only", VpcId=vpc[0]["VpcId"], + TagSpecifications=[{"ResourceType": "security-group", "Tags": [{"Key": "labagent", "Value": "rag-ide"}, {"Key": "cohort", "Value": slug(cohort)}]}])["GroupId"] + self.ec2.authorize_security_group_ingress(GroupId=sg, IpPermissions=[ + {"IpProtocol": "tcp", "FromPort": p, "ToPort": p, "IpRanges": [{"CidrIp": "0.0.0.0/0", "Description": "labagent learners"}]} for p in (80, 443)]) + return sg + + def run(self, cohort: str, sg: str, user_data: str, instance_type: str, spot: bool) -> str: + kwargs: Dict[str, Any] = dict( + ImageId=self.ami(), InstanceType=instance_type, MinCount=1, MaxCount=1, UserData=user_data, SecurityGroupIds=[sg], + InstanceInitiatedShutdownBehavior="terminate", + MetadataOptions={"HttpTokens": "required", "HttpPutResponseHopLimit": 1, "HttpEndpoint": "enabled"}, + BlockDeviceMappings=[{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 60, "VolumeType": "gp3", "DeleteOnTermination": True, "Encrypted": True}}], + TagSpecifications=[{"ResourceType": "instance", "Tags": [{"Key": "labagent", "Value": "rag-ide"}, {"Key": "cohort", "Value": slug(cohort)}, + {"Key": "Name", "Value": f"labagent-rag-ide-{slug(cohort)}"}]}]) + if spot: + kwargs["InstanceMarketOptions"] = {"MarketType": "spot", "SpotOptions": {"SpotInstanceType": "one-time", "InstanceInterruptionBehavior": "terminate"}} + return self.ec2.run_instances(**kwargs)["Instances"][0]["InstanceId"] + + def public_ip(self, instance_id: str, timeout_s: int = 300) -> str: + waited = 0 + while waited <= timeout_s: + r = self.ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0]["Instances"][0] + if r["State"]["Name"] == "running" and r.get("PublicIpAddress"): + return r["PublicIpAddress"] + if r["State"]["Name"] in ("shutting-down", "terminated"): + raise PackError("the instance terminated during startup (spot capacity or a quota?)") + self.ctx.sleep(5) + waited += 5 + raise PackError("the instance did not get a public IP in time") + + def pause(self, seconds: float) -> None: + self.ctx.sleep(seconds) + + def ready(self, host: str, learner: str, verify: bool = True) -> bool: + """True once Caddy serves the learner's login page. verify=False only for tls=internal, whose certificate is self-signed.""" + import ssl + import urllib.request + ctx = None if verify else ssl._create_unverified_context() + try: + with urllib.request.urlopen(learner_url(host, learner) + "login", timeout=10, context=ctx) as r: + return r.status == 200 + except Exception: + return False + + def alive(self, instance_id: str) -> bool: + try: + r = self.ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0]["Instances"][0] + return r["State"]["Name"] in ("pending", "running") + except Exception: + return False + + def find(self, cohort: str) -> List[str]: + res = self.ec2.describe_instances(Filters=[{"Name": "tag:labagent", "Values": ["rag-ide"]}, {"Name": "tag:cohort", "Values": [slug(cohort)]}, + {"Name": "instance-state-name", "Values": ["pending", "running", "stopping", "stopped"]}]) + return [i["InstanceId"] for r in res["Reservations"] for i in r["Instances"]] + + def terminate(self, ids: List[str]) -> None: + if ids: + self.ec2.terminate_instances(InstanceIds=ids) + self.ec2.get_waiter("instance_terminated").wait(InstanceIds=ids, WaiterConfig={"Delay": 5, "MaxAttempts": 60}) + + def delete_security_group(self, cohort: str) -> None: + name = f"labagent-rag-ide-{slug(cohort, 30)}" + for g in self.ec2.describe_security_groups(Filters=[{"Name": "group-name", "Values": [name]}, {"Name": "tag:labagent", "Values": ["rag-ide"]}])["SecurityGroups"]: + try: + self.ec2.delete_security_group(GroupId=g["GroupId"]) + except Exception: + pass # still attached for a few seconds after termination; the next destroy removes it diff --git a/labagent/packs/rag_ide/local.py b/labagent/packs/rag_ide/local.py new file mode 100644 index 0000000..356bd63 --- /dev/null +++ b/labagent/packs/rag_ide/local.py @@ -0,0 +1,112 @@ +"""Local platform for rag-ide: the same one-container-per-learner lab, on this machine through Docker. For demos and tests. + +It mirrors the EC2 workflow (same pack steps, credentials file and destroy), minus the cloud parts: no HTTPS proxy, each learner +gets http://127.0.0.1:. Every port is bound to loopback only, containers are unprivileged and capped, and a password is +always required. Nothing here talks to a cloud account. +""" +from __future__ import annotations + +import subprocess +import urllib.request +from pathlib import Path +from typing import Dict, List, Optional + +from ..base import PackError +from .ec2 import IMAGE +from .modal_gateway import slug + +LAB_DIR = "/home/coder/lab" + + +def container_name(cohort: str, learner: str) -> str: + return f"labagent-lab-{slug(cohort, 20)}-{slug(learner, 20)}" + + +def volume_name(cohort: str, learner: str) -> str: + return f"labagent-lab-{slug(cohort, 20)}-{slug(learner, 20)}-work" + + +def learner_url(port: int) -> str: + return f"http://127.0.0.1:{port}/" + + +def run_args(cohort: str, learner: str, password: str, port: int, *, image: str, cpu: float, memory_mib: int, hours: int, + idle_s: int, env: Optional[Dict[str, str]] = None) -> List[str]: + """The docker run command for one learner (a list, never a shell string). The password travels in the environment of the + docker CLI process (`-e PASSWORD`), not in the argument list, so it does not show up in `ps`.""" + args = ["docker", "run", "-d", "--name", container_name(cohort, learner), "-p", f"127.0.0.1:{port}:8080", + "--label", "labagent=rag-ide", "--label", f"cohort={slug(cohort)}", "--label", f"learner={slug(learner)}", + "--cpus", f"{cpu:g}", "--memory", f"{memory_mib}m", "--pids-limit", "1024", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "-e", "PASSWORD", "-e", "HF_HUB_OFFLINE=1"] + for k in (env or {}): + args += ["-e", k] + args += ["-v", f"{volume_name(cohort, learner)}:{LAB_DIR}/work", image, + "sh", "-c", f"exec timeout {hours * 3600} code-server --bind-addr 0.0.0.0:8080 --auth password " + f"--idle-timeout-seconds {idle_s} {LAB_DIR}"] + return args + + +class LocalDockerGateway: + def __init__(self) -> None: + self.docker = "docker" + + def _run(self, args: List[str], env: Optional[Dict[str, str]] = None, check: bool = True) -> subprocess.CompletedProcess: + import os + r = subprocess.run(args, capture_output=True, text=True, env={**os.environ, **(env or {})}) + if check and r.returncode != 0: + raise PackError(f"docker failed: {(r.stderr or r.stdout).strip().splitlines()[-1][:200] if (r.stderr or r.stdout).strip() else 'unknown error'}") + return r + + def check_auth(self) -> str: + try: + r = self._run([self.docker, "info", "--format", "{{.ServerVersion}}"], check=False) + except FileNotFoundError as e: + raise PackError("docker is not installed. Install Docker Desktop, or use platform=ec2 or platform=modal.") from e + if r.returncode != 0: + raise PackError("docker is installed but its daemon is not running. Start Docker Desktop and retry.") + return f"local docker {r.stdout.strip()}" + + def ensure_image(self, image: str, source: str) -> None: + if self._run([self.docker, "image", "inspect", image], check=False).returncode == 0: + return + dockerfile = Path(source) / "Dockerfile.lab" + if not dockerfile.is_file(): + raise PackError(f"image '{image}' not found and there is no Dockerfile.lab to build it from. " + "Run from a labagent checkout, or pass image=.") + r = subprocess.run([self.docker, "build", "-f", str(dockerfile), "-t", image, str(dockerfile.parent)]) + if r.returncode != 0: + raise PackError("docker build failed") + + def launch(self, cohort: str, learner: str, password: str, port: int, **kw) -> Dict[str, str]: + vol = volume_name(cohort, learner) + image = kw["image"] + self._run([self.docker, "volume", "create", vol]) + self._run([self.docker, "run", "--rm", "-u", "root", "-v", f"{vol}:/w", image, "chown", "-R", "1000:1000", "/w"]) + self._run(run_args(cohort, learner, password, port, **kw), env={"PASSWORD": password, **(kw.get("env") or {})}) + return {"container": container_name(cohort, learner), "url": learner_url(port), "volume": vol, "port": str(port)} + + def ready(self, url: str) -> bool: + try: + with urllib.request.urlopen(url + "login", timeout=5) as r: + return r.status == 200 + except Exception: + return False + + def alive(self, container: str) -> bool: + r = self._run([self.docker, "inspect", "-f", "{{.State.Running}}", container], check=False) + return r.returncode == 0 and r.stdout.strip() == "true" + + def find(self, cohort: str) -> List[str]: + r = self._run([self.docker, "ps", "-a", "--filter", "label=labagent=rag-ide", "--filter", f"label=cohort={slug(cohort)}", "--format", "{{.Names}}"], check=False) + return [n for n in r.stdout.split() if n] + + def remove(self, container: str) -> None: + self._run([self.docker, "rm", "-f", container], check=False) + + def delete_volume(self, name: str) -> None: + self._run([self.docker, "volume", "rm", name], check=False) + + def pause(self, seconds: float) -> None: + import time + time.sleep(seconds) diff --git a/labagent/packs/rag_ide/pack.py b/labagent/packs/rag_ide/pack.py index 420ddad..3048ec5 100644 --- a/labagent/packs/rag_ide/pack.py +++ b/labagent/packs/rag_ide/pack.py @@ -13,9 +13,12 @@ from typing import Any, Dict, List from ..base import Pack, PackContext, PackError, Param, Step, state_dir +from . import ec2 as ec2mod +from . import local as localmod from .modal_gateway import ModalGateway, app_name, slug MAX_LEARNERS = 60 +WAIT_S = 1800 # image build on the box plus certificate issuance def learner_names(spec: Any) -> List[str]: @@ -57,6 +60,30 @@ def __getattr__(self, name: str): return getattr(self._gw, name) +class _LazyEc2: + """Same idea as _LazyModal; region and profile are applied in preflight, before the first AWS call.""" + + def __init__(self) -> None: + self._gw = None + self.region = None + self.profile = None + + def __getattr__(self, name: str): + if self._gw is None: + self._gw = ec2mod.Ec2Gateway(region=self.region, profile=self.profile) + return getattr(self._gw, name) + + +class _LazyLocal: + def __init__(self) -> None: + self._gw = None + + def __getattr__(self, name: str): + if self._gw is None: + self._gw = localmod.LocalDockerGateway() + return getattr(self._gw, name) + + class RagIde(Pack): id = "rag-ide" title = "Browser VS Code lab per learner (Modal)" @@ -64,7 +91,15 @@ class RagIde(Pack): aliases = ("ide", "web-ide", "rag-web-ide") default_name = "cohort" params = [ + Param("platform", "modal", "modal (one serverless sandbox per learner) | ec2 (one shared instance per cohort, much cheaper at scale) | local (Docker on this machine, for demos and tests)"), + Param("port", 8080, "local: first port; learner N gets port + N - 1 on 127.0.0.1"), Param("learners", "1", "Count (5) or comma list of names (ana,ben)"), + Param("instance_type", "auto", "ec2: instance type, or auto"), + Param("spot", False, "ec2: use a spot instance (cheaper, can be reclaimed)"), + Param("region", None, "ec2: AWS region (default from labagent config)"), + Param("profile", None, "ec2: AWS profile"), + Param("tls", "letsencrypt", "ec2: letsencrypt | internal (self-signed, browser warning)"), + Param("ref", ec2mod.PINNED_REF, "ec2: labagent commit (40-hex SHA) the lab image is built from"), Param("hours", 4, "Hard cap per session, 1 to 24"), Param("idle_minutes", 15, "Shut a session down after this long with no browser connected"), Param("cpu", 2.0, "CPU cores per learner"), @@ -77,10 +112,14 @@ class RagIde(Pack): ] def services(self, profile): - return {"modal": _LazyModal()} + return {"modal": _LazyModal(), "ec2": _LazyEc2(), "local": _LazyLocal()} def _validate(self, ctx: PackContext) -> None: p = ctx.params + if p["platform"] not in ("modal", "ec2", "local"): + raise PackError("platform must be modal, ec2 or local") + if p["tls"] not in ("letsencrypt", "internal"): + raise PackError("tls must be letsencrypt or internal") names = learner_names(p["learners"]) if not 1 <= int(p["hours"]) <= 24: raise PackError("hours must be between 1 and 24") @@ -130,9 +169,12 @@ def _learners(self, ctx: PackContext): passwords[n] = pw ctx.state["learners"] = learners # persist progress so a failure part-way resumes ctx.mem["passwords"] = passwords + self._write_creds(ctx) return {"learners": learners} - def _credentials(self, ctx: PackContext): + def _write_creds(self, ctx: PackContext) -> Path: + """Write the 0600 credentials file. Called as soon as learners exist (so a later failure cannot lose their passwords) + and again as the last step. Passwords already on file are kept.""" path = creds_path(ctx.name) old: Dict[str, str] = {} if path.exists(): @@ -144,6 +186,12 @@ def _credentials(self, ctx: PackContext): w.writerow(["learner", "url", "password"]) for n, info in sorted(ctx.state["learners"].items()): w.writerow([n, info["url"], pw.get(n, "(lost: destroy and relaunch this learner)")]) + ctx.mem["passwords"] = pw + return path + + def _credentials(self, ctx: PackContext): + path = self._write_creds(ctx) + pw = ctx.mem.get("passwords", {}) learners = sorted(ctx.state["learners"].items()) sp = ctx.params["show_passwords"] show = sp == "yes" or (sp == "auto" and len(learners) <= 3) @@ -153,7 +201,163 @@ def _credentials(self, ctx: PackContext): ctx.log(f"\n Passwords are in {path} (0600, one row per learner)" + ("" if show else f"\n View them: cat {path}")) return {"credentials_file": str(path)} + # ---- EC2 platform ------------------------------------------------------------------------------------------------------- + def _ec2_preflight(self, ctx: PackContext): + self._validate(ctx) + gw = ctx.services["ec2"] + gw.region, gw.profile = ctx.params["region"], ctx.params["profile"] + where = gw.check_auth() + itype = ec2mod.pick_instance_type(len(ctx.mem["names"]), int(ctx.params["memory_mib"]), ctx.params["instance_type"]) + if not re.fullmatch(r"[0-9a-f]{40}", str(ctx.params["ref"])): + raise PackError("ref must be a full 40-character commit SHA") + ctx.log(f" AWS: {where} | instance type: {itype}{' (spot)' if ctx.params['spot'] else ''} | learners: {len(ctx.mem['names'])}") + return {"aws": where, "instance_type": itype} + + def _ec2_network(self, ctx: PackContext): + return {"security_group": ctx.services["ec2"].security_group(ctx.name)} + + def _ec2_instance(self, ctx: PackContext): + gw, p = ctx.services["ec2"], ctx.params + names = ctx.mem.get("names") or learner_names(p["learners"]) + iid = ctx.state.get("instance_id") + if iid and gw.alive(iid): + ctx.log(f" instance {iid} already running") + ctx.mem["passwords"] = {} + return {} + pw = {n: secrets.token_urlsafe(12) for n in names} + env = {"GROQ_API_KEY": os.environ["GROQ_API_KEY"], "GROQ_MODEL": p["groq_model"], "LLM_PROVIDER": "groq"} if p["llm"] == "groq-shared" else {} + ud = ec2mod.build_user_data(pw, ref=p["ref"], hours=int(p["hours"]), idle_minutes=int(p["idle_minutes"]), cpu=float(p["cpu"]), + memory_mib=int(p["memory_mib"]), tls=p["tls"], env=env) + iid = gw.run(ctx.name, ctx.state["security_group"], ud, ctx.state["instance_type"], bool(p["spot"])) + ctx.log(f" launched {iid}, waiting for a public address ...") + ip = gw.public_ip(iid) + ctx.mem["passwords"] = pw + host = ec2mod.sslip_host(ip) + out = {"instance_id": iid, "host": host, "learners": {n: {"url": ec2mod.learner_url(host, n)} for n in names}} + ctx.state.update(out) + self._write_creds(ctx) # before the long wait: a failure from here on must not lose the passwords + return out + + def _ec2_wait(self, ctx: PackContext): + gw = ctx.services["ec2"] + host, names = ctx.state["host"], sorted(ctx.state["learners"]) + ctx.log(" the box is building the lab image (about 8 to 12 minutes the first time) ...") + for _ in range(WAIT_S // 15): + if all(gw.ready(host, n, verify=ctx.params["tls"] == "letsencrypt") for n in names): + ctx.log(" all learner IDEs answer over HTTPS") + return {"ready": True} + gw.pause(15) + raise PackError("learner IDEs were not reachable in time. Check the instance console log (/var/log/labagent-boot.log). Re-running resumes.") + + def _ec2_steps(self) -> List[Step]: + return [ + Step("preflight", "Check AWS credentials, account allow list and settings", self._ec2_preflight), + Step("network", "Create the HTTP/HTTPS-only security group", self._ec2_network), + Step("instance", "Launch the shared instance with one container per learner", self._ec2_instance), + Step("wait", "Wait for every learner's IDE over HTTPS", self._ec2_wait), + Step("credentials", "Write the credentials file", self._credentials), + ] + + def _ec2_destroy(self) -> List[Step]: + def preflight(c): + gw = c.services["ec2"] + gw.region, gw.profile = c.params["region"], c.params["profile"] + return {"aws": gw.check_auth()} + + def terminate(c): + gw = c.services["ec2"] + ids = sorted(({c.state["instance_id"]} if c.state.get("instance_id") else set()) | set(gw.find(c.name))) + gw.terminate(ids) + return {"instance_id": None, "host": None, "learners": {}, "terminated": len(ids)} + + def sg(c): + c.services["ec2"].delete_security_group(c.name) + return {"security_group": None} + + return [Step("preflight", "Check AWS credentials and account allow list", preflight), + Step("terminate", "Terminate the cohort instance (also finds orphans by tag)", terminate), + Step("network", "Delete the security group", sg), + Step("credentials", "Remove the credentials file", self._forget)] + + # ---- local Docker platform ------------------------------------------------------------------------------------------------ + def _local_image(self, ctx: PackContext): + image = ctx.params["image"] + image = localmod.IMAGE if image in ("auto", "pypi") else image # anything else is an existing image tag + ctx.services["local"].ensure_image(image, checkout_root()) + return {"image": image} + + def _local_preflight(self, ctx: PackContext): + self._validate(ctx) + where = ctx.services["local"].check_auth() + if not 1024 <= int(ctx.params["port"]) <= 65000 - len(ctx.mem["names"]): + raise PackError("port must be between 1024 and 65000") + ctx.log(f" {where} | learners: {len(ctx.mem['names'])} | first port: {ctx.params['port']}") + return {"docker": where} + + def _local_learners(self, ctx: PackContext): + gw, p = ctx.services["local"], ctx.params + names = ctx.mem.get("names") or learner_names(p["learners"]) + learners: Dict[str, Dict[str, str]] = dict(ctx.state.get("learners", {})) + passwords: Dict[str, str] = {} + env = {"GROQ_API_KEY": os.environ["GROQ_API_KEY"], "GROQ_MODEL": p["groq_model"], "LLM_PROVIDER": "groq"} if p["llm"] == "groq-shared" else {} + for i, n in enumerate(names): + if n in learners and gw.alive(learners[n]["container"]): + ctx.log(f" {n}: already running") + continue + gw.remove(localmod.container_name(ctx.name, n)) # a stopped container of the same name would block the new one + pw = secrets.token_urlsafe(12) + ctx.log(f" starting {n} ...") + learners[n] = gw.launch(ctx.name, n, pw, int(p["port"]) + i, image=ctx.state["image"], cpu=float(p["cpu"]), memory_mib=int(p["memory_mib"]), + hours=int(p["hours"]), idle_s=int(p["idle_minutes"]) * 60, env=env) + passwords[n] = pw + ctx.state["learners"] = learners + ctx.mem["passwords"] = passwords + self._write_creds(ctx) + return {"learners": learners} + + def _local_wait(self, ctx: PackContext): + gw = ctx.services["local"] + for _ in range(60): + if all(gw.ready(info["url"]) for info in ctx.state["learners"].values()): + ctx.log(" all learner IDEs answer") + return {"ready": True} + gw.pause(2) + raise PackError("learner IDEs did not answer in time. Check: docker logs ") + + def _local_steps(self) -> List[Step]: + return [ + Step("preflight", "Check Docker and settings", self._local_preflight), + Step("image", "Use or build the lab image", self._local_image), + Step("learners", "Start one container per learner on loopback", self._local_learners), + Step("wait", "Wait for every learner's IDE", self._local_wait), + Step("credentials", "Write the credentials file", self._credentials), + ] + + def _local_destroy(self) -> List[Step]: + def remove(c): + gw = c.services["local"] + names = {i["container"] for i in c.state.get("learners", {}).values()} | set(gw.find(c.name)) + for n in names: + gw.remove(n) + return {"learners": {}, "removed": len(names), "volumes": sorted({i["volume"] for i in c.state.get("learners", {}).values()} | set(c.state.get("volumes", [])))} + + def volumes(c): + if not c.params["keep_work"]: + for v in c.state.get("volumes", []): + c.services["local"].delete_volume(v) + return {"volumes": []} + return {} + + return [Step("preflight", "Check Docker", lambda c: {"docker": c.services["local"].check_auth()}), + Step("remove", "Remove every learner's container", remove), + Step("volumes", "Delete learner volumes (unless keep_work)", volumes), + Step("credentials", "Remove the credentials file", self._forget)] + def deploy_steps(self, ctx: PackContext) -> List[Step]: + if ctx.params["platform"] == "local": + return self._local_steps() + if ctx.params["platform"] == "ec2": + return self._ec2_steps() return [ Step("preflight", "Check the Modal login and settings", self._preflight), Step("image", "Build the lab image (first time: several minutes)", self._build), @@ -181,6 +385,10 @@ def _forget(self, ctx: PackContext): return {"credentials_file": None} def destroy_steps(self, ctx: PackContext) -> List[Step]: + if ctx.params["platform"] == "local": + return self._local_destroy() + if ctx.params["platform"] == "ec2": + return self._ec2_destroy() # remember volume names before the learner records are cleared ctx.state["volumes"] = sorted({i["volume"] for i in ctx.state.get("learners", {}).values() if i.get("volume")} | set(ctx.state.get("volumes", []))) return [ diff --git a/scripts/check_no_pii.py b/scripts/check_no_pii.py index 7513497..e0196be 100644 --- a/scripts/check_no_pii.py +++ b/scripts/check_no_pii.py @@ -2,7 +2,7 @@ """Fail if tracked files contain what must never be committed: real emails, AWS account IDs, keys. Run in CI and before every push. Allowed: example.com / example.org placeholders, -and the documentation account 123456789012. +the documentation account 123456789012, and Canonical's public AMI publisher id (a vendor id, not ours). """ import re import subprocess @@ -10,7 +10,7 @@ ALLOWED_EMAIL = re.compile(r"@(example\.(com|org)|users\.noreply\.github\.com)$", re.I) EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") -ACCOUNT = re.compile(r"(? self.ready_after + + def terminate(self, ids): + self.terminated += ids + for i in ids: self.instances[i] = False + + def delete_security_group(self, cohort): self.sg_deleted = True + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("LABAGENT_HOME", str(tmp_path)) + + +def launch(fake, **params): + lines = [] + env = run_pack(RagIde(), "launch", "c1", {"platform": "ec2", "learners": "2", **params}, services={"ec2": fake}, log=lines.append) + return env, "\n".join(lines) + + +def test_instance_type_scales_with_learners_and_rejects_nonsense(): + assert e.pick_instance_type(1, 4096) == "m6i.large" + assert e.pick_instance_type(5, 4096) == "m6i.xlarge" + assert e.pick_instance_type(30, 4096) == "m6i.4xlarge" + assert e.pick_instance_type(30, 4096, "c6i.4xlarge") == "c6i.4xlarge" + with pytest.raises(Exception): + e.pick_instance_type(30, 4096, "big; rm -rf /") + with pytest.raises(Exception): + e.pick_instance_type(200, 4096) + + +def test_user_data_is_pinned_limited_and_self_terminating(): + ud = e.build_user_data({"ana": "pw_ana_12345", "ben": "pw_ben_12345"}, ref=e.PINNED_REF, hours=3, idle_minutes=10, cpu=2, memory_mib=4096) + assert "shutdown -h +180" in ud and f'test "$(git rev-parse HEAD)" = "{e.PINNED_REF}"' in ud + assert "--cap-drop ALL" in ud and "no-new-privileges" in ud and "--memory 4096m" in ud and "--pids-limit" in ud and "--cpus 2" in ud + assert "--idle-timeout-seconds 600" in ud and "handle_path /ana/*" in ud and "reverse_proxy lab-ben:8080" in ud + assert "shutdown -h now" in ud and "sslip.io" in ud and "169.254.169.254" in ud + assert "tls internal" not in ud and "tls internal" in e.build_user_data({"a": "pw_a_123456"}, ref=e.PINNED_REF, hours=1, idle_minutes=5, cpu=1, memory_mib=2048, tls="internal") + + +def test_user_data_rejects_unsafe_values(): + ok = dict(ref=e.PINNED_REF, hours=1, idle_minutes=5, cpu=1, memory_mib=2048) + for bad in ({"a b": "pw_12345678"}, {"a": "short"}, {"a": "pw_1234; reboot"}, {"A": "pw_12345678"}): + with pytest.raises(Exception): + e.build_user_data(bad, **ok) + with pytest.raises(Exception): + e.build_user_data({"a": "pw_12345678"}, **{**ok, "ref": "main"}) + + +def test_launch_runs_one_instance_and_prints_https_paths_and_writes_credentials(): + fake = FakeEc2() + env, out = launch(fake, hours=2) + assert env.exit_code == 0 and len(fake.runs) == 1 and fake.runs[0]["type"] == "m6i.large" and fake.runs[0]["sg"] == "sg-123" + rows = list(csv.DictReader(creds_path("c1").open())) + assert [r["url"] for r in rows] == ["https://203-0-113-7.sslip.io/learner-01/", "https://203-0-113-7.sslip.io/learner-02/"] + assert all(r["password"] in fake.runs[0]["ud"] and r["password"] in out for r in rows) # 2 learners: shown + assert creds_path("c1").stat().st_mode & 0o777 == 0o600 + assert not any(r["password"] in str(load_state("rag-ide", "c1")) for r in rows) + + +def test_waits_until_all_ides_answer_then_continues(): + fake = FakeEc2(ready_after=5) + env, _ = launch(fake) + assert env.exit_code == 0 and fake.pauses and fake.polls > 5 + + +def test_not_ready_in_time_fails_with_where_to_look(monkeypatch): + from labagent.packs.rag_ide import pack as pk + monkeypatch.setattr(pk, "WAIT_S", 45) + env, _ = launch(FakeEc2(ready_after=10_000)) + assert env.exit_code != 0 and "boot.log" in str(env.__dict__) + + +def test_account_guard_blocks_before_anything_is_created(): + fake = FakeEc2(authed=False) + env, _ = launch(fake) + assert env.exit_code != 0 and "allowed_account_ids" in str(env.__dict__) and fake.runs == [] + + +def test_rerun_keeps_the_running_instance_and_passwords(): + fake = FakeEc2() + launch(fake) + first = list(csv.DictReader(creds_path("c1").open())) + launch(fake) + assert len(fake.runs) == 1 and list(csv.DictReader(creds_path("c1").open())) == first + + +def test_destroy_terminates_orphans_deletes_group_and_credentials(): + fake = FakeEc2() + launch(fake) + fake.instances["i-orphan"] = True + env = run_pack(RagIde(), "destroy", "c1", {"platform": "ec2"}, services={"ec2": fake}) + assert env.exit_code == 0 and sorted(fake.terminated) == ["i-0001", "i-orphan"] and fake.sg_deleted and not creds_path("c1").exists() + + +def test_bad_platform_and_ref_are_rejected(): + fake = FakeEc2() + assert launch(fake, ref="main")[0].exit_code != 0 and launch(fake, tls="none")[0].exit_code != 0 and fake.runs == [] + assert run_pack(RagIde(), "launch", "c1", {"platform": "gcp"}, services={"ec2": fake}).exit_code != 0 + + +# ---- the real gateway against moto ------------------------------------------------------------------------------------------ +boto3 = pytest.importorskip("boto3") +moto = pytest.importorskip("moto") + + +@pytest.fixture +def aws(monkeypatch): + from labagent.config import Config + with moto.mock_aws(): + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "x"); monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "x") + sess = boto3.Session(region_name="us-east-1") + ami = sess.client("ec2").describe_images()["Images"][0]["ImageId"] + cfg = Config(region="us-east-1", allowed_account_ids=["123456789012"]) + gw = e.Ec2Gateway(session=sess, config=cfg) + gw.ctx.sleep = lambda s: None + gw.ami = lambda: ami # moto cannot hold the reserved /aws/ SSM name; the name itself is checked against real AWS by hand + yield gw, sess + + +def test_gateway_enforces_the_account_allow_list(aws): + gw, sess = aws + assert "123456789012" in gw.check_auth() + gw.cfg.allowed_account_ids = ["999999999999"] + with pytest.raises(Exception, match="allowed_account_ids"): + gw.check_auth() + gw.cfg.allowed_account_ids = [] + with pytest.raises(Exception, match="allowed_account_ids"): + gw.check_auth() + + +def test_gateway_creates_a_locked_down_instance_and_cleans_up(aws): + gw, sess = aws + sg = gw.security_group("c1") + assert gw.security_group("c1") == sg # idempotent + perms = sess.client("ec2").describe_security_groups(GroupIds=[sg])["SecurityGroups"][0]["IpPermissions"] + assert sorted(p["FromPort"] for p in perms) == [80, 443] # no SSH + iid = gw.run("c1", sg, "#!/bin/bash\ntrue\n", "m6i.large", spot=False) + inst = sess.client("ec2").describe_instances(InstanceIds=[iid])["Reservations"][0]["Instances"][0] + assert inst["MetadataOptions"]["HttpTokens"] == "required" and inst["MetadataOptions"]["HttpPutResponseHopLimit"] == 1 + assert "IamInstanceProfile" not in inst # learners get no AWS credentials + assert {"Key": "cohort", "Value": "c1"} in inst["Tags"] and {"Key": "labagent", "Value": "rag-ide"} in inst["Tags"] + assert gw.public_ip(iid) and gw.alive(iid) and gw.find("c1") == [iid] and gw.find("other") == [] + gw.terminate([iid]) + assert not gw.alive(iid) and gw.find("c1") == [] + gw.delete_security_group("c1") + assert not sess.client("ec2").describe_security_groups(Filters=[{"Name": "group-name", "Values": ["labagent-rag-ide-c1"]}])["SecurityGroups"] + + +def test_gateway_spot_option_is_passed(aws): + gw, sess = aws + iid = gw.run("c1", gw.security_group("c1"), "#!/bin/bash\n", "m6i.large", spot=True) + inst = sess.client("ec2").describe_instances(InstanceIds=[iid])["Reservations"][0]["Instances"][0] + assert inst.get("InstanceLifecycle") == "spot" + + +def test_ami_lookup_falls_back_to_describe_images_when_ssm_is_denied(aws): + gw, sess = aws + del gw.ami # use the real method, not the test stub + class Denied: + def get_parameter(self, **k): raise RuntimeError("AccessDenied") + real = gw.ctx.client + gw.ctx.client = lambda svc, region=None: Denied() if svc == "ssm" else real(svc, region) + calls = {} + class Ec2Stub: + def describe_images(self, **k): + calls.update(k) + return {"Images": [{"ImageId": "ami-old", "CreationDate": "2026-01-01T00:00:00Z"}, {"ImageId": "ami-new", "CreationDate": "2026-09-01T00:00:00Z"}]} + gw.ctx.client = lambda svc, region=None: Denied() if svc == "ssm" else Ec2Stub() + assert gw.ami() == "ami-new" and calls["Owners"] == [e.CANONICAL_OWNER] + + +def test_dockerfile_hands_the_workspace_to_the_user_before_switching_to_it(): + """Regression: the classic builder makes WORKDIR root-owned even after USER, which broke the first real EC2 boot.""" + from pathlib import Path + lines = (Path(__file__).parent.parent / "Dockerfile.lab").read_text().splitlines() + chown = next(i for i, l in enumerate(lines) if "chown coder:coder /home/coder/lab" in l) + user = next(i for i, l in enumerate(lines) if l.strip() == "USER coder") + workdir = next(i for i, l in enumerate(lines) if l.startswith("WORKDIR /home/coder/lab")) + assert chown < user < workdir + + +def test_self_signed_tls_skips_certificate_verification_only_when_internal(): + f1 = FakeEc2() + launch(f1, tls="internal") + assert set(f1.verify_seen) == {False} + f2 = FakeEc2() + launch(f2, tls="letsencrypt") + assert set(f2.verify_seen) == {True} + + +def test_passwords_survive_a_failure_after_the_instance_starts(monkeypatch): + """Regression: the credentials file used to be written only at the very end, so a failed wait lost every password.""" + from labagent.packs.rag_ide import pack as pk + monkeypatch.setattr(pk, "WAIT_S", 30) + fake = FakeEc2(ready_after=10_000) + env, _ = launch(fake) + assert env.exit_code != 0 + rows = list(csv.DictReader(creds_path("c1").open())) + assert len(rows) == 2 and all(r["password"] in fake.runs[0]["ud"] for r in rows) + fake.ready_after = 0 + assert launch(fake)[0].exit_code == 0 and len(fake.runs) == 1 # resume reuses the instance + assert list(csv.DictReader(creds_path("c1").open())) == rows # and keeps the same passwords diff --git a/tests/test_rag_ide_local.py b/tests/test_rag_ide_local.py new file mode 100644 index 0000000..8a5a44b --- /dev/null +++ b/tests/test_rag_ide_local.py @@ -0,0 +1,135 @@ +import csv +import shutil +import subprocess +import urllib.request + +import pytest + +from labagent.packs.base import PackError, load_state, run_pack +from labagent.packs.rag_ide import local as lc +from labagent.packs.rag_ide.pack import RagIde, creds_path + + +class FakeLocal: + def __init__(self, ready_after=0, docker_up=True): + self.docker_up, self.ready_after, self.polls = docker_up, ready_after, 0 + self.running, self.launches, self.removed, self.volumes_deleted, self.images = {}, [], [], [], [] + self.pauses = [] + + def check_auth(self): + if not self.docker_up: + raise PackError("docker is installed but its daemon is not running. Start Docker Desktop and retry.") + return "local docker 29.0" + + def ensure_image(self, image, source): self.images.append((image, source)) + + def launch(self, cohort, learner, password, port, **kw): + c = lc.container_name(cohort, learner) + self.launches.append({"learner": learner, "password": password, "port": port, **kw}) + self.running[c] = True + return {"container": c, "url": lc.learner_url(port), "volume": lc.volume_name(cohort, learner), "port": str(port)} + + def ready(self, url): + self.polls += 1 + return self.polls > self.ready_after + + def alive(self, c): return self.running.get(c, False) + def find(self, cohort): return [c for c, up in self.running.items() if up] + def remove(self, c): self.removed.append(c); self.running[c] = False + def delete_volume(self, v): self.volumes_deleted.append(v) + def pause(self, s): self.pauses.append(s) + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("LABAGENT_HOME", str(tmp_path)) + + +def launch(fake, **params): + lines = [] + env = run_pack(RagIde(), "launch", "c1", {"platform": "local", "learners": "2", **params}, services={"local": fake}, log=lines.append) + return env, "\n".join(lines) + + +def test_launch_starts_each_learner_on_its_own_loopback_port_and_writes_credentials(): + fake = FakeLocal() + env, out = launch(fake, port=9100, hours=2, idle_minutes=5) + assert env.exit_code == 0 and [l["port"] for l in fake.launches] == [9100, 9101] + assert fake.launches[0]["hours"] == 2 and fake.launches[0]["idle_s"] == 300 and fake.images[0][0] == lc.IMAGE + rows = list(csv.DictReader(creds_path("c1").open())) + assert [r["url"] for r in rows] == ["http://127.0.0.1:9100/", "http://127.0.0.1:9101/"] + assert all(r["url"] in out and r["password"] in out for r in rows) # 2 learners: passwords shown + assert creds_path("c1").stat().st_mode & 0o777 == 0o600 + assert not any(r["password"] in str(load_state("rag-ide", "c1")) for r in rows) + + +def test_docker_not_running_fails_with_the_fix_and_starts_nothing(): + fake = FakeLocal(docker_up=False) + env, _ = launch(fake) + assert env.exit_code != 0 and "Start Docker Desktop" in str(env.__dict__) and fake.launches == [] + + +def test_rerun_keeps_running_learners_and_restarts_dead_ones(): + fake = FakeLocal() + launch(fake) + first = {r["learner"]: r["password"] for r in csv.DictReader(creds_path("c1").open())} + fake.running[lc.container_name("c1", "learner-01")] = False + launch(fake) + after = {r["learner"]: r["password"] for r in csv.DictReader(creds_path("c1").open())} + assert len(fake.launches) == 3 and fake.launches[2]["learner"] == "learner-01" + assert after["learner-02"] == first["learner-02"] and after["learner-01"] != first["learner-01"] + + +def test_waits_for_ides_and_fails_with_where_to_look(): + fake = FakeLocal(ready_after=3) + assert launch(fake)[0].exit_code == 0 and fake.pauses + env, _ = launch(FakeLocal(ready_after=10_000)) + assert env.exit_code != 0 and "docker logs" in str(env.__dict__) + + +def test_bad_port_and_platform_are_rejected(): + assert launch(FakeLocal(), port=80)[0].exit_code != 0 + assert run_pack(RagIde(), "launch", "c1", {"platform": "docker"}, services={"local": FakeLocal()}).exit_code != 0 + + +def test_destroy_removes_containers_orphans_volumes_and_credentials(): + fake = FakeLocal() + launch(fake) + fake.running["labagent-lab-c1-stray"] = True + env = run_pack(RagIde(), "destroy", "c1", {"platform": "local"}, services={"local": fake}) + assert env.exit_code == 0 and not any(fake.running.values()) and "labagent-lab-c1-stray" in fake.removed + assert sorted(fake.volumes_deleted) == [lc.volume_name("c1", "learner-01"), lc.volume_name("c1", "learner-02")] and not creds_path("c1").exists() + f2 = FakeLocal() + launch(f2) + run_pack(RagIde(), "destroy", "c1", {"platform": "local", "keep_work": True}, services={"local": f2}) + assert f2.volumes_deleted == [] + + +def test_docker_run_arguments_are_locked_down_and_keep_secrets_off_the_command_line(): + args = lc.run_args("c1", "ana", "pw_secret_12345", 9100, image="img:1", cpu=1.5, memory_mib=2048, hours=3, idle_s=600, env={"GROQ_API_KEY": "gsk_x"}) + joined = " ".join(args) + assert "127.0.0.1:9100:8080" in joined and "--cap-drop ALL" in joined and "no-new-privileges" in joined + assert "--memory 2048m" in joined and "--cpus 1.5" in joined and "--pids-limit" in joined + assert "pw_secret_12345" not in joined and "gsk_x" not in joined and "-e PASSWORD" in joined and "-e GROQ_API_KEY" in joined + assert "timeout 10800" in joined and "--idle-timeout-seconds 600" in joined and "--auth password" in joined + assert "--privileged" not in joined and "docker.sock" not in joined + + +def test_missing_image_without_a_dockerfile_gives_a_clear_error(tmp_path, monkeypatch): + gw = lc.LocalDockerGateway() + monkeypatch.setattr(gw, "_run", lambda *a, **k: subprocess.CompletedProcess(a, 1, "", "no such image")) + with pytest.raises(PackError, match="Dockerfile.lab"): + gw.ensure_image("nope:1", str(tmp_path)) + + +@pytest.mark.skipif(not shutil.which("docker") or subprocess.run(["docker", "image", "inspect", lc.IMAGE], capture_output=True).returncode != 0, + reason="needs a running Docker daemon and the labagent-rag-lab:local image") +def test_real_docker_end_to_end(): + env = run_pack(RagIde(), "launch", "itest", {"platform": "local", "learners": "1", "port": 18190, "hours": 1, "idle_minutes": 5}) + try: + assert env.exit_code == 0 + row = next(csv.DictReader(creds_path("itest").open())) + assert urllib.request.urlopen(row["url"] + "login", timeout=10).status == 200 + finally: + run_pack(RagIde(), "destroy", "itest", {"platform": "local"}) + assert not creds_path("itest").exists()