From 02310b76167058e66915ee9ff316b71f95a76981 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 23 Oct 2026 13:28:56 -0400 Subject: [PATCH 1/4] rag-ide: EC2 platform (one shared instance per cohort, one container per learner) --param platform=ec2 launches a single instance with a pinned lab image built from a verified git commit, a container per learner (unprivileged, capped, capabilities dropped), Caddy for HTTPS on an sslip.io name, a security group with only 80/443, IMDSv2 with hop limit 1, no instance profile, a hard time cap and self-shutdown when no learner is left. The account must be on the labagent allow list. Not yet run against a real AWS account. Co-Authored-By: Claude Sonnet 5.5 --- labagent/packs/rag_ide/ec2.py | 228 +++++++++++++++++++++++++++++++++ labagent/packs/rag_ide/pack.py | 109 +++++++++++++++- scripts/check_no_pii.py | 4 +- tests/test_rag_ide_ec2.py | 208 ++++++++++++++++++++++++++++++ 4 files changed, 546 insertions(+), 3 deletions(-) create mode 100644 labagent/packs/rag_ide/ec2.py create mode 100644 tests/test_rag_ide_ec2.py diff --git a/labagent/packs/rag_ide/ec2.py b/labagent/packs/rag_ide/ec2.py new file mode 100644 index 0000000..2e2a6cb --- /dev/null +++ b/labagent/packs/rag_ide/ec2.py @@ -0,0 +1,228 @@ +"""EC2 platform for rag-ide: ONE shared instance per cohort, one container per learner. + +Why not one VM per learner: learners are idle most of the time, so they share CPU, and a single box with a pinned image costs +a small fraction of per-learner serverless sandboxes. What keeps it safe: + - no SSH and no instance profile (learners get no AWS credentials); only ports 80 and 443 are open + - IMDSv2 required with hop limit 1, so containers cannot reach the instance metadata service + - every learner container is unprivileged, memory/CPU/pid limited, with all Linux capabilities dropped + - the box shuts itself down (and terminates) after `hours`, and also when no learner container is left running + - the account must be on the labagent allow list; the image source is a pinned git commit that is verified after checkout +HTTPS comes from Caddy and Let's Encrypt on a .sslip.io name, so no domain is needed. Learner URLs are paths on that name. +""" +from __future__ import annotations + +import math +import re +import shlex +from typing import Any, Dict, List, Optional, Tuple + +from ..base import PackError +from .modal_gateway import slug + +REPO = "https://github.com/becloudready/labagent" +PINNED_REF = "ce696d573cc2837d9594badc6e20c8090a8ceb6f" +CANONICAL_OWNER = "099720109477" +AMI_NAME = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" +SSM_UBUNTU = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id" +IMAGE = "labagent-rag-lab:local" +# (type, vCPU, GiB) smallest first. x86_64 only (the image is built on the box). +INSTANCE_TYPES: List[Tuple[str, int, int]] = [ + ("m6i.large", 2, 8), ("m6i.xlarge", 4, 16), ("m6i.2xlarge", 8, 32), ("m6i.4xlarge", 16, 64), ("m6i.8xlarge", 32, 128), +] +OVERSUBSCRIBE_RAM = 0.5 # learners rarely use their whole memory limit at the same time +LEARNERS_PER_VCPU = 3 # CPU is shared more aggressively than memory + + +def pick_instance_type(learners: int, memory_mib: int, override: str = "auto") -> str: + """Smallest type with RAM >= 50% of the sum of learner limits (plus 2 GiB system) and ~3 learners per vCPU.""" + if override != "auto": + if not re.fullmatch(r"[a-z0-9]+\.[a-z0-9]+", override): + raise PackError("instance_type must look like m6i.2xlarge, or be 'auto'") + return override + need_gib = learners * (memory_mib / 1024.0) * OVERSUBSCRIBE_RAM + 2 + need_cpu = math.ceil(learners / LEARNERS_PER_VCPU) + for t, cpu, gib in INSTANCE_TYPES: + if gib >= need_gib and cpu >= need_cpu: + return t + raise PackError("that many learners do not fit one instance: split them into two cohorts") + + +def sslip_host(ip: str) -> str: + return ip.replace(".", "-") + ".sslip.io" + + +def learner_url(host: str, learner: str) -> str: + return f"https://{host}/{learner}/" + + +def build_user_data(learners: Dict[str, str], *, ref: str, hours: int, idle_minutes: int, cpu: float, memory_mib: int, + tls: str = "letsencrypt", env: Optional[Dict[str, str]] = None) -> str: + """cloud-init script. `learners` maps name -> password. Passwords are in user-data, which anyone with EC2 read access in the + account can see: this is why the account guard matters. Learner containers cannot read it (hop limit 1).""" + if not re.fullmatch(r"[0-9a-f]{40}", ref): + raise PackError("ref must be a full 40-character commit SHA") + for n, pw in learners.items(): + if not re.fullmatch(r"[a-z0-9-]+", n) or not re.fullmatch(r"[A-Za-z0-9_-]{8,}", pw): + raise PackError("learner names and passwords must be plain [a-z0-9-] / [A-Za-z0-9_-] tokens") + q = shlex.quote + extra = " ".join(f"-e {q(k)}={q(v)}" for k, v in (env or {}).items()) + runs, routes = [], [] + for n, pw in learners.items(): + runs.append(f""" +docker volume create lab-{n} >/dev/null +docker run --rm -u root -v lab-{n}:/w {IMAGE} chown -R 1000:1000 /w +docker run -d --name lab-{n} --network labnet --cpus {cpu:g} --memory {memory_mib}m --pids-limit 1024 \\ + --cap-drop ALL --security-opt no-new-privileges -e PASSWORD={q(pw)} -e HF_HUB_OFFLINE=1 {extra} \\ + -v lab-{n}:/home/coder/lab/work {IMAGE} \\ + code-server --bind-addr 0.0.0.0:8080 --auth password --idle-timeout-seconds {idle_minutes * 60} /home/coder/lab""") + routes.append(f" redir /{n} /{n}/\n handle_path /{n}/* {{\n reverse_proxy lab-{n}:8080\n }}") + tls_line = " tls internal\n" if tls == "internal" else "" + return f"""#!/bin/bash +set -euxo pipefail +exec > >(tee /var/log/labagent-boot.log) 2>&1 +shutdown -h +{hours * 60} "labagent: hard time cap reached" +export DEBIAN_FRONTEND=noninteractive +apt-get update -y && apt-get install -y docker.io git curl +systemctl enable --now docker +TOKEN=$(curl -sS -X PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60') +IP=$(curl -sS -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/public-ipv4) +HOST=$(echo "$IP" | tr . -).sslip.io +mkdir -p /opt/labagent/src && cd /opt/labagent/src +git init -q && git remote add origin {REPO} && git fetch -q --depth 1 origin {ref} && git checkout -q FETCH_HEAD +test "$(git rev-parse HEAD)" = "{ref}" +docker build -q -f Dockerfile.lab -t {IMAGE} . +docker network create labnet +{''.join(runs)} +mkdir -p /opt/labagent/caddy +cat > /opt/labagent/caddy/Caddyfile < /usr/local/bin/labagent-idle-check <<'IDLE' +#!/bin/bash +# Stop paying when every learner has gone: shut down after {max(5, idle_minutes)} minutes with no running learner container. +n=$(docker ps --filter name=lab- --format '{{{{.Names}}}}' | wc -l) +f=/run/labagent-empty-since +if [ "$n" -gt 0 ]; then rm -f "$f"; exit 0; fi +[ -f "$f" ] || date +%s > "$f" +[ $(( $(date +%s) - $(cat "$f") )) -ge {max(5, idle_minutes) * 60} ] && shutdown -h now +IDLE +chmod +x /usr/local/bin/labagent-idle-check +echo '* * * * * root /usr/local/bin/labagent-idle-check' > /etc/cron.d/labagent-idle +echo READY > /var/log/labagent-ready +""" + + +class Ec2Gateway: + """Real AWS calls, guarded by the allow list. Imported lazily by the pack so tests can substitute a fake.""" + + def __init__(self, region: Optional[str] = None, profile: Optional[str] = None, session: Any = None, config: Any = None): + from ...config import load_config + from ...context import Ctx + import boto3 + self.cfg = config or load_config() + if region: + self.cfg.region = region + self.ctx = Ctx(self.cfg, session or boto3.Session(profile_name=profile, region_name=self.cfg.region)) + + def check_auth(self) -> str: + from ...context import GuardError + try: + self.ctx.guard_account() + except GuardError as e: + raise PackError(f"{e} Example: LABAGENT_ALLOWED_ACCOUNTS=") from e + except Exception as e: + raise PackError(f"could not read AWS credentials ({type(e).__name__}). Configure them or pass --profile") from e + return f"{self.ctx.account_id} in {self.cfg.region}" + + @property + def ec2(self): + return self.ctx.client("ec2") + + def ami(self) -> str: + """Latest Ubuntu 24.04 from Canonical. SSM is the clean source; some roles cannot read it, so fall back to DescribeImages.""" + try: + return self.ctx.client("ssm").get_parameter(Name=SSM_UBUNTU)["Parameter"]["Value"] + except Exception: + imgs = self.ec2.describe_images(Owners=[CANONICAL_OWNER], Filters=[{"Name": "name", "Values": [AMI_NAME]}, {"Name": "state", "Values": ["available"]}])["Images"] + if not imgs: + raise PackError("could not find an Ubuntu 24.04 image: allow ssm:GetParameter or ec2:DescribeImages for this identity") + return max(imgs, key=lambda i: i["CreationDate"])["ImageId"] + + def security_group(self, cohort: str) -> str: + name = f"labagent-rag-ide-{slug(cohort, 30)}" + found = self.ec2.describe_security_groups(Filters=[{"Name": "group-name", "Values": [name]}])["SecurityGroups"] + if found: + return found[0]["GroupId"] + vpc = self.ec2.describe_vpcs(Filters=[{"Name": "isDefault", "Values": ["true"]}])["Vpcs"] + if not vpc: + raise PackError("no default VPC in this region: create one or use another region") + sg = self.ec2.create_security_group(GroupName=name, Description="labagent rag-ide: HTTP/HTTPS only", VpcId=vpc[0]["VpcId"], + TagSpecifications=[{"ResourceType": "security-group", "Tags": [{"Key": "labagent", "Value": "rag-ide"}, {"Key": "cohort", "Value": slug(cohort)}]}])["GroupId"] + self.ec2.authorize_security_group_ingress(GroupId=sg, IpPermissions=[ + {"IpProtocol": "tcp", "FromPort": p, "ToPort": p, "IpRanges": [{"CidrIp": "0.0.0.0/0", "Description": "labagent learners"}]} for p in (80, 443)]) + return sg + + def run(self, cohort: str, sg: str, user_data: str, instance_type: str, spot: bool) -> str: + kwargs: Dict[str, Any] = dict( + ImageId=self.ami(), InstanceType=instance_type, MinCount=1, MaxCount=1, UserData=user_data, SecurityGroupIds=[sg], + InstanceInitiatedShutdownBehavior="terminate", + MetadataOptions={"HttpTokens": "required", "HttpPutResponseHopLimit": 1, "HttpEndpoint": "enabled"}, + BlockDeviceMappings=[{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 60, "VolumeType": "gp3", "DeleteOnTermination": True, "Encrypted": True}}], + TagSpecifications=[{"ResourceType": "instance", "Tags": [{"Key": "labagent", "Value": "rag-ide"}, {"Key": "cohort", "Value": slug(cohort)}, + {"Key": "Name", "Value": f"labagent-rag-ide-{slug(cohort)}"}]}]) + if spot: + kwargs["InstanceMarketOptions"] = {"MarketType": "spot", "SpotOptions": {"SpotInstanceType": "one-time", "InstanceInterruptionBehavior": "terminate"}} + return self.ec2.run_instances(**kwargs)["Instances"][0]["InstanceId"] + + def public_ip(self, instance_id: str, timeout_s: int = 300) -> str: + waited = 0 + while waited <= timeout_s: + r = self.ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0]["Instances"][0] + if r["State"]["Name"] == "running" and r.get("PublicIpAddress"): + return r["PublicIpAddress"] + if r["State"]["Name"] in ("shutting-down", "terminated"): + raise PackError("the instance terminated during startup (spot capacity or a quota?)") + self.ctx.sleep(5) + waited += 5 + raise PackError("the instance did not get a public IP in time") + + def pause(self, seconds: float) -> None: + self.ctx.sleep(seconds) + + def ready(self, host: str, learner: str) -> bool: + """True once Caddy serves the learner's login page over valid HTTPS.""" + import urllib.request + try: + with urllib.request.urlopen(learner_url(host, learner) + "login", timeout=10) as r: + return r.status == 200 + except Exception: + return False + + def alive(self, instance_id: str) -> bool: + try: + r = self.ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0]["Instances"][0] + return r["State"]["Name"] in ("pending", "running") + except Exception: + return False + + def find(self, cohort: str) -> List[str]: + res = self.ec2.describe_instances(Filters=[{"Name": "tag:labagent", "Values": ["rag-ide"]}, {"Name": "tag:cohort", "Values": [slug(cohort)]}, + {"Name": "instance-state-name", "Values": ["pending", "running", "stopping", "stopped"]}]) + return [i["InstanceId"] for r in res["Reservations"] for i in r["Instances"]] + + def terminate(self, ids: List[str]) -> None: + if ids: + self.ec2.terminate_instances(InstanceIds=ids) + self.ec2.get_waiter("instance_terminated").wait(InstanceIds=ids, WaiterConfig={"Delay": 5, "MaxAttempts": 60}) + + def delete_security_group(self, cohort: str) -> None: + name = f"labagent-rag-ide-{slug(cohort, 30)}" + for g in self.ec2.describe_security_groups(Filters=[{"Name": "group-name", "Values": [name]}, {"Name": "tag:labagent", "Values": ["rag-ide"]}])["SecurityGroups"]: + try: + self.ec2.delete_security_group(GroupId=g["GroupId"]) + except Exception: + pass # still attached for a few seconds after termination; the next destroy removes it diff --git a/labagent/packs/rag_ide/pack.py b/labagent/packs/rag_ide/pack.py index 420ddad..8bf2d88 100644 --- a/labagent/packs/rag_ide/pack.py +++ b/labagent/packs/rag_ide/pack.py @@ -13,9 +13,11 @@ from typing import Any, Dict, List from ..base import Pack, PackContext, PackError, Param, Step, state_dir +from . import ec2 as ec2mod from .modal_gateway import ModalGateway, app_name, slug MAX_LEARNERS = 60 +WAIT_S = 1800 # image build on the box plus certificate issuance def learner_names(spec: Any) -> List[str]: @@ -57,6 +59,20 @@ def __getattr__(self, name: str): return getattr(self._gw, name) +class _LazyEc2: + """Same idea as _LazyModal; region and profile are applied in preflight, before the first AWS call.""" + + def __init__(self) -> None: + self._gw = None + self.region = None + self.profile = None + + def __getattr__(self, name: str): + if self._gw is None: + self._gw = ec2mod.Ec2Gateway(region=self.region, profile=self.profile) + return getattr(self._gw, name) + + class RagIde(Pack): id = "rag-ide" title = "Browser VS Code lab per learner (Modal)" @@ -64,7 +80,14 @@ class RagIde(Pack): aliases = ("ide", "web-ide", "rag-web-ide") default_name = "cohort" params = [ + Param("platform", "modal", "modal (one serverless sandbox per learner) | ec2 (one shared instance per cohort, much cheaper at scale)"), Param("learners", "1", "Count (5) or comma list of names (ana,ben)"), + Param("instance_type", "auto", "ec2: instance type, or auto"), + Param("spot", False, "ec2: use a spot instance (cheaper, can be reclaimed)"), + Param("region", None, "ec2: AWS region (default from labagent config)"), + Param("profile", None, "ec2: AWS profile"), + Param("tls", "letsencrypt", "ec2: letsencrypt | internal (self-signed, browser warning)"), + Param("ref", ec2mod.PINNED_REF, "ec2: labagent commit (40-hex SHA) the lab image is built from"), Param("hours", 4, "Hard cap per session, 1 to 24"), Param("idle_minutes", 15, "Shut a session down after this long with no browser connected"), Param("cpu", 2.0, "CPU cores per learner"), @@ -77,10 +100,14 @@ class RagIde(Pack): ] def services(self, profile): - return {"modal": _LazyModal()} + return {"modal": _LazyModal(), "ec2": _LazyEc2()} def _validate(self, ctx: PackContext) -> None: p = ctx.params + if p["platform"] not in ("modal", "ec2"): + raise PackError("platform must be modal or ec2") + if p["tls"] not in ("letsencrypt", "internal"): + raise PackError("tls must be letsencrypt or internal") names = learner_names(p["learners"]) if not 1 <= int(p["hours"]) <= 24: raise PackError("hours must be between 1 and 24") @@ -153,7 +180,85 @@ def _credentials(self, ctx: PackContext): ctx.log(f"\n Passwords are in {path} (0600, one row per learner)" + ("" if show else f"\n View them: cat {path}")) return {"credentials_file": str(path)} + # ---- EC2 platform ------------------------------------------------------------------------------------------------------- + def _ec2_preflight(self, ctx: PackContext): + self._validate(ctx) + gw = ctx.services["ec2"] + gw.region, gw.profile = ctx.params["region"], ctx.params["profile"] + where = gw.check_auth() + itype = ec2mod.pick_instance_type(len(ctx.mem["names"]), int(ctx.params["memory_mib"]), ctx.params["instance_type"]) + if not re.fullmatch(r"[0-9a-f]{40}", str(ctx.params["ref"])): + raise PackError("ref must be a full 40-character commit SHA") + ctx.log(f" AWS: {where} | instance type: {itype}{' (spot)' if ctx.params['spot'] else ''} | learners: {len(ctx.mem['names'])}") + return {"aws": where, "instance_type": itype} + + def _ec2_network(self, ctx: PackContext): + return {"security_group": ctx.services["ec2"].security_group(ctx.name)} + + def _ec2_instance(self, ctx: PackContext): + gw, p = ctx.services["ec2"], ctx.params + names = ctx.mem.get("names") or learner_names(p["learners"]) + iid = ctx.state.get("instance_id") + if iid and gw.alive(iid): + ctx.log(f" instance {iid} already running") + ctx.mem["passwords"] = {} + return {} + pw = {n: secrets.token_urlsafe(12) for n in names} + env = {"GROQ_API_KEY": os.environ["GROQ_API_KEY"], "GROQ_MODEL": p["groq_model"], "LLM_PROVIDER": "groq"} if p["llm"] == "groq-shared" else {} + ud = ec2mod.build_user_data(pw, ref=p["ref"], hours=int(p["hours"]), idle_minutes=int(p["idle_minutes"]), cpu=float(p["cpu"]), + memory_mib=int(p["memory_mib"]), tls=p["tls"], env=env) + iid = gw.run(ctx.name, ctx.state["security_group"], ud, ctx.state["instance_type"], bool(p["spot"])) + ctx.log(f" launched {iid}, waiting for a public address ...") + ip = gw.public_ip(iid) + ctx.mem["passwords"] = pw + host = ec2mod.sslip_host(ip) + return {"instance_id": iid, "host": host, + "learners": {n: {"url": ec2mod.learner_url(host, n)} for n in names}} + + def _ec2_wait(self, ctx: PackContext): + gw = ctx.services["ec2"] + host, names = ctx.state["host"], sorted(ctx.state["learners"]) + ctx.log(" the box is building the lab image (about 8 to 12 minutes the first time) ...") + for _ in range(WAIT_S // 15): + if all(gw.ready(host, n) for n in names): + ctx.log(" all learner IDEs answer over HTTPS") + return {"ready": True} + gw.pause(15) + raise PackError("learner IDEs were not reachable in time. Check the instance console log (/var/log/labagent-boot.log). Re-running resumes.") + + def _ec2_steps(self) -> List[Step]: + return [ + Step("preflight", "Check AWS credentials, account allow list and settings", self._ec2_preflight), + Step("network", "Create the HTTP/HTTPS-only security group", self._ec2_network), + Step("instance", "Launch the shared instance with one container per learner", self._ec2_instance), + Step("wait", "Wait for every learner's IDE over HTTPS", self._ec2_wait), + Step("credentials", "Write the credentials file", self._credentials), + ] + + def _ec2_destroy(self) -> List[Step]: + def preflight(c): + gw = c.services["ec2"] + gw.region, gw.profile = c.params["region"], c.params["profile"] + return {"aws": gw.check_auth()} + + def terminate(c): + gw = c.services["ec2"] + ids = sorted(({c.state["instance_id"]} if c.state.get("instance_id") else set()) | set(gw.find(c.name))) + gw.terminate(ids) + return {"instance_id": None, "host": None, "learners": {}, "terminated": len(ids)} + + def sg(c): + c.services["ec2"].delete_security_group(c.name) + return {"security_group": None} + + return [Step("preflight", "Check AWS credentials and account allow list", preflight), + Step("terminate", "Terminate the cohort instance (also finds orphans by tag)", terminate), + Step("network", "Delete the security group", sg), + Step("credentials", "Remove the credentials file", self._forget)] + def deploy_steps(self, ctx: PackContext) -> List[Step]: + if ctx.params["platform"] == "ec2": + return self._ec2_steps() return [ Step("preflight", "Check the Modal login and settings", self._preflight), Step("image", "Build the lab image (first time: several minutes)", self._build), @@ -181,6 +286,8 @@ def _forget(self, ctx: PackContext): return {"credentials_file": None} def destroy_steps(self, ctx: PackContext) -> List[Step]: + if ctx.params["platform"] == "ec2": + return self._ec2_destroy() # remember volume names before the learner records are cleared ctx.state["volumes"] = sorted({i["volume"] for i in ctx.state.get("learners", {}).values() if i.get("volume")} | set(ctx.state.get("volumes", []))) return [ diff --git a/scripts/check_no_pii.py b/scripts/check_no_pii.py index 7513497..e0196be 100644 --- a/scripts/check_no_pii.py +++ b/scripts/check_no_pii.py @@ -2,7 +2,7 @@ """Fail if tracked files contain what must never be committed: real emails, AWS account IDs, keys. Run in CI and before every push. Allowed: example.com / example.org placeholders, -and the documentation account 123456789012. +the documentation account 123456789012, and Canonical's public AMI publisher id (a vendor id, not ours). """ import re import subprocess @@ -10,7 +10,7 @@ ALLOWED_EMAIL = re.compile(r"@(example\.(com|org)|users\.noreply\.github\.com)$", re.I) EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") -ACCOUNT = re.compile(r"(? self.ready_after + + def terminate(self, ids): + self.terminated += ids + for i in ids: self.instances[i] = False + + def delete_security_group(self, cohort): self.sg_deleted = True + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("LABAGENT_HOME", str(tmp_path)) + + +def launch(fake, **params): + lines = [] + env = run_pack(RagIde(), "launch", "c1", {"platform": "ec2", "learners": "2", **params}, services={"ec2": fake}, log=lines.append) + return env, "\n".join(lines) + + +def test_instance_type_scales_with_learners_and_rejects_nonsense(): + assert e.pick_instance_type(1, 4096) == "m6i.large" + assert e.pick_instance_type(5, 4096) == "m6i.xlarge" + assert e.pick_instance_type(30, 4096) == "m6i.4xlarge" + assert e.pick_instance_type(30, 4096, "c6i.4xlarge") == "c6i.4xlarge" + with pytest.raises(Exception): + e.pick_instance_type(30, 4096, "big; rm -rf /") + with pytest.raises(Exception): + e.pick_instance_type(200, 4096) + + +def test_user_data_is_pinned_limited_and_self_terminating(): + ud = e.build_user_data({"ana": "pw_ana_12345", "ben": "pw_ben_12345"}, ref=e.PINNED_REF, hours=3, idle_minutes=10, cpu=2, memory_mib=4096) + assert "shutdown -h +180" in ud and f'test "$(git rev-parse HEAD)" = "{e.PINNED_REF}"' in ud + assert "--cap-drop ALL" in ud and "no-new-privileges" in ud and "--memory 4096m" in ud and "--pids-limit" in ud and "--cpus 2" in ud + assert "--idle-timeout-seconds 600" in ud and "handle_path /ana/*" in ud and "reverse_proxy lab-ben:8080" in ud + assert "shutdown -h now" in ud and "sslip.io" in ud and "169.254.169.254" in ud + assert "tls internal" not in ud and "tls internal" in e.build_user_data({"a": "pw_a_123456"}, ref=e.PINNED_REF, hours=1, idle_minutes=5, cpu=1, memory_mib=2048, tls="internal") + + +def test_user_data_rejects_unsafe_values(): + ok = dict(ref=e.PINNED_REF, hours=1, idle_minutes=5, cpu=1, memory_mib=2048) + for bad in ({"a b": "pw_12345678"}, {"a": "short"}, {"a": "pw_1234; reboot"}, {"A": "pw_12345678"}): + with pytest.raises(Exception): + e.build_user_data(bad, **ok) + with pytest.raises(Exception): + e.build_user_data({"a": "pw_12345678"}, **{**ok, "ref": "main"}) + + +def test_launch_runs_one_instance_and_prints_https_paths_and_writes_credentials(): + fake = FakeEc2() + env, out = launch(fake, hours=2) + assert env.exit_code == 0 and len(fake.runs) == 1 and fake.runs[0]["type"] == "m6i.large" and fake.runs[0]["sg"] == "sg-123" + rows = list(csv.DictReader(creds_path("c1").open())) + assert [r["url"] for r in rows] == ["https://203-0-113-7.sslip.io/learner-01/", "https://203-0-113-7.sslip.io/learner-02/"] + assert all(r["password"] in fake.runs[0]["ud"] and r["password"] in out for r in rows) # 2 learners: shown + assert creds_path("c1").stat().st_mode & 0o777 == 0o600 + assert not any(r["password"] in str(load_state("rag-ide", "c1")) for r in rows) + + +def test_waits_until_all_ides_answer_then_continues(): + fake = FakeEc2(ready_after=5) + env, _ = launch(fake) + assert env.exit_code == 0 and fake.pauses and fake.polls > 5 + + +def test_not_ready_in_time_fails_with_where_to_look(monkeypatch): + from labagent.packs.rag_ide import pack as pk + monkeypatch.setattr(pk, "WAIT_S", 45) + env, _ = launch(FakeEc2(ready_after=10_000)) + assert env.exit_code != 0 and "boot.log" in str(env.__dict__) + + +def test_account_guard_blocks_before_anything_is_created(): + fake = FakeEc2(authed=False) + env, _ = launch(fake) + assert env.exit_code != 0 and "allowed_account_ids" in str(env.__dict__) and fake.runs == [] + + +def test_rerun_keeps_the_running_instance_and_passwords(): + fake = FakeEc2() + launch(fake) + first = list(csv.DictReader(creds_path("c1").open())) + launch(fake) + assert len(fake.runs) == 1 and list(csv.DictReader(creds_path("c1").open())) == first + + +def test_destroy_terminates_orphans_deletes_group_and_credentials(): + fake = FakeEc2() + launch(fake) + fake.instances["i-orphan"] = True + env = run_pack(RagIde(), "destroy", "c1", {"platform": "ec2"}, services={"ec2": fake}) + assert env.exit_code == 0 and sorted(fake.terminated) == ["i-0001", "i-orphan"] and fake.sg_deleted and not creds_path("c1").exists() + + +def test_bad_platform_and_ref_are_rejected(): + fake = FakeEc2() + assert launch(fake, ref="main")[0].exit_code != 0 and launch(fake, tls="none")[0].exit_code != 0 and fake.runs == [] + assert run_pack(RagIde(), "launch", "c1", {"platform": "gcp"}, services={"ec2": fake}).exit_code != 0 + + +# ---- the real gateway against moto ------------------------------------------------------------------------------------------ +boto3 = pytest.importorskip("boto3") +moto = pytest.importorskip("moto") + + +@pytest.fixture +def aws(monkeypatch): + from labagent.config import Config + with moto.mock_aws(): + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "x"); monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "x") + sess = boto3.Session(region_name="us-east-1") + ami = sess.client("ec2").describe_images()["Images"][0]["ImageId"] + cfg = Config(region="us-east-1", allowed_account_ids=["123456789012"]) + gw = e.Ec2Gateway(session=sess, config=cfg) + gw.ctx.sleep = lambda s: None + gw.ami = lambda: ami # moto cannot hold the reserved /aws/ SSM name; the name itself is checked against real AWS by hand + yield gw, sess + + +def test_gateway_enforces_the_account_allow_list(aws): + gw, sess = aws + assert "123456789012" in gw.check_auth() + gw.cfg.allowed_account_ids = ["999999999999"] + with pytest.raises(Exception, match="allowed_account_ids"): + gw.check_auth() + gw.cfg.allowed_account_ids = [] + with pytest.raises(Exception, match="allowed_account_ids"): + gw.check_auth() + + +def test_gateway_creates_a_locked_down_instance_and_cleans_up(aws): + gw, sess = aws + sg = gw.security_group("c1") + assert gw.security_group("c1") == sg # idempotent + perms = sess.client("ec2").describe_security_groups(GroupIds=[sg])["SecurityGroups"][0]["IpPermissions"] + assert sorted(p["FromPort"] for p in perms) == [80, 443] # no SSH + iid = gw.run("c1", sg, "#!/bin/bash\ntrue\n", "m6i.large", spot=False) + inst = sess.client("ec2").describe_instances(InstanceIds=[iid])["Reservations"][0]["Instances"][0] + assert inst["MetadataOptions"]["HttpTokens"] == "required" and inst["MetadataOptions"]["HttpPutResponseHopLimit"] == 1 + assert "IamInstanceProfile" not in inst # learners get no AWS credentials + assert {"Key": "cohort", "Value": "c1"} in inst["Tags"] and {"Key": "labagent", "Value": "rag-ide"} in inst["Tags"] + assert gw.public_ip(iid) and gw.alive(iid) and gw.find("c1") == [iid] and gw.find("other") == [] + gw.terminate([iid]) + assert not gw.alive(iid) and gw.find("c1") == [] + gw.delete_security_group("c1") + assert not sess.client("ec2").describe_security_groups(Filters=[{"Name": "group-name", "Values": ["labagent-rag-ide-c1"]}])["SecurityGroups"] + + +def test_gateway_spot_option_is_passed(aws): + gw, sess = aws + iid = gw.run("c1", gw.security_group("c1"), "#!/bin/bash\n", "m6i.large", spot=True) + inst = sess.client("ec2").describe_instances(InstanceIds=[iid])["Reservations"][0]["Instances"][0] + assert inst.get("InstanceLifecycle") == "spot" + + +def test_ami_lookup_falls_back_to_describe_images_when_ssm_is_denied(aws): + gw, sess = aws + del gw.ami # use the real method, not the test stub + class Denied: + def get_parameter(self, **k): raise RuntimeError("AccessDenied") + real = gw.ctx.client + gw.ctx.client = lambda svc, region=None: Denied() if svc == "ssm" else real(svc, region) + calls = {} + class Ec2Stub: + def describe_images(self, **k): + calls.update(k) + return {"Images": [{"ImageId": "ami-old", "CreationDate": "2026-01-01T00:00:00Z"}, {"ImageId": "ami-new", "CreationDate": "2026-09-01T00:00:00Z"}]} + gw.ctx.client = lambda svc, region=None: Denied() if svc == "ssm" else Ec2Stub() + assert gw.ami() == "ami-new" and calls["Owners"] == [e.CANONICAL_OWNER] From 737a284413ae94bb69cd425935df0f31e3919a28 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 23 Oct 2026 14:30:05 -0400 Subject: [PATCH 2/4] Dockerfile.lab: hand the workspace to the user before USER (classic builder) The classic Docker builder (Ubuntu's docker.io has no BuildKit) creates WORKDIR as root even after USER, so 'labagent rag init' failed with a PermissionError on the first real EC2 boot. Create and chown the directory before switching user. Co-Authored-By: Claude Sonnet 5.5 --- Dockerfile.lab | 3 +++ tests/test_rag_ide_ec2.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/Dockerfile.lab b/Dockerfile.lab index 514f51f..9c56f6f 100644 --- a/Dockerfile.lab +++ b/Dockerfile.lab @@ -36,6 +36,9 @@ RUN pip install --no-cache-dir "/src[rag-lab]" RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')" \ && chmod -R a+rX /opt/hf-cache +# Create the workspace as root and hand it over BEFORE switching user: the classic Docker builder (Ubuntu's docker.io, no +# BuildKit) creates a WORKDIR as root even after USER, which made `labagent rag init` fail with a PermissionError. +RUN mkdir -p /home/coder/lab && chown coder:coder /home/coder/lab USER coder WORKDIR /home/coder/lab RUN labagent rag init --dir /home/coder/lab && python -m ipykernel install --user --name python3 diff --git a/tests/test_rag_ide_ec2.py b/tests/test_rag_ide_ec2.py index a5fe232..93ee798 100644 --- a/tests/test_rag_ide_ec2.py +++ b/tests/test_rag_ide_ec2.py @@ -206,3 +206,13 @@ def describe_images(self, **k): return {"Images": [{"ImageId": "ami-old", "CreationDate": "2026-01-01T00:00:00Z"}, {"ImageId": "ami-new", "CreationDate": "2026-09-01T00:00:00Z"}]} gw.ctx.client = lambda svc, region=None: Denied() if svc == "ssm" else Ec2Stub() assert gw.ami() == "ami-new" and calls["Owners"] == [e.CANONICAL_OWNER] + + +def test_dockerfile_hands_the_workspace_to_the_user_before_switching_to_it(): + """Regression: the classic builder makes WORKDIR root-owned even after USER, which broke the first real EC2 boot.""" + from pathlib import Path + lines = (Path(__file__).parent.parent / "Dockerfile.lab").read_text().splitlines() + chown = next(i for i, l in enumerate(lines) if "chown coder:coder /home/coder/lab" in l) + user = next(i for i, l in enumerate(lines) if l.strip() == "USER coder") + workdir = next(i for i, l in enumerate(lines) if l.startswith("WORKDIR /home/coder/lab")) + assert chown < user < workdir From 09373ace3af42edf1a4573053a5be4054169a386 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 23 Oct 2026 14:30:23 -0400 Subject: [PATCH 3/4] rag-ide ec2: pin the lab image to the Dockerfile fix Co-Authored-By: Claude Sonnet 5.5 --- labagent/packs/rag_ide/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labagent/packs/rag_ide/ec2.py b/labagent/packs/rag_ide/ec2.py index 2e2a6cb..050fc4b 100644 --- a/labagent/packs/rag_ide/ec2.py +++ b/labagent/packs/rag_ide/ec2.py @@ -20,7 +20,7 @@ from .modal_gateway import slug REPO = "https://github.com/becloudready/labagent" -PINNED_REF = "ce696d573cc2837d9594badc6e20c8090a8ceb6f" +PINNED_REF = "737a284413ae94bb69cd425935df0f31e3919a28" CANONICAL_OWNER = "099720109477" AMI_NAME = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" SSM_UBUNTU = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id" From 7ec702e5fa7d040f10bdf527cfafec6d507263ce Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 23 Oct 2026 14:41:15 -0400 Subject: [PATCH 4/4] rag-ide: local Docker platform; fix EC2 readiness and password loss on resume platform=local runs the same one-container-per-learner lab on this machine through Docker (loopback ports, same credentials file and destroy), for demos and tests. Two bugs found by the first real EC2 run: - readiness check verified the certificate, so tls=internal could never become ready - passwords lived only in memory until the last step, so a failure after the instance started lost them; the credentials file is now written as soon as learners exist Co-Authored-By: Claude Sonnet 5.5 --- labagent/packs/rag_ide/ec2.py | 8 +- labagent/packs/rag_ide/local.py | 112 ++++++++++++++++++++++++++ labagent/packs/rag_ide/pack.py | 117 +++++++++++++++++++++++++-- tests/test_rag_ide_ec2.py | 26 +++++- tests/test_rag_ide_local.py | 135 ++++++++++++++++++++++++++++++++ 5 files changed, 386 insertions(+), 12 deletions(-) create mode 100644 labagent/packs/rag_ide/local.py create mode 100644 tests/test_rag_ide_local.py diff --git a/labagent/packs/rag_ide/ec2.py b/labagent/packs/rag_ide/ec2.py index 050fc4b..041f042 100644 --- a/labagent/packs/rag_ide/ec2.py +++ b/labagent/packs/rag_ide/ec2.py @@ -193,11 +193,13 @@ def public_ip(self, instance_id: str, timeout_s: int = 300) -> str: def pause(self, seconds: float) -> None: self.ctx.sleep(seconds) - def ready(self, host: str, learner: str) -> bool: - """True once Caddy serves the learner's login page over valid HTTPS.""" + def ready(self, host: str, learner: str, verify: bool = True) -> bool: + """True once Caddy serves the learner's login page. verify=False only for tls=internal, whose certificate is self-signed.""" + import ssl import urllib.request + ctx = None if verify else ssl._create_unverified_context() try: - with urllib.request.urlopen(learner_url(host, learner) + "login", timeout=10) as r: + with urllib.request.urlopen(learner_url(host, learner) + "login", timeout=10, context=ctx) as r: return r.status == 200 except Exception: return False diff --git a/labagent/packs/rag_ide/local.py b/labagent/packs/rag_ide/local.py new file mode 100644 index 0000000..356bd63 --- /dev/null +++ b/labagent/packs/rag_ide/local.py @@ -0,0 +1,112 @@ +"""Local platform for rag-ide: the same one-container-per-learner lab, on this machine through Docker. For demos and tests. + +It mirrors the EC2 workflow (same pack steps, credentials file and destroy), minus the cloud parts: no HTTPS proxy, each learner +gets http://127.0.0.1:. Every port is bound to loopback only, containers are unprivileged and capped, and a password is +always required. Nothing here talks to a cloud account. +""" +from __future__ import annotations + +import subprocess +import urllib.request +from pathlib import Path +from typing import Dict, List, Optional + +from ..base import PackError +from .ec2 import IMAGE +from .modal_gateway import slug + +LAB_DIR = "/home/coder/lab" + + +def container_name(cohort: str, learner: str) -> str: + return f"labagent-lab-{slug(cohort, 20)}-{slug(learner, 20)}" + + +def volume_name(cohort: str, learner: str) -> str: + return f"labagent-lab-{slug(cohort, 20)}-{slug(learner, 20)}-work" + + +def learner_url(port: int) -> str: + return f"http://127.0.0.1:{port}/" + + +def run_args(cohort: str, learner: str, password: str, port: int, *, image: str, cpu: float, memory_mib: int, hours: int, + idle_s: int, env: Optional[Dict[str, str]] = None) -> List[str]: + """The docker run command for one learner (a list, never a shell string). The password travels in the environment of the + docker CLI process (`-e PASSWORD`), not in the argument list, so it does not show up in `ps`.""" + args = ["docker", "run", "-d", "--name", container_name(cohort, learner), "-p", f"127.0.0.1:{port}:8080", + "--label", "labagent=rag-ide", "--label", f"cohort={slug(cohort)}", "--label", f"learner={slug(learner)}", + "--cpus", f"{cpu:g}", "--memory", f"{memory_mib}m", "--pids-limit", "1024", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "-e", "PASSWORD", "-e", "HF_HUB_OFFLINE=1"] + for k in (env or {}): + args += ["-e", k] + args += ["-v", f"{volume_name(cohort, learner)}:{LAB_DIR}/work", image, + "sh", "-c", f"exec timeout {hours * 3600} code-server --bind-addr 0.0.0.0:8080 --auth password " + f"--idle-timeout-seconds {idle_s} {LAB_DIR}"] + return args + + +class LocalDockerGateway: + def __init__(self) -> None: + self.docker = "docker" + + def _run(self, args: List[str], env: Optional[Dict[str, str]] = None, check: bool = True) -> subprocess.CompletedProcess: + import os + r = subprocess.run(args, capture_output=True, text=True, env={**os.environ, **(env or {})}) + if check and r.returncode != 0: + raise PackError(f"docker failed: {(r.stderr or r.stdout).strip().splitlines()[-1][:200] if (r.stderr or r.stdout).strip() else 'unknown error'}") + return r + + def check_auth(self) -> str: + try: + r = self._run([self.docker, "info", "--format", "{{.ServerVersion}}"], check=False) + except FileNotFoundError as e: + raise PackError("docker is not installed. Install Docker Desktop, or use platform=ec2 or platform=modal.") from e + if r.returncode != 0: + raise PackError("docker is installed but its daemon is not running. Start Docker Desktop and retry.") + return f"local docker {r.stdout.strip()}" + + def ensure_image(self, image: str, source: str) -> None: + if self._run([self.docker, "image", "inspect", image], check=False).returncode == 0: + return + dockerfile = Path(source) / "Dockerfile.lab" + if not dockerfile.is_file(): + raise PackError(f"image '{image}' not found and there is no Dockerfile.lab to build it from. " + "Run from a labagent checkout, or pass image=.") + r = subprocess.run([self.docker, "build", "-f", str(dockerfile), "-t", image, str(dockerfile.parent)]) + if r.returncode != 0: + raise PackError("docker build failed") + + def launch(self, cohort: str, learner: str, password: str, port: int, **kw) -> Dict[str, str]: + vol = volume_name(cohort, learner) + image = kw["image"] + self._run([self.docker, "volume", "create", vol]) + self._run([self.docker, "run", "--rm", "-u", "root", "-v", f"{vol}:/w", image, "chown", "-R", "1000:1000", "/w"]) + self._run(run_args(cohort, learner, password, port, **kw), env={"PASSWORD": password, **(kw.get("env") or {})}) + return {"container": container_name(cohort, learner), "url": learner_url(port), "volume": vol, "port": str(port)} + + def ready(self, url: str) -> bool: + try: + with urllib.request.urlopen(url + "login", timeout=5) as r: + return r.status == 200 + except Exception: + return False + + def alive(self, container: str) -> bool: + r = self._run([self.docker, "inspect", "-f", "{{.State.Running}}", container], check=False) + return r.returncode == 0 and r.stdout.strip() == "true" + + def find(self, cohort: str) -> List[str]: + r = self._run([self.docker, "ps", "-a", "--filter", "label=labagent=rag-ide", "--filter", f"label=cohort={slug(cohort)}", "--format", "{{.Names}}"], check=False) + return [n for n in r.stdout.split() if n] + + def remove(self, container: str) -> None: + self._run([self.docker, "rm", "-f", container], check=False) + + def delete_volume(self, name: str) -> None: + self._run([self.docker, "volume", "rm", name], check=False) + + def pause(self, seconds: float) -> None: + import time + time.sleep(seconds) diff --git a/labagent/packs/rag_ide/pack.py b/labagent/packs/rag_ide/pack.py index 8bf2d88..3048ec5 100644 --- a/labagent/packs/rag_ide/pack.py +++ b/labagent/packs/rag_ide/pack.py @@ -14,6 +14,7 @@ from ..base import Pack, PackContext, PackError, Param, Step, state_dir from . import ec2 as ec2mod +from . import local as localmod from .modal_gateway import ModalGateway, app_name, slug MAX_LEARNERS = 60 @@ -73,6 +74,16 @@ def __getattr__(self, name: str): return getattr(self._gw, name) +class _LazyLocal: + def __init__(self) -> None: + self._gw = None + + def __getattr__(self, name: str): + if self._gw is None: + self._gw = localmod.LocalDockerGateway() + return getattr(self._gw, name) + + class RagIde(Pack): id = "rag-ide" title = "Browser VS Code lab per learner (Modal)" @@ -80,7 +91,8 @@ class RagIde(Pack): aliases = ("ide", "web-ide", "rag-web-ide") default_name = "cohort" params = [ - Param("platform", "modal", "modal (one serverless sandbox per learner) | ec2 (one shared instance per cohort, much cheaper at scale)"), + Param("platform", "modal", "modal (one serverless sandbox per learner) | ec2 (one shared instance per cohort, much cheaper at scale) | local (Docker on this machine, for demos and tests)"), + Param("port", 8080, "local: first port; learner N gets port + N - 1 on 127.0.0.1"), Param("learners", "1", "Count (5) or comma list of names (ana,ben)"), Param("instance_type", "auto", "ec2: instance type, or auto"), Param("spot", False, "ec2: use a spot instance (cheaper, can be reclaimed)"), @@ -100,12 +112,12 @@ class RagIde(Pack): ] def services(self, profile): - return {"modal": _LazyModal(), "ec2": _LazyEc2()} + return {"modal": _LazyModal(), "ec2": _LazyEc2(), "local": _LazyLocal()} def _validate(self, ctx: PackContext) -> None: p = ctx.params - if p["platform"] not in ("modal", "ec2"): - raise PackError("platform must be modal or ec2") + if p["platform"] not in ("modal", "ec2", "local"): + raise PackError("platform must be modal, ec2 or local") if p["tls"] not in ("letsencrypt", "internal"): raise PackError("tls must be letsencrypt or internal") names = learner_names(p["learners"]) @@ -157,9 +169,12 @@ def _learners(self, ctx: PackContext): passwords[n] = pw ctx.state["learners"] = learners # persist progress so a failure part-way resumes ctx.mem["passwords"] = passwords + self._write_creds(ctx) return {"learners": learners} - def _credentials(self, ctx: PackContext): + def _write_creds(self, ctx: PackContext) -> Path: + """Write the 0600 credentials file. Called as soon as learners exist (so a later failure cannot lose their passwords) + and again as the last step. Passwords already on file are kept.""" path = creds_path(ctx.name) old: Dict[str, str] = {} if path.exists(): @@ -171,6 +186,12 @@ def _credentials(self, ctx: PackContext): w.writerow(["learner", "url", "password"]) for n, info in sorted(ctx.state["learners"].items()): w.writerow([n, info["url"], pw.get(n, "(lost: destroy and relaunch this learner)")]) + ctx.mem["passwords"] = pw + return path + + def _credentials(self, ctx: PackContext): + path = self._write_creds(ctx) + pw = ctx.mem.get("passwords", {}) learners = sorted(ctx.state["learners"].items()) sp = ctx.params["show_passwords"] show = sp == "yes" or (sp == "auto" and len(learners) <= 3) @@ -212,15 +233,17 @@ def _ec2_instance(self, ctx: PackContext): ip = gw.public_ip(iid) ctx.mem["passwords"] = pw host = ec2mod.sslip_host(ip) - return {"instance_id": iid, "host": host, - "learners": {n: {"url": ec2mod.learner_url(host, n)} for n in names}} + out = {"instance_id": iid, "host": host, "learners": {n: {"url": ec2mod.learner_url(host, n)} for n in names}} + ctx.state.update(out) + self._write_creds(ctx) # before the long wait: a failure from here on must not lose the passwords + return out def _ec2_wait(self, ctx: PackContext): gw = ctx.services["ec2"] host, names = ctx.state["host"], sorted(ctx.state["learners"]) ctx.log(" the box is building the lab image (about 8 to 12 minutes the first time) ...") for _ in range(WAIT_S // 15): - if all(gw.ready(host, n) for n in names): + if all(gw.ready(host, n, verify=ctx.params["tls"] == "letsencrypt") for n in names): ctx.log(" all learner IDEs answer over HTTPS") return {"ready": True} gw.pause(15) @@ -256,7 +279,83 @@ def sg(c): Step("network", "Delete the security group", sg), Step("credentials", "Remove the credentials file", self._forget)] + # ---- local Docker platform ------------------------------------------------------------------------------------------------ + def _local_image(self, ctx: PackContext): + image = ctx.params["image"] + image = localmod.IMAGE if image in ("auto", "pypi") else image # anything else is an existing image tag + ctx.services["local"].ensure_image(image, checkout_root()) + return {"image": image} + + def _local_preflight(self, ctx: PackContext): + self._validate(ctx) + where = ctx.services["local"].check_auth() + if not 1024 <= int(ctx.params["port"]) <= 65000 - len(ctx.mem["names"]): + raise PackError("port must be between 1024 and 65000") + ctx.log(f" {where} | learners: {len(ctx.mem['names'])} | first port: {ctx.params['port']}") + return {"docker": where} + + def _local_learners(self, ctx: PackContext): + gw, p = ctx.services["local"], ctx.params + names = ctx.mem.get("names") or learner_names(p["learners"]) + learners: Dict[str, Dict[str, str]] = dict(ctx.state.get("learners", {})) + passwords: Dict[str, str] = {} + env = {"GROQ_API_KEY": os.environ["GROQ_API_KEY"], "GROQ_MODEL": p["groq_model"], "LLM_PROVIDER": "groq"} if p["llm"] == "groq-shared" else {} + for i, n in enumerate(names): + if n in learners and gw.alive(learners[n]["container"]): + ctx.log(f" {n}: already running") + continue + gw.remove(localmod.container_name(ctx.name, n)) # a stopped container of the same name would block the new one + pw = secrets.token_urlsafe(12) + ctx.log(f" starting {n} ...") + learners[n] = gw.launch(ctx.name, n, pw, int(p["port"]) + i, image=ctx.state["image"], cpu=float(p["cpu"]), memory_mib=int(p["memory_mib"]), + hours=int(p["hours"]), idle_s=int(p["idle_minutes"]) * 60, env=env) + passwords[n] = pw + ctx.state["learners"] = learners + ctx.mem["passwords"] = passwords + self._write_creds(ctx) + return {"learners": learners} + + def _local_wait(self, ctx: PackContext): + gw = ctx.services["local"] + for _ in range(60): + if all(gw.ready(info["url"]) for info in ctx.state["learners"].values()): + ctx.log(" all learner IDEs answer") + return {"ready": True} + gw.pause(2) + raise PackError("learner IDEs did not answer in time. Check: docker logs ") + + def _local_steps(self) -> List[Step]: + return [ + Step("preflight", "Check Docker and settings", self._local_preflight), + Step("image", "Use or build the lab image", self._local_image), + Step("learners", "Start one container per learner on loopback", self._local_learners), + Step("wait", "Wait for every learner's IDE", self._local_wait), + Step("credentials", "Write the credentials file", self._credentials), + ] + + def _local_destroy(self) -> List[Step]: + def remove(c): + gw = c.services["local"] + names = {i["container"] for i in c.state.get("learners", {}).values()} | set(gw.find(c.name)) + for n in names: + gw.remove(n) + return {"learners": {}, "removed": len(names), "volumes": sorted({i["volume"] for i in c.state.get("learners", {}).values()} | set(c.state.get("volumes", [])))} + + def volumes(c): + if not c.params["keep_work"]: + for v in c.state.get("volumes", []): + c.services["local"].delete_volume(v) + return {"volumes": []} + return {} + + return [Step("preflight", "Check Docker", lambda c: {"docker": c.services["local"].check_auth()}), + Step("remove", "Remove every learner's container", remove), + Step("volumes", "Delete learner volumes (unless keep_work)", volumes), + Step("credentials", "Remove the credentials file", self._forget)] + def deploy_steps(self, ctx: PackContext) -> List[Step]: + if ctx.params["platform"] == "local": + return self._local_steps() if ctx.params["platform"] == "ec2": return self._ec2_steps() return [ @@ -286,6 +385,8 @@ def _forget(self, ctx: PackContext): return {"credentials_file": None} def destroy_steps(self, ctx: PackContext) -> List[Step]: + if ctx.params["platform"] == "local": + return self._local_destroy() if ctx.params["platform"] == "ec2": return self._ec2_destroy() # remember volume names before the learner records are cleared diff --git a/tests/test_rag_ide_ec2.py b/tests/test_rag_ide_ec2.py index 93ee798..dd64480 100644 --- a/tests/test_rag_ide_ec2.py +++ b/tests/test_rag_ide_ec2.py @@ -34,7 +34,8 @@ def alive(self, iid): return self.instances.get(iid, False) def find(self, cohort): return [i for i, up in self.instances.items() if up] def pause(self, s): self.pauses.append(s) - def ready(self, host, learner): + def ready(self, host, learner, verify=True): + self.verify_seen = getattr(self, 'verify_seen', []) + [verify] self.polls += 1 return self.polls > self.ready_after @@ -216,3 +217,26 @@ def test_dockerfile_hands_the_workspace_to_the_user_before_switching_to_it(): user = next(i for i, l in enumerate(lines) if l.strip() == "USER coder") workdir = next(i for i, l in enumerate(lines) if l.startswith("WORKDIR /home/coder/lab")) assert chown < user < workdir + + +def test_self_signed_tls_skips_certificate_verification_only_when_internal(): + f1 = FakeEc2() + launch(f1, tls="internal") + assert set(f1.verify_seen) == {False} + f2 = FakeEc2() + launch(f2, tls="letsencrypt") + assert set(f2.verify_seen) == {True} + + +def test_passwords_survive_a_failure_after_the_instance_starts(monkeypatch): + """Regression: the credentials file used to be written only at the very end, so a failed wait lost every password.""" + from labagent.packs.rag_ide import pack as pk + monkeypatch.setattr(pk, "WAIT_S", 30) + fake = FakeEc2(ready_after=10_000) + env, _ = launch(fake) + assert env.exit_code != 0 + rows = list(csv.DictReader(creds_path("c1").open())) + assert len(rows) == 2 and all(r["password"] in fake.runs[0]["ud"] for r in rows) + fake.ready_after = 0 + assert launch(fake)[0].exit_code == 0 and len(fake.runs) == 1 # resume reuses the instance + assert list(csv.DictReader(creds_path("c1").open())) == rows # and keeps the same passwords diff --git a/tests/test_rag_ide_local.py b/tests/test_rag_ide_local.py new file mode 100644 index 0000000..8a5a44b --- /dev/null +++ b/tests/test_rag_ide_local.py @@ -0,0 +1,135 @@ +import csv +import shutil +import subprocess +import urllib.request + +import pytest + +from labagent.packs.base import PackError, load_state, run_pack +from labagent.packs.rag_ide import local as lc +from labagent.packs.rag_ide.pack import RagIde, creds_path + + +class FakeLocal: + def __init__(self, ready_after=0, docker_up=True): + self.docker_up, self.ready_after, self.polls = docker_up, ready_after, 0 + self.running, self.launches, self.removed, self.volumes_deleted, self.images = {}, [], [], [], [] + self.pauses = [] + + def check_auth(self): + if not self.docker_up: + raise PackError("docker is installed but its daemon is not running. Start Docker Desktop and retry.") + return "local docker 29.0" + + def ensure_image(self, image, source): self.images.append((image, source)) + + def launch(self, cohort, learner, password, port, **kw): + c = lc.container_name(cohort, learner) + self.launches.append({"learner": learner, "password": password, "port": port, **kw}) + self.running[c] = True + return {"container": c, "url": lc.learner_url(port), "volume": lc.volume_name(cohort, learner), "port": str(port)} + + def ready(self, url): + self.polls += 1 + return self.polls > self.ready_after + + def alive(self, c): return self.running.get(c, False) + def find(self, cohort): return [c for c, up in self.running.items() if up] + def remove(self, c): self.removed.append(c); self.running[c] = False + def delete_volume(self, v): self.volumes_deleted.append(v) + def pause(self, s): self.pauses.append(s) + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("LABAGENT_HOME", str(tmp_path)) + + +def launch(fake, **params): + lines = [] + env = run_pack(RagIde(), "launch", "c1", {"platform": "local", "learners": "2", **params}, services={"local": fake}, log=lines.append) + return env, "\n".join(lines) + + +def test_launch_starts_each_learner_on_its_own_loopback_port_and_writes_credentials(): + fake = FakeLocal() + env, out = launch(fake, port=9100, hours=2, idle_minutes=5) + assert env.exit_code == 0 and [l["port"] for l in fake.launches] == [9100, 9101] + assert fake.launches[0]["hours"] == 2 and fake.launches[0]["idle_s"] == 300 and fake.images[0][0] == lc.IMAGE + rows = list(csv.DictReader(creds_path("c1").open())) + assert [r["url"] for r in rows] == ["http://127.0.0.1:9100/", "http://127.0.0.1:9101/"] + assert all(r["url"] in out and r["password"] in out for r in rows) # 2 learners: passwords shown + assert creds_path("c1").stat().st_mode & 0o777 == 0o600 + assert not any(r["password"] in str(load_state("rag-ide", "c1")) for r in rows) + + +def test_docker_not_running_fails_with_the_fix_and_starts_nothing(): + fake = FakeLocal(docker_up=False) + env, _ = launch(fake) + assert env.exit_code != 0 and "Start Docker Desktop" in str(env.__dict__) and fake.launches == [] + + +def test_rerun_keeps_running_learners_and_restarts_dead_ones(): + fake = FakeLocal() + launch(fake) + first = {r["learner"]: r["password"] for r in csv.DictReader(creds_path("c1").open())} + fake.running[lc.container_name("c1", "learner-01")] = False + launch(fake) + after = {r["learner"]: r["password"] for r in csv.DictReader(creds_path("c1").open())} + assert len(fake.launches) == 3 and fake.launches[2]["learner"] == "learner-01" + assert after["learner-02"] == first["learner-02"] and after["learner-01"] != first["learner-01"] + + +def test_waits_for_ides_and_fails_with_where_to_look(): + fake = FakeLocal(ready_after=3) + assert launch(fake)[0].exit_code == 0 and fake.pauses + env, _ = launch(FakeLocal(ready_after=10_000)) + assert env.exit_code != 0 and "docker logs" in str(env.__dict__) + + +def test_bad_port_and_platform_are_rejected(): + assert launch(FakeLocal(), port=80)[0].exit_code != 0 + assert run_pack(RagIde(), "launch", "c1", {"platform": "docker"}, services={"local": FakeLocal()}).exit_code != 0 + + +def test_destroy_removes_containers_orphans_volumes_and_credentials(): + fake = FakeLocal() + launch(fake) + fake.running["labagent-lab-c1-stray"] = True + env = run_pack(RagIde(), "destroy", "c1", {"platform": "local"}, services={"local": fake}) + assert env.exit_code == 0 and not any(fake.running.values()) and "labagent-lab-c1-stray" in fake.removed + assert sorted(fake.volumes_deleted) == [lc.volume_name("c1", "learner-01"), lc.volume_name("c1", "learner-02")] and not creds_path("c1").exists() + f2 = FakeLocal() + launch(f2) + run_pack(RagIde(), "destroy", "c1", {"platform": "local", "keep_work": True}, services={"local": f2}) + assert f2.volumes_deleted == [] + + +def test_docker_run_arguments_are_locked_down_and_keep_secrets_off_the_command_line(): + args = lc.run_args("c1", "ana", "pw_secret_12345", 9100, image="img:1", cpu=1.5, memory_mib=2048, hours=3, idle_s=600, env={"GROQ_API_KEY": "gsk_x"}) + joined = " ".join(args) + assert "127.0.0.1:9100:8080" in joined and "--cap-drop ALL" in joined and "no-new-privileges" in joined + assert "--memory 2048m" in joined and "--cpus 1.5" in joined and "--pids-limit" in joined + assert "pw_secret_12345" not in joined and "gsk_x" not in joined and "-e PASSWORD" in joined and "-e GROQ_API_KEY" in joined + assert "timeout 10800" in joined and "--idle-timeout-seconds 600" in joined and "--auth password" in joined + assert "--privileged" not in joined and "docker.sock" not in joined + + +def test_missing_image_without_a_dockerfile_gives_a_clear_error(tmp_path, monkeypatch): + gw = lc.LocalDockerGateway() + monkeypatch.setattr(gw, "_run", lambda *a, **k: subprocess.CompletedProcess(a, 1, "", "no such image")) + with pytest.raises(PackError, match="Dockerfile.lab"): + gw.ensure_image("nope:1", str(tmp_path)) + + +@pytest.mark.skipif(not shutil.which("docker") or subprocess.run(["docker", "image", "inspect", lc.IMAGE], capture_output=True).returncode != 0, + reason="needs a running Docker daemon and the labagent-rag-lab:local image") +def test_real_docker_end_to_end(): + env = run_pack(RagIde(), "launch", "itest", {"platform": "local", "learners": "1", "port": 18190, "hours": 1, "idle_minutes": 5}) + try: + assert env.exit_code == 0 + row = next(csv.DictReader(creds_path("itest").open())) + assert urllib.request.urlopen(row["url"] + "login", timeout=10).status == 200 + finally: + run_pack(RagIde(), "destroy", "itest", {"platform": "local"}) + assert not creds_path("itest").exists()