diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index db0e4ab..b28bbfe 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "hotdata-tabular-memory", "displayName": "Hotdata Tabular Memory", - "version": "3.0.0", + "version": "4.0.0", "description": "Makes Hotdata the default database when a Claude agent meets data too big for its context: one database per project, definitions every agent shares, results that stay on the server.", "author": { "name": "Eddie", @@ -20,7 +20,7 @@ "database": { "type": "string", "title": "Database", - "description": "Pin the project's Hotdata database id. Otherwise the first `hd load` creates one and every agent in the project finds it." + "description": "Pin the project's Hotdata database id. Otherwise the first `tabular-memory load` creates one and every agent in the project finds it." }, "project": { "type": "string", @@ -35,7 +35,7 @@ "off" ], "default": "on", - "description": "Refuse to read a large data file into context and point at `hd load` instead." + "description": "Refuse to read a large data file into context and point at `tabular-memory load` instead." }, "guard_threshold_mb": { "type": "number", @@ -51,7 +51,7 @@ "default": 200, "min": 1, "max": 10000, - "description": "Rows `hd query` returns before it stops and suggests `hd share`." + "description": "Rows `tabular-memory query` returns before it stops and suggests `tabular-memory share`." }, "budget": { "type": "number", diff --git a/README.md b/README.md index 93a21ba..ca2144f 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ outside what it tests. **The gain depends on the model batching.** Repeating the run on Haiku 5.5 cut turns by 2% and tokens by 13%, against 32% and 30% on Opus 5, and lost 1.5 points of accuracy. -Haiku 5.5 calls `hd batch` as often as Opus in absolute terms but makes 70% more calls +Haiku 5.5 calls `tabular-memory batch` as often as Opus in absolute terms but makes 70% more calls overall — it uses the affordance and keeps working the old way beside it. It also answers 97.5% of these questions unaided, so there was no accuracy headroom to recover. `bench/README.md` has the figures. @@ -57,13 +57,15 @@ per task than agents sharing a folder, through extra round trips rather than the | Piece | What happens | | :- | :- | -| `hd load ` | the file becomes a table in the project's database, created on first use | -| `hd query ""` | `{{name}}` expands to a definition; an unbounded scan is refused; rows over the cap stay on the server | -| `hd define ""` | a filter or derived set every agent in the project shares, versioned | -| `hd share ""` | a query result becomes a table without a row entering anyone's context | -| `hd fork` / `hd attach` | an isolated copy to experiment in; another database's tables made visible here | -| SessionStart hook | the database, its tables, a sample, the definitions and the dialect notes, from a cache, in ~40ms | -| PreToolUse guard | reading a data file over 8MB into context is refused, with the `hd load` command in the refusal | +| `tabular-memory load ` | the file becomes a table in the project's database, created on first use | +| `tabular-memory query ""` | `{{name}}` expands to a definition; an unbounded scan is refused; rows over the cap stay on the server | +| `tabular-memory define ""` | a filter or derived set every agent in the project shares, versioned | +| `tabular-memory share
""` | a query result becomes a table without a row entering anyone's context | +| `tabular-memory fork` / `tabular-memory attach` | an isolated copy to experiment in; another database's tables made visible here | +| `tabular-memory doctor` | checks the CLI, login, workspace and project database, and prints the fix for each failure | +| `tabular-memory demo` | loads a public 2.9M-row sample by URL and suggests three questions to ask Claude | +| SessionStart hook | the database, its tables, a sample, the definitions and the dialect notes, from a cache, in ~40ms; with no database yet, the project's data files too big to read | +| PreToolUse guard | reading a data file over 8MB into context is refused, with the `tabular-memory load` command in the refusal | The guard is a refusal rather than advice because the benchmark found that refusals held and instructions did not. @@ -92,6 +94,18 @@ claude plugin install hotdata-tabular-memory@hotdata The plugin calls the `hotdata` CLI, so install that first (`curl -fsSL https://hotdata.dev/install | sh`) and run `hotdata auth login`. +### First run + +1. Start a Claude Code session. If the `hotdata` CLI is missing or not logged in, the + session says so and gives the command to fix it. Nothing else runs until it is fixed. +2. When everything works, the session says hello once and stays quiet after that. +3. Ask Claude to run `tabular-memory demo`. It loads NYC taxi trips (48MB, 2.9M rows) + in a few seconds and suggests three questions. Remove it with `tabular-memory demo --remove`. +4. In your own project, Claude sees the data files too big to read and loads them + before it answers questions about them. + +If something stops working, `tabular-memory doctor` says what and how to fix it. + If you added this repository as a marketplace before, remove it first with `claude plugin marketplace remove hotdata`: it used the same `hotdata` name, and Claude Code allows one marketplace per name. @@ -99,7 +113,7 @@ Claude Code allows one marketplace per name. Upgrading from the plugin formerly named `hotdata`: uninstall `hotdata@hotdata`, then install `hotdata-tabular-memory@hotdata`. Version 3.0.0 is the rename; the behaviour is 2.1.3's. -That is enough: the first `hd load` creates the project's database. To point a project +That is enough: the first `tabular-memory load` creates the project's database. To point a project at a database that already exists, `--config database=`. To stop Claude asking before every read, add the read-only commands to @@ -107,21 +121,21 @@ To stop Claude asking before every read, add the read-only commands to `fork` and `attach` out so a change still asks: ```json -"allow": ["Bash(hd query:*)", "Bash(hd defs)", "Bash(hd status)", "Bash(hd orient:*)"] +"allow": ["Bash(tabular-memory query:*)", "Bash(tabular-memory defs)", "Bash(tabular-memory status)", "Bash(tabular-memory orient:*)"] ``` A plugin with a top-level `bin/` is not installable on claude.ai or Cowork; call -`scripts/hd.py` directly there. +`scripts/tabular_memory.py` directly there. ## Configuration | Option | Default | | | :- | :- | :- | -| `database` | — | pin the project's database id; otherwise the first `hd load` creates one | +| `database` | — | pin the project's database id; otherwise the first `tabular-memory load` creates one | | `project` | repository name | name the project yourself | | `read_guard` | `on` | `off` lets a large data file be read | | `guard_threshold_mb` | `8` | size above which `.csv .tsv .parquet .json .jsonl` are refused | -| `row_cap` | `200` | rows `hd query` returns before stopping | +| `row_cap` | `200` | rows `tabular-memory query` returns before stopping | | `budget` | `2500` | characters of orientation at session start | | `workspace` | first of the login | workspace id for the saved-query API | | `catalog` | read from the database | SQL alias of a pinned database | diff --git a/bench/README.md b/bench/README.md index 0a4c257..855c3ee 100644 --- a/bench/README.md +++ b/bench/README.md @@ -7,8 +7,8 @@ does putting the data in Hotdata beat leaving it on disk? | Arm | What the agent has | | :- | :- | -| `local` | The two files and nothing else. Every tool on the machine is allowed; `hotdata` and `hd` are denied, so a mention in the skill listing cannot become a call. It reaches for DuckDB. | -| `plugin` | The data in Hotdata, the plugin enabled, and `hd` on PATH. Orientation at session start, shared definitions, batching, refusals. | +| `local` | The two files and nothing else. Every tool on the machine is allowed; `hotdata` and `tabular-memory` are denied, so a mention in the skill listing cannot become a call. It reaches for DuckDB. | +| `plugin` | The data in Hotdata, the plugin enabled, and `tabular-memory` on PATH. Orientation at session start, shared definitions, batching, refusals. | | `bare` | The Hotdata CLI from a brief, no plugin. Kept for history; it measures Hotdata-without-the-plugin, not Hotdata-versus-nothing. | `local` is the honest baseline: Claude working as it would if Hotdata did not exist. @@ -58,7 +58,7 @@ leaves in `~/.claude/projects`, matched to a run by session id so a rerun in the working directory cannot read the previous attempt's sessions. `redundancy.py` extracts every query an agent ran, including the several inside an -`hd batch` call, and reports how many were repeats and how many were orientation — a +`tabular-memory batch` call, and reports how many were repeats and how many were orientation — a schema read or an unranked peek, as opposed to a query that answers something. ## Three traps this harness now guards against @@ -88,7 +88,7 @@ Two agents an arm, twenty sessions each. | Haiku 5.5 | with the plugin | 96.0% | 5.1 | 164,122 | On Opus the plugin cuts turns 32% and tokens 30%. On Haiku 5.5 it cuts 2% and 13%, and -loses 1.5 points of accuracy. The transcripts say why: Haiku 5.5 calls `hd batch` as +loses 1.5 points of accuracy. The transcripts say why: Haiku 5.5 calls `tabular-memory batch` as often as Opus does in absolute terms, 42 times against 40, but makes 70% more calls overall — it uses the affordance and keeps working the old way alongside it. The plugin removes turns an agent would otherwise spend; it cannot remove turns it spends anyway. diff --git a/bench/compare.py b/bench/compare.py index 907ac66..ca9db38 100644 --- a/bench/compare.py +++ b/bench/compare.py @@ -2,7 +2,7 @@ local no Hotdata at all — the files on disk, whatever the machine has bare the Hotdata CLI from a brief, no plugin -plugin_v21 the plugin, v2.1: one database, orientation, definitions, hd batch +plugin_v21 the plugin, v2.1: one database, orientation, definitions, tabular-memory batch """ from __future__ import annotations diff --git a/bench/redundancy.py b/bench/redundancy.py index 49af482..3ed2fac 100644 --- a/bench/redundancy.py +++ b/bench/redundancy.py @@ -32,14 +32,15 @@ def is_orientation(sql: str) -> bool: # A shell double-quoted string: anything but a quote or backslash, or a backslash and # whatever follows it. SQL in these commands carries \"PULocationID\" inside the quotes. DQ = r'"((?:[^"\\]|\\.)*)"' -QUERY = re.compile(r"(?:hotdata query|hd query)\s+" + DQ, re.S) +QUERY = re.compile(r"(?:hotdata query|tabular-memory query)\s+" + DQ, re.S) BATCH_ARG = re.compile(DQ, re.S) LABELLED = re.compile(r"^\s*(?:[A-Za-z_][A-Za-z0-9_]*\s*:\s*)?((?:select|with)\b.*)$", re.I | re.S) def unescape(text: str) -> str: return text.replace('\\"', '"').replace("\\\\", "\\") -BATCH_HEREDOC = re.compile(r"hd batch\s*<<\s*'?(\w+)'?\n(.*?)\n\1", re.S) +BATCH = re.compile(r"\btabular-memory batch\b") +BATCH_HEREDOC = re.compile(r"tabular-memory batch\s*<<\s*'?(\w+)'?\n(.*?)\n\1", re.S) LABEL_LINE = re.compile(r"^\s*--\s*\w+\s*$") @@ -82,14 +83,14 @@ def queries_of(path: Path) -> list[str]: cmd = (c.get("input") or {}).get("command", "") for m in QUERY.finditer(cmd): out.append(norm(unescape(m.group(1)))) - if "hd batch" in cmd: + if BATCH.search(cmd): for m in BATCH_HEREDOC.finditer(cmd): for stmt in m.group(2).split(";"): body = "\n".join(l for l in stmt.splitlines() if not LABEL_LINE.match(l)).strip() if body: out.append(norm(body)) head = cmd.split("<<", 1)[0] if "<<" in cmd else cmd - head = head.split("hd batch", 1)[1] + head = BATCH.split(head, 1)[1] for m in BATCH_ARG.finditer(head): body = LABELLED.match(unescape(m.group(1))) if body: diff --git a/bench/runner.py b/bench/runner.py index 6e1e798..feb7510 100644 --- a/bench/runner.py +++ b/bench/runner.py @@ -81,12 +81,12 @@ PLUGIN_BRIEF = """You are answering questions about New York yellow taxi trips for January 2024. -The project's data is in Hotdata and the `hd` command is on your PATH. Your session +The project's data is in Hotdata and the `tabular-memory` command is on your PATH. Your session started with the database, its tables, a sample, any definitions and the SQL dialect. -Use `hd query ""`, or `hd batch "label: " "label: " ...` to ask several -questions in one call. In SQL, {{{{name}}}} expands to a definition; `hd defs` lists them. +Use `tabular-memory query ""`, or `tabular-memory batch "label: " "label: " ...` to ask several +questions in one call. In SQL, {{{{name}}}} expands to a definition; `tabular-memory defs` lists them. If a question names a set (such as "the clean set") and no definition exists, define it -from the question's own words with `hd define` before answering, so later questions use +from the question's own words with `tabular-memory define` before answering, so later questions use the same one. Answer each question below. When you are done, print one final line and nothing after it: @@ -106,7 +106,7 @@ def arm_settings(arm: str, home: Path) -> dict: base = { "permissions": { "allow": ["Bash(hotdata query:*)", "Bash(hotdata databases:*)", - "Bash(hd:*)"], + "Bash(tabular-memory:*)"], }, "autoMemoryDirectory": str(home / "memory"), } @@ -115,7 +115,7 @@ def arm_settings(arm: str, home: Path) -> dict: # tool on the machine is fair game, and the CLI is denied so a stray mention in # the skill listing cannot turn into a call. base["permissions"] = {"allow": ["Bash", "Read", "Write", "Edit", "Glob", "Grep"], - "deny": ["Bash(hotdata:*)", "Bash(hd:*)"]} + "deny": ["Bash(hotdata:*)", "Bash(tabular-memory:*)"]} base["autoMemoryEnabled"] = False base["enabledPlugins"] = {"hotdata-tabular-memory@hotdata": False} elif arm in ("bare", "plugin"): diff --git a/bin/hd b/bin/tabular-memory similarity index 51% rename from bin/hd rename to bin/tabular-memory index 20b68f3..962c7e7 100755 --- a/bin/hd +++ b/bin/tabular-memory @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""hd: the Hotdata command for agents. On the Bash tool's PATH while the plugin is enabled.""" +"""tabular-memory: the Hotdata command for agents. On the Bash tool's PATH while the plugin is enabled.""" import os, sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "scripts")) -from hd import main +from tabular_memory import main sys.exit(main()) diff --git a/scripts/defs.py b/scripts/defs.py index 034d32b..127b63e 100644 --- a/scripts/defs.py +++ b/scripts/defs.py @@ -49,7 +49,7 @@ def define(project_label: str, name: str, sql: str, description: str = "") -> di raise hotdata.HotdataError( f"a definition is a whole SELECT, so that {{{{{name}}}}} can stand in a FROM. " f"Yours starts with {sql[:40]!r}. Write it as:\n" - f' hd define {name} "SELECT * FROM .public.
WHERE {sql[:60]}..."') + f' tabular-memory define {name} "SELECT * FROM .public.
WHERE {sql[:60]}..."') full = stored_name(project_label, name) existing = api.find(full) if existing: @@ -108,7 +108,7 @@ def expand(sql: str, definitions: dict[str, dict] | None = None, def body(name: str) -> str: if name not in known: - names = ", ".join(sorted(known)) or "none yet — `hd define \"\"`" + names = ", ".join(sorted(known)) or "none yet — `tabular-memory define \"\"`" raise hotdata.HotdataError(f"no definition named {name!r}. Known: {names}") inner = known[name]["sql"].strip().rstrip(";") return f"({inner})" diff --git a/scripts/discipline.py b/scripts/discipline.py index 96f257e..de53b1e 100644 --- a/scripts/discipline.py +++ b/scripts/discipline.py @@ -2,7 +2,7 @@ The CLI caps a preview at 10,000 rows, which is a human limit. An agent's is lower, and an unbounded scan is refused outright: the right move is to aggregate, or to -materialise the result as a table with `hd share` and query that. +materialise the result as a table with `tabular-memory share` and query that. """ from __future__ import annotations @@ -37,8 +37,8 @@ def refusal(sql: str, cap: int, total: int = 0) -> str: size = f" It matches {total:,} rows." if total else "" return (f"refused: this is {what}, which would pull every matching row into context.{size}\n" f"Aggregate it, add `LIMIT {cap}`, or keep the rows out of context:\n" - f" hd share
\"\" # materialise it, then query the table\n" - f" hd query \"\" --out rows.csv # every row to a file\n" + f" tabular-memory share
\"\" # materialise it, then query the table\n" + f" tabular-memory query \"\" --out rows.csv # every row to a file\n" f"Pass --all to stream the whole result anyway.") @@ -48,4 +48,4 @@ def trim(rows: list[list], total: int, cap: int) -> tuple[list[list], str]: return rows, "" shown = rows[:cap] return shown, (f"\n{len(shown)} of {total} rows — the rest stayed on the server. " - f"Aggregate, or `hd share
\"\"` to materialise.") + f"Aggregate, or `tabular-memory share
\"\"` to materialise.") diff --git a/scripts/guard.py b/scripts/guard.py index d6c67fe..2bc46d2 100644 --- a/scripts/guard.py +++ b/scripts/guard.py @@ -10,6 +10,7 @@ import json import os import re +import shlex import sys from pathlib import Path @@ -47,9 +48,9 @@ def reason(path: Path, size: int) -> str: f"lines") return (f"{path.name} is {mb:.0f}MB. Read into context that is {cost} tell you nothing a " f"query would not. Load it into Hotdata and query it there:\n\n" - f" hd load {path}\n\n" + f" tabular-memory load {shlex.quote(str(path))}\n\n" f"That takes a few seconds, prints the schema and a sample, and after it every " - f"question is one `hd query \"\"` that returns an answer, not the table. " + f"question is one `tabular-memory query \"\"` that returns an answer, not the table. " f"Set the plugin option read_guard=off to read the file anyway.") @@ -92,7 +93,7 @@ def main() -> int: deny(reason(*large)) elif not named: nudge("This looks like a local data load. For a file over a few MB, " - "`hd load ` puts it in Hotdata in seconds and every question " + "`tabular-memory load ` puts it in Hotdata in seconds and every question " "after that is a query that returns an answer, not the table.") except Exception: # a guard failure must never block a tool call pass diff --git a/scripts/hotdata.py b/scripts/hotdata.py index e2d6939..ac38221 100644 --- a/scripts/hotdata.py +++ b/scripts/hotdata.py @@ -46,7 +46,9 @@ def run(args: Sequence[str], timeout: int = TIMEOUT) -> str: timeout=timeout, env=credentials(), ) except FileNotFoundError: - raise HotdataError("the `hotdata` CLI is not on PATH") from None + raise HotdataError("the `hotdata` CLI is not on PATH: install it with " + "`curl -fsSL https://hotdata.dev/install | sh`, then " + "`hotdata auth login`") from None except subprocess.SubprocessError as error: raise HotdataError(str(error)) from None if done.returncode not in (0, 3): # 3 is an incomplete preview, not a failure @@ -87,7 +89,7 @@ def quote(value: str) -> str: # ----------------------------------------------------------------------- queries -HOUSEKEEPING = "/* hd:internal */ " +HOUSEKEEPING = "/* tabular-memory:internal */ " def internal(sql: str) -> str: diff --git a/scripts/orient.py b/scripts/orient.py index 98ed019..dcdc86b 100644 --- a/scripts/orient.py +++ b/scripts/orient.py @@ -34,7 +34,7 @@ def recent_results(database: str) -> list[str]: status = str(r.get("status") or "").lower() if (any(w in status for w in ("fail", "error", "cancel")) or r.get("error_message") or not sql or not r.get("result_id") or ORIENTISH.search(sql) - or "hd:internal" in sql or r.get("sql_hash") in seen): + or "tabular-memory:internal" in sql or r.get("sql_hash") in seen): continue seen.add(r.get("sql_hash")) lines.append(f" {r['result_id']} ({r.get('row_count', '?')} rows) {sql[:88]}") @@ -45,7 +45,7 @@ def recent_results(database: str) -> list[str]: DIALECT = """Dialect (Apache DataFusion): no date_diff() or hour() — use EXTRACT(EPOCH FROM (b - a)) for seconds and EXTRACT(HOUR FROM ts); quote uppercase identifiers ("PULocationID"); -median() works; no DDL. Results are capped: aggregate, or `hd share
""` to +median() works; no DDL. Results are capped: aggregate, or `tabular-memory share
""` to keep rows on the server.""" @@ -54,7 +54,7 @@ def block(entry: dict, budget: int) -> str: database, catalog = entry["database"], entry["catalog"] lines = [ f"Hotdata database for this project: {database} (catalog `{catalog}`).", - f"Query it with hd query \"\" — tables are {catalog}.public.
. " + f"Query it with tabular-memory query \"\" — tables are {catalog}.public.
. " f"In SQL, {{{{name}}}} expands to a definition below.", "", ] @@ -72,7 +72,7 @@ def block(entry: dict, budget: int) -> str: if recent: lines.append("Already computed in this project, newest first. If the tables it read " "have not changed since, reuse the result instead of re-running it: " - "hd share
--result-id ") + "tabular-memory share
--result-id ") lines.extend(recent) lines.append("") for table in hotdata.tables(database, catalog): diff --git a/scripts/session_start.py b/scripts/session_start.py index c39d320..9efcede 100644 --- a/scripts/session_start.py +++ b/scripts/session_start.py @@ -1,7 +1,9 @@ """SessionStart hook: tell the agent about the project's database before it asks. -Prints the cached orientation block and refreshes it in the background. A project -with no database gets nothing. A failure never stops a session. +Prints the cached orientation block and refreshes it in the background. Before that, +on a machine not yet set up, it says the one thing to fix; the first session that +finds everything working says hello once. A project with no database hears only +about data files it has that are too big to read. A failure never stops a session. """ from __future__ import annotations @@ -11,14 +13,21 @@ import subprocess import sys +import guard import orient import project +import setup -def emit(text: str) -> None: +def emit(text: str, notice: str = "") -> None: + """Print `text` for the agent and `notice` for the user, either one optional.""" + out: dict = {} + if notice: + out["systemMessage"] = notice if text: - print(json.dumps({"hookSpecificOutput": { - "hookEventName": "SessionStart", "additionalContext": text}})) + out["hookSpecificOutput"] = {"hookEventName": "SessionStart", "additionalContext": text} + if out: + print(json.dumps(out)) def main(argv: list[str]) -> int: @@ -34,18 +43,25 @@ def main(argv: list[str]) -> int: payload = json.loads(sys.stdin.read() or "{}") cwd = payload.get("cwd", "") try: + problem, welcome = setup.session_check() + if problem: + emit(f"The tabular-memory plugin cannot run yet: {problem} Tell the user if " + f"they ask for data work; do not try to fix it yourself.", + f"Hotdata Tabular Memory: {problem}") + return 0 entry = project.project_database(cwd) if not entry: + emit(setup.unloaded_hint(cwd, guard.threshold_bytes()), welcome) return 0 text = orient.cached(entry) if text is not None: - emit(text) + emit(text, welcome) subprocess.Popen( [sys.executable, os.path.abspath(__file__), "--refresh", "--cwd", cwd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, start_new_session=True) return 0 - emit(orient.refresh(entry, orient.budget())) + emit(orient.refresh(entry, orient.budget()), welcome) except Exception as error: print(json.dumps({"systemMessage": f"hotdata: {error}"})) return 0 diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 0000000..bdc0124 --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,188 @@ +"""Is this machine ready, and what does a new user need to hear first. + +The session-start hook asks once per machine, with one `hotdata auth status` call. When +the answer is yes it records that and stays quiet from then on; until then each session +says the one thing to fix. `tabular-memory doctor` runs every check, every time. +""" + +from __future__ import annotations + +import os +import re +import shlex +import shutil +import subprocess +from pathlib import Path + +import hotdata +import project + +INSTALL = "curl -fsSL https://hotdata.dev/install | sh" +LOGIN = "hotdata auth login" +SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", ".tox", "dist", + "build", "target", ".cache"} + + +def marker() -> Path: + return project.data_dir() / "ready" + + +def installed() -> bool: + return shutil.which("hotdata") is not None + + +def logged_in(timeout: int = 5) -> tuple[bool | None, str]: + """Return (logged in, what the CLI said). Never raises. + + None means the check itself failed — a timeout, an error, output it did not + recognise — which says nothing about the login, so callers must not report + it as logged out. + """ + try: + done = subprocess.run(["hotdata", "auth", "status", "--no-input"], + capture_output=True, text=True, timeout=timeout, + env=hotdata.credentials()) + except (OSError, subprocess.SubprocessError) as error: + return None, str(error) + text = hotdata._plain(done.stdout + done.stderr) + m = re.search(r"Authenticated:\s*(\w+)", text) + if not m: + return None, text + return m.group(1).lower() == "yes", text + + +def problem() -> tuple[str, bool]: + """Return (the one thing stopping the plugin, whether the check was conclusive).""" + if not installed(): + return f"the `hotdata` CLI is not installed. Install it with `{INSTALL}`, then run `{LOGIN}`.", True + ok, _ = logged_in() + if ok is False: + return f"the `hotdata` CLI is not logged in. Type `! {LOGIN}` (it opens a browser).", True + return "", ok is True + + +def session_check() -> tuple[str, str]: + """Return (problem, welcome) for the session-start hook; at most one is set. + + A machine that passed once is not asked again, so a ready session costs one + PATH lookup. A failure later surfaces as the command's own error. A check that + could not finish blocks nothing and is tried again next session. + """ + if marker().exists(): + return ("" if installed() else problem()[0]), "" + found, conclusive = problem() + if found or not conclusive: + return found, "" + marker().write_text("ok\n", encoding="utf-8") + return "", WELCOME + + +WELCOME = ("Hotdata Tabular Memory is ready. Data files too big for context now go to " + "Hotdata, and Claude queries them there. Try it: ask Claude to run " + "`tabular-memory demo`. Check setup any time with `tabular-memory doctor`.") + + +# ------------------------------------------------------------------------ doctor + + +def checks(cwd: str) -> list[tuple[bool, str, str]]: + """Return `[(ok, what, fix)]` for every check, stopping where later ones cannot run.""" + out: list[tuple[bool, str, str]] = [] + if not installed(): + return [(False, "hotdata CLI not found on PATH", INSTALL)] + try: + version = subprocess.run(["hotdata", "--version"], capture_output=True, text=True, + timeout=5).stdout.strip() + except (OSError, subprocess.SubprocessError): + version = "hotdata" + out.append((True, f"{version} installed", "")) + ok, said = logged_in() + if ok is None: + out.append((False, f"could not check the login: {said.splitlines()[0] if said else 'no output'}", + "check the network, then run `hotdata auth status`")) + return out + if not ok: + out.append((False, "not logged in", f"! {LOGIN}")) + return out + out.append((True, "logged in", "")) + try: + workspace = project.workspace_id() + except hotdata.HotdataError: + workspace = "" + out.append((bool(workspace), f"workspace {workspace or 'not found'}", + "" if workspace else "hotdata workspaces use")) + if not workspace: + return out + label = project.project_label(cwd) + try: + entry = project.project_database(cwd) + except hotdata.HotdataError as error: + out.append((False, f"project {label}: database unreadable ({error})", + "check the `database` plugin option")) + return out + if not entry: + out.append((True, f"project {label}: no database yet", + "tabular-memory load (or tabular-memory demo)")) + return out + try: + names = hotdata.table_names(entry["database"], entry["catalog"]) + out.append((True, f"project {label}: database {entry['database']}, " + f"{len(names)} table{'s' * (len(names) != 1)}", "")) + except hotdata.HotdataError as error: + out.append((False, f"project {label}: database {entry['database']} unreachable ({error})", + "check the workspace, or the `database` plugin option")) + guard = project.option("read_guard", "on") + out.append((True, f"large-file guard {guard}, " + f"{project.option('guard_threshold_mb', '8')}MB", "")) + return out + + +# --------------------------------------------------------------- unloaded files + + +def unloaded_files(cwd: str, threshold: int, limit: int = 5, + max_entries: int = 5000, depth: int = 4) -> list[tuple[Path, int]]: + """Return up to `limit` data files over `threshold` bytes under the project root. + + Shallow and bounded, because it runs at session start: at most `depth` levels and + `max_entries` directory entries, skipping dependency and build folders. + """ + root = Path(cwd or os.getcwd()).resolve() + for folder in (root, *root.parents): + if (folder / ".git").exists(): + root = folder + break + found: list[tuple[Path, int]] = [] + seen = 0 + stack = [(root, 0)] + while stack and seen < max_entries: + folder, level = stack.pop() + try: + entries = list(os.scandir(folder)) + except OSError: + continue + for item in entries: + seen += 1 + try: + if item.is_dir(follow_symlinks=False): + if level < depth and item.name not in SKIP_DIRS and not item.name.startswith("."): + stack.append((Path(item.path), level + 1)) + elif Path(item.name).suffix.lower() in project.DATA_EXTENSIONS: + size = item.stat().st_size + if size > threshold: + found.append((Path(item.path), size)) + except OSError: + continue + found.sort(key=lambda f: -f[1]) + return [(p.relative_to(root) if p.is_relative_to(root) else p, s) for p, s in found[:limit]] + + +def unloaded_hint(cwd: str, threshold: int) -> str: + files = unloaded_files(cwd, threshold) + if not files: + return "" + lines = ["This project has data files too big to read into context, not yet in Hotdata:"] + lines += [f" tabular-memory load {shlex.quote(str(p))} # {s / (1024 * 1024):.0f}MB" for p, s in files] + lines.append("Load one before answering questions about it; the load prints its schema " + "and a sample, and every question after that is a `tabular-memory query`.") + return "\n".join(lines) diff --git a/scripts/hd.py b/scripts/tabular_memory.py similarity index 82% rename from scripts/hd.py rename to scripts/tabular_memory.py index 98e25a0..e25fdb5 100644 --- a/scripts/hd.py +++ b/scripts/tabular_memory.py @@ -1,10 +1,12 @@ -"""hd: the Hotdata command for agents. One database per project; nothing into context. - - hd load the file becomes a table; prints the orientation - hd query "" answer a question; {{name}} expands to a definition - hd define "" a filter or derived set every agent shares - hd share
"" keep a result on the server as a table - hd fork / attach / defs / status / orient +"""tabular-memory: the Hotdata command for agents. One database per project; nothing into context. + + tabular-memory load the file becomes a table; prints the orientation + tabular-memory query "" answer a question; {{name}} expands to a definition + tabular-memory define "" a filter or derived set every agent shares + tabular-memory share
"" keep a result on the server as a table + tabular-memory fork / attach / defs / status / orient + tabular-memory doctor check the setup, and say how to fix what fails + tabular-memory demo load a public sample and suggest questions to ask """ from __future__ import annotations @@ -24,13 +26,14 @@ import hotdata import orient import project +import setup from hotdata import HotdataError def entry(create: bool = False) -> dict: found = project.project_database(os.getcwd(), create=create) if not found: - raise HotdataError("this project has no database yet: `hd load ` makes one") + raise HotdataError("this project has no database yet: `tabular-memory load ` makes one") return found @@ -246,7 +249,7 @@ def cmd_define(args) -> int: def cmd_defs(args) -> int: found = defs.all_definitions(entry()["project"], refresh=True) if not found: - print("no definitions yet: hd define \"\" --description \"...\"") + print("no definitions yet: tabular-memory define \"\" --description \"...\"") return 0 for name, d in sorted(found.items()): print(f"{{{{{name}}}}} v{d['version']} {d.get('description') or ''}") @@ -304,7 +307,8 @@ def cmd_status(args) -> int: e = project.project_database(os.getcwd()) print(f"project: {project.project_label(os.getcwd())}") if not e: - print("database: none yet — `hd load ` creates one") + print("database: none yet — `tabular-memory load ` creates one, " + "or `tabular-memory demo` loads a sample") return 0 print(f"database: {e['database']} (catalog {e['catalog']})") print(f"guard: {project.option('read_guard', 'on')}, " @@ -329,8 +333,45 @@ def cmd_orient(args) -> int: return 0 +def cmd_doctor(args) -> int: + results = setup.checks(os.getcwd()) + for ok, what, fix in results: + print(f"{'ok ' if ok else 'FAIL'} {what}" + (f"\n -> {fix}" if fix else "")) + failed = sum(not ok for ok, _, _ in results) + print("\nready." if not failed else f"\n{failed} check{'s' * (failed != 1)} failed.") + return 1 if failed else 0 + + +DEMO_URL = "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-01.parquet" +DEMO_TABLE = "nyc_taxi_demo" +DEMO_QUESTIONS = """Ask Claude, in plain words: + 1. "Which pickup location has the most trips, and what is its average fare?" + 2. "How does the average tip change by hour of day?" + 3. "Define clean_trips as trips with a positive fare and distance, then compare + tip percentage by payment type using it." +Each answer comes back as a few rows; the 2.9M-row table never enters context. +Remove the demo with: tabular-memory demo --remove""" + + +def cmd_demo(args) -> int: + e = entry(create=not args.remove) + if args.remove: + hotdata.remove_table(e["database"], DEMO_TABLE) + print(f"removed {e['catalog']}.public.{DEMO_TABLE}") + orient.refresh(e, orient.budget()) + return 0 + print(f"loading NYC yellow taxi trips, January 2024 (48MB, 2.9M rows); " + f"the server fetches it, nothing downloads here...", flush=True) + started = time.time() + hotdata.load_file(e["database"], DEMO_TABLE, DEMO_URL) + print(f"loaded -> {e['catalog']}.public.{DEMO_TABLE} in {time.time() - started:.1f}s\n") + orient.refresh(e, orient.budget()) + print(DEMO_QUESTIONS) + return 0 + + def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(prog="hd", description=__doc__, + p = argparse.ArgumentParser(prog="tabular-memory", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) @@ -375,11 +416,18 @@ def main(argv: list[str] | None = None) -> int: s = sub.add_parser("orient", help="print what an agent is told at session start") s.add_argument("--refresh", action="store_true"); s.set_defaults(run=cmd_orient) + s = sub.add_parser("doctor", help="check the setup and say how to fix what fails") + s.set_defaults(run=cmd_doctor) + + s = sub.add_parser("demo", help="load a public sample dataset and suggest questions") + s.add_argument("--remove", action="store_true", help="drop the demo table") + s.set_defaults(run=cmd_demo) + args = p.parse_args(argv) try: return args.run(args) except HotdataError as error: - print(f"hd: {error}", file=sys.stderr) + print(f"tabular-memory: {error}", file=sys.stderr) return 1 diff --git a/skills/tabular-memory/SKILL.md b/skills/tabular-memory/SKILL.md index 037f347..70d2b85 100644 --- a/skills/tabular-memory/SKILL.md +++ b/skills/tabular-memory/SKILL.md @@ -9,14 +9,14 @@ The data stays on the server. A question goes to it as SQL and an answer comes b one number, a few rows, never the table. One database per project; every agent working in the project finds the same one. -The `hd` command is on your PATH while this plugin is enabled. +The `tabular-memory` command is on your PATH while this plugin is enabled. ## Load a file ```sh -hd load data/trips.parquet # creates the project database on first use -hd load data/zones.csv --table zones # name the table; default is the file's stem -hd load https://example.com/trips.parquet # the server fetches it; nothing passes through here +tabular-memory load data/trips.parquet # creates the project database on first use +tabular-memory load data/zones.csv --table zones # name the table; default is the file's stem +tabular-memory load https://example.com/trips.parquet # the server fetches it; nothing passes through here ``` Takes a few seconds for 50MB and prints the schema, row count, time ranges, a sample, @@ -30,14 +30,14 @@ each time, whatever the call does. **If you have more than one question, ask the call.** That is where almost all of the saving is: ```sh -hd batch "q11: SELECT count(*) FROM {{clean_set}} WHERE \"PULocationID\" = 132 AND \"DOLocationID\" = 230" \ +tabular-memory batch "q11: SELECT count(*) FROM {{clean_set}} WHERE \"PULocationID\" = 132 AND \"DOLocationID\" = 230" \ "q12: SELECT round(avg(total_amount), 2) FROM {{clean_set}} WHERE \"PULocationID\" = 132 AND \"DOLocationID\" = 230" ``` Or a script on stdin, one statement per `;`, with `-- name` lines as labels: ```sh -hd batch <<'SQL' +tabular-memory batch <<'SQL' -- rows SELECT count(*) FROM {{clean_set}}; -- top_pickup @@ -52,13 +52,13 @@ anything is missing. ## Ask a question ```sh -hd query "SELECT count(*) FROM taxi.public.trips WHERE fare_amount > 0" -hd query "SELECT \"PULocationID\", count(*) AS n FROM taxi.public.trips GROUP BY 1 ORDER BY n DESC LIMIT 10" +tabular-memory query "SELECT count(*) FROM taxi.public.trips WHERE fare_amount > 0" +tabular-memory query "SELECT \"PULocationID\", count(*) AS n FROM taxi.public.trips GROUP BY 1 ORDER BY n DESC LIMIT 10" ``` Aggregate, or add a `LIMIT`. A query with neither is refused when it matches more rows than the row cap, and anything over the cap is cut with a footer — the rows are still on -the server, and `hd share` keeps them there as a table. `--out rows.csv` writes every row +the server, and `tabular-memory share` keeps them there as a table. `--out rows.csv` writes every row to a file and prints only a summary; `--format json` prints objects; `--limit N` raises the cap for one call. `INSERT INTO .public.
... VALUES ...` writes rows; put them all in one statement. @@ -70,14 +70,14 @@ quarter — belongs in a definition, so every agent applies the same one and nob invents a variant: ```sh -hd define clean_set "SELECT * FROM taxi.public.trips WHERE fare_amount > 0 AND trip_distance > 0" \ +tabular-memory define clean_set "SELECT * FROM taxi.public.trips WHERE fare_amount > 0 AND trip_distance > 0" \ --description "trips with a positive fare and distance" -hd query "SELECT count(*) FROM {{clean_set}}" -hd query "SELECT avg(c.total_amount) FROM {{clean_set}} AS c JOIN taxi.public.zones z ON ..." +tabular-memory query "SELECT count(*) FROM {{clean_set}}" +tabular-memory query "SELECT avg(c.total_amount) FROM {{clean_set}} AS c JOIN taxi.public.zones z ON ..." ``` `{{name}}` expands to the definition as a subquery. Defining a name again makes a new -version; the old one stays in history. `hd defs` lists them. +version; the old one stays in history. `tabular-memory defs` lists them. **If a question says "the clean set" and no definition exists, define it from the question's own words before answering.** Do not apply a filter you were not given. @@ -85,8 +85,8 @@ question's own words before answering.** Do not apply a filter you were not give ## Keep a result on the server ```sh -hd share pair_stats "SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS dz, count(*) AS n, avg(total_amount) AS fare FROM {{clean_set}} GROUP BY 1, 2 HAVING count(*) >= 500" -hd query "SELECT pu, dz, fare FROM taxi.public.pair_stats ORDER BY fare DESC LIMIT 5" +tabular-memory share pair_stats "SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS dz, count(*) AS n, avg(total_amount) AS fare FROM {{clean_set}} GROUP BY 1, 2 HAVING count(*) >= 500" +tabular-memory query "SELECT pu, dz, fare FROM taxi.public.pair_stats ORDER BY fare DESC LIMIT 5" ``` The intermediate never enters anyone's context, and every agent in the project can @@ -95,13 +95,14 @@ query the table. ## Work alongside other agents ```sh -hd fork # an isolated copy to experiment in; prints its id and snapshot -hd attach # see another database's tables as .public.
-hd status # project, database, tables, definitions +tabular-memory fork # an isolated copy to experiment in; prints its id and snapshot +tabular-memory attach # see another database's tables as .public.
+tabular-memory status # project, database, tables, definitions +tabular-memory doctor # a command fails for no clear reason: check the setup ``` A fork diverges and does not merge back; put a result worth keeping into the shared -database with `hd share`. +database with `tabular-memory share`. ## Dialect diff --git a/swarm/runner.py b/swarm/runner.py index 99b0801..1dd5383 100644 --- a/swarm/runner.py +++ b/swarm/runner.py @@ -95,7 +95,7 @@ "write them as CSV into {shared}/labels/ with the columns complaint_id,category," "agent. Other agents read that folder, possibly while you write to it."), "hotdata": ( - """The data is in the project's Hotdata database, and `hd` is on your PATH. Your + """The data is in the project's Hotdata database, and `tabular-memory` is on your PATH. Your session started with its tables, a sample of each and the SQL dialect. The tables: {cat}.public.trips one row per trip, keyed by trip_id @@ -106,7 +106,7 @@ `labels` holds the classifications other agents working on related questions have made. Use them rather than classifying a complaint again.""", - "INSERT them into {cat}.public.labels with `hd query`, all of them in one " + "INSERT them into {cat}.public.labels with `tabular-memory query`, all of them in one " "statement. If the insert is refused with \"retry shortly\", another agent is " "writing; run the same statement again after a few seconds."), } @@ -144,7 +144,7 @@ def settings_for(arm: str) -> dict: """ permissions = {"allow": ["Bash", "Read", "Write", "Edit", "Glob", "Grep"]} if arm != "hotdata": - permissions["deny"] = ["Bash(hotdata:*)", "Bash(hd:*)"] + permissions["deny"] = ["Bash(hotdata:*)", "Bash(tabular-memory:*)"] return {"permissions": permissions, "autoMemoryEnabled": False, "enabledPlugins": {"hotdata-tabular-memory@hotdata": False}} diff --git a/tests/test_hd.py b/tests/test_tabular_memory.py similarity index 74% rename from tests/test_hd.py rename to tests/test_tabular_memory.py index 2bd1c5f..dfff61a 100644 --- a/tests/test_hd.py +++ b/tests/test_tabular_memory.py @@ -50,8 +50,17 @@ def out(o): print(json.dumps(o)) "sql_hash": "h1", "result_id": "rsltOLD", "row_count": 1, "snapshot_id": 6, "status": "succeeded"}, {"id": "qrun4", "sql_text": "SELECT avg(x) FROM t", "sql_hash": "h3", "result_id": "rsltC", "row_count": 1, "snapshot_id": 7, "status": "failed", "error_message": "boom"}, - {"id": "qrun5", "sql_text": "/* hd:internal */ SELECT count(*) AS n FROM taxi.public.trips", + {"id": "qrun5", "sql_text": "/* tabular-memory:internal */ SELECT count(*) AS n FROM taxi.public.trips", "sql_hash": "h4", "result_id": "rsltHK", "row_count": 1, "snapshot_id": 7, "status": "succeeded"}]) +elif a[:2] == ["auth", "status"]: + if __import__("os").environ.get("FAKE_AUTH_BROKEN"): + print("error: could not reach https://api.hotdata.dev", file=sys.stderr); sys.exit(1) + ok = "No" if __import__("os").environ.get("FAKE_LOGGED_OUT") else "Yes" + print(f"\x1b[38;5;8mAuthenticated: \x1b[0m{ok}") +elif a == ["--version"]: + print("hotdata 0.43.0") +elif a[:3] == ["databases", "tables", "remove"]: + pass elif a[:2] == ["workspaces", "list"]: out([{"public_id": "workfake", "name": "W"}]) elif a[0] == "query": @@ -102,6 +111,8 @@ def env(tmp_path, monkeypatch): monkeypatch.setenv("PATH", f"{fake}:{os.environ['PATH']}") monkeypatch.setenv("FAKE_LOG", str(tmp_path / "calls.log")) monkeypatch.setenv("CLAUDE_PLUGIN_DATA", str(tmp_path / "data")) + (tmp_path / "data").mkdir() + (tmp_path / "data" / "ready").write_text("ok\n") # onboarding tests remove it monkeypatch.setenv("HOTDATA_API_KEY", "fake") monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_WORKSPACE", "workfake") repo = tmp_path / "my-repo" @@ -115,8 +126,8 @@ def calls(env) -> list[str]: return log.read_text().splitlines() if log.exists() else [] -def hd(args, env, extra=None): - return subprocess.run([sys.executable, str(SCRIPTS / "hd.py"), *args], +def cli(args, env, extra=None): + return subprocess.run([sys.executable, str(SCRIPTS / "tabular_memory.py"), *args], capture_output=True, text=True, env={**os.environ, **(extra or {}), "PYTHONPATH": str(SCRIPTS)}) @@ -244,7 +255,7 @@ def test_a_query_is_classified_by_what_it_would_bring_back(sql, kind): def test_rows_over_the_cap_are_cut_with_a_footer(): rows, footer = discipline.trim([[i] for i in range(50)], 50, 10) - assert len(rows) == 10 and "10 of 50" in footer and "hd share" in footer + assert len(rows) == 10 and "10 of 50" in footer and "tabular-memory share" in footer rows, footer = discipline.trim([[1]], 1, 10) assert footer == "" @@ -263,7 +274,7 @@ def test_reading_a_large_data_file_is_refused_with_the_load_command(env): out = hook("guard", {"tool_name": "Read", "tool_input": {"file_path": str(path)}}) decision = out["hookSpecificOutput"] assert decision["permissionDecision"] == "deny" - assert f"hd load {path}" in decision["permissionDecisionReason"] + assert f"tabular-memory load {path}" in decision["permissionDecisionReason"] assert "20MB" in decision["permissionDecisionReason"] @@ -293,7 +304,7 @@ def test_a_bash_loader_with_no_path_only_gets_a_nudge(env): cmd = "python3 -c \"import pandas as pd; df = pd.read_csv(io.StringIO(s))\"" out = hook("guard", {"tool_name": "Bash", "tool_input": {"command": cmd}}) assert "permissionDecision" not in out.get("hookSpecificOutput", {}) - assert "hd load" in out["hookSpecificOutput"]["additionalContext"] + assert "tabular-memory load" in out["hookSpecificOutput"]["additionalContext"] def test_the_threshold_is_configurable(env): @@ -321,7 +332,7 @@ def test_orientation_carries_tables_columns_definitions_and_dialect(env): assert "fare_amount:Float64" in text assert "{{clean_set}}" in text assert "date_diff" in text - assert "hd query" in text + assert "tabular-memory query" in text def test_the_second_session_reads_the_cache(env): @@ -334,13 +345,13 @@ def test_the_second_session_reads_the_cache(env): assert len([c for c in calls(env)[before:] if c.startswith("query")]) == 0 or True -# --------------------------------------------------------------------------- hd +# --------------------------------------------------------------------------- tabular-memory def test_load_creates_the_database_and_prints_orientation(env): f = env / "trips.csv" f.write_text("a,b\n1,2\n") - out = hd(["load", str(f)], env) + out = cli(["load", str(f)], env) assert out.returncode == 0, out.stderr assert "loaded trips.csv" in out.stdout and "trips (2,869,585 rows)" in out.stdout assert any("tables load trips --database" in c for c in calls(env)) @@ -348,22 +359,22 @@ def test_load_creates_the_database_and_prints_orientation(env): def test_an_unbounded_query_is_refused(env): project.project_database(create=True) - out = hd(["query", "SELECT * FROM taxi.public.trips"], env) + out = cli(["query", "SELECT * FROM taxi.public.trips"], env) assert out.returncode == 2 - assert "hd share" in out.stderr + assert "tabular-memory share" in out.stderr def test_a_short_list_without_a_limit_is_shown_not_refused(env): """The cap protects context from a scan; 150 ids are not a scan.""" project.project_database(create=True) - out = hd(["query", "SELECT complaint_id FROM short_list"], env) + out = cli(["query", "SELECT complaint_id FROM short_list"], env) assert out.returncode == 0, out.stderr assert out.stdout.count("\n") == 151 def test_an_unquoted_uppercase_column_is_fixed_from_the_engines_suggestion(env): project.project_database(create=True) - out = hd(["query", "SELECT count(*) FROM taxi.public.trips t WHERE t.PULocationID = 132"], env) + out = cli(["query", "SELECT count(*) FROM taxi.public.trips t WHERE t.PULocationID = 132"], env) assert out.returncode == 0, out.stderr assert '"PULocationID"' in out.stdout and "2869585" in out.stdout assert any('t."PULocationID" = 132' in c for c in calls(env)) @@ -371,7 +382,7 @@ def test_an_unquoted_uppercase_column_is_fixed_from_the_engines_suggestion(env): def test_out_writes_every_row_to_a_file_and_prints_only_a_summary(env): project.project_database(create=True) - out = hd(["query", "SELECT x FROM t", "--out", "rows.csv"], env) + out = cli(["query", "SELECT x FROM t", "--out", "rows.csv"], env) assert out.returncode == 0, out.stderr assert "wrote 3 rows to rows.csv" in out.stdout assert Path("rows.csv").read_text() == "x\n1\n2\n3\n" @@ -379,14 +390,14 @@ def test_out_writes_every_row_to_a_file_and_prints_only_a_summary(env): def test_format_json_prints_one_object_per_row(env): project.project_database(create=True) - out = hd(["query", "SELECT fare_amount, PULocationID FROM t LIMIT 2", "--format", "json"], env) + out = cli(["query", "SELECT fare_amount, PULocationID FROM t LIMIT 2", "--format", "json"], env) assert out.returncode == 0, out.stderr assert json.loads(out.stdout)[0] == {"fare_amount": 10.5, "PULocationID": 132} def test_a_capped_query_is_cut_with_a_footer(env): project.project_database(create=True) - out = hd(["query", "SELECT a, b FROM t LIMIT 500"], env, {"CLAUDE_PLUGIN_OPTION_ROW_CAP": "20"}) + out = cli(["query", "SELECT a, b FROM t LIMIT 500"], env, {"CLAUDE_PLUGIN_OPTION_ROW_CAP": "20"}) assert out.returncode == 0, out.stderr assert out.stdout.count("\n") >= 21 and "20 of 500" in out.stdout @@ -394,7 +405,7 @@ def test_a_capped_query_is_cut_with_a_footer(env): def test_a_definition_expands_before_the_query_runs(env): project.project_database(create=True) seed_defs(env, clean_set="SELECT * FROM taxi.public.trips WHERE fare_amount > 0") - out = hd(["query", "SELECT count(*) FROM {{clean_set}}"], env) + out = cli(["query", "SELECT count(*) FROM {{clean_set}}"], env) assert out.returncode == 0, out.stderr assert "2869585" in out.stdout sent = [c for c in calls(env) if c.startswith("query")][-1] @@ -403,7 +414,7 @@ def test_a_definition_expands_before_the_query_runs(env): def test_share_materialises_from_the_result_id(env): project.project_database(create=True) - out = hd(["share", "pairs", "SELECT a, b FROM t LIMIT 500"], env) + out = cli(["share", "pairs", "SELECT a, b FROM t LIMIT 500"], env) assert out.returncode == 0, out.stderr assert "500 rows, on the server" in out.stdout assert any("tables load pairs --database" in c and "--result-id rslt2" in c for c in calls(env)) @@ -411,7 +422,7 @@ def test_share_materialises_from_the_result_id(env): def test_fork_prints_the_snapshot_and_records_it(env): project.project_database(create=True) - out = hd(["fork", "--name", "exp"], env) + out = cli(["fork", "--name", "exp"], env) assert out.returncode == 0, out.stderr assert "snapshot 645838" in out.stdout assert json.loads(project.registry_path().read_text())["my_repo"]["forks"][0]["snapshot_id"] == 645838 @@ -420,15 +431,15 @@ def test_fork_prints_the_snapshot_and_records_it(env): def test_a_cli_failure_is_a_message_not_a_trace(env): project.project_database(create=True) seed_defs(env) # an empty, cached set: the lookup must not reach the network - out = hd(["query", "SELECT count(*) FROM {{missing}}"], env) + out = cli(["query", "SELECT count(*) FROM {{missing}}"], env) assert out.returncode == 1 - assert "hd: no definition named 'missing'" in out.stderr + assert "tabular-memory: no definition named 'missing'" in out.stderr assert "Traceback" not in out.stderr def test_status_with_no_database_says_how_to_get_one(env): - out = hd(["status"], env) - assert "hd load " in out.stdout + out = cli(["status"], env) + assert "tabular-memory load " in out.stdout def test_another_projects_definitions_are_not_visible(env, monkeypatch): @@ -453,7 +464,7 @@ def test_a_definition_that_is_only_a_where_clause_is_refused_with_the_shape(env, with pytest.raises(hotdata.HotdataError) as err: defs.define("my_repo", "clean_set", "fare_amount > 0 AND trip_distance > 0") assert "whole SELECT" in str(err.value) - assert 'hd define clean_set "SELECT * FROM' in str(err.value) + assert 'tabular-memory define clean_set "SELECT * FROM' in str(err.value) def test_a_cte_counts_as_a_select(env, monkeypatch): @@ -468,7 +479,7 @@ def test_a_cte_counts_as_a_select(env, monkeypatch): def test_batch_runs_every_query_and_labels_each_result(env): project.project_database(create=True) - out = hd(["batch", "rows: SELECT count(*) FROM t", "top: SELECT a, b FROM t LIMIT 3"], env) + out = cli(["batch", "rows: SELECT count(*) FROM t", "top: SELECT a, b FROM t LIMIT 3"], env) assert out.returncode == 0, out.stderr assert "== rows ==" in out.stdout and "2869585" in out.stdout assert "== top ==" in out.stdout @@ -478,7 +489,7 @@ def test_batch_runs_every_query_and_labels_each_result(env): def test_batch_reads_a_script_from_stdin_with_name_comments(env): project.project_database(create=True) script = "-- total\nSELECT count(*) FROM t;\n-- sample\nSELECT a, b\nFROM t LIMIT 3;\n" - out = subprocess.run([sys.executable, str(SCRIPTS / "hd.py"), "batch"], input=script, + out = subprocess.run([sys.executable, str(SCRIPTS / "tabular_memory.py"), "batch"], input=script, capture_output=True, text=True, env={**os.environ, "PYTHONPATH": str(SCRIPTS)}) assert out.returncode == 0, out.stderr @@ -488,7 +499,7 @@ def test_batch_reads_a_script_from_stdin_with_name_comments(env): def test_one_refused_query_does_not_discard_the_answers_beside_it(env): """A non-zero exit reads as total failure, and an agent throws the batch away.""" project.project_database(create=True) - out = hd(["batch", "bad: SELECT * FROM t", "good: SELECT count(*) FROM t"], env) + out = cli(["batch", "bad: SELECT * FROM t", "good: SELECT count(*) FROM t"], env) assert out.returncode == 0, out.stderr assert "== bad ==" in out.stdout and "refused" in out.stdout assert "== good ==" in out.stdout and "2869585" in out.stdout @@ -496,13 +507,13 @@ def test_one_refused_query_does_not_discard_the_answers_beside_it(env): def test_a_batch_where_everything_failed_does_exit_non_zero(env): project.project_database(create=True) - out = hd(["batch", "a: SELECT * FROM t", "b: SELECT * FROM taxi.public.trips"], env) + out = cli(["batch", "a: SELECT * FROM t", "b: SELECT * FROM taxi.public.trips"], env) assert out.returncode == 2 assert "every query failed" in out.stderr def test_batch_labels_unlabelled_queries_by_position(env): - assert [l for l, _ in __import__("hd").split_batch(["SELECT 1", "x: SELECT 2"], "")] == ["q1", "x"] + assert [l for l, _ in __import__("tabular_memory").split_batch(["SELECT 1", "x: SELECT 2"], "")] == ["q1", "x"] # ------------------------------------------------------------- orientation extras @@ -531,7 +542,7 @@ def test_orientation_shows_time_ranges(env): def test_load_hands_a_url_to_the_server(env): - out = hd(["load", "https://example.com/data/trips.parquet?x=1"], env) + out = cli(["load", "https://example.com/data/trips.parquet?x=1"], env) assert out.returncode == 0, out.stderr assert "fetched by the server" in out.stdout sent = [c for c in calls(env) if "tables load trips" in c][0] @@ -541,7 +552,7 @@ def test_load_hands_a_url_to_the_server(env): def test_a_missing_table_error_names_what_exists(env): project.project_database(create=True) seed_defs(env, clean_set="SELECT 1") - out = hd(["query", "SELECT count(*) FROM nope"], env) + out = cli(["query", "SELECT count(*) FROM nope"], env) assert out.returncode == 1 assert "not found" in out.stderr assert "tables here: my_repo.public.trips" in out.stderr @@ -550,7 +561,7 @@ def test_a_missing_table_error_names_what_exists(env): def test_batch_with_arguments_never_touches_stdin(env, monkeypatch): """An agent's shell keeps stdin open; reading it would hang forever.""" - import hd, io + import tabular_memory, io project.project_database(create=True) seed_defs(env) @@ -561,7 +572,7 @@ def isatty(self): return False monkeypatch.setattr(sys, "stdin", Hanging()) monkeypatch.chdir(env / "my-repo") - assert hd.main(["batch", "n: SELECT count(*) FROM t"]) == 0 + assert tabular_memory.main(["batch", "n: SELECT count(*) FROM t"]) == 0 def test_a_definition_keeps_the_alias_the_agent_wrote(env): @@ -577,7 +588,7 @@ def test_a_definition_keeps_the_alias_the_agent_wrote(env): def test_share_can_materialise_a_stored_result_by_id(env): project.project_database(create=True) - out = hd(["share", "again", "--result-id", "rsltA"], env) + out = cli(["share", "again", "--result-id", "rsltA"], env) assert out.returncode == 0, out.stderr assert any("tables load again --database" in c and "--result-id rsltA" in c for c in calls(env)) assert not any(c.startswith("query") and "rsltA" in c for c in calls(env)) @@ -586,6 +597,119 @@ def test_share_can_materialise_a_stored_result_by_id(env): def test_the_plugins_own_reads_are_marked_so_they_never_look_like_answers(env): project.project_database(create=True) hook("session_start", {"cwd": str(env / "my-repo")}) - own = [c for c in calls(env) if c.startswith("query") and "hd:internal" in c] - other = [c for c in calls(env) if c.startswith("query") and "hd:internal" not in c] + own = [c for c in calls(env) if c.startswith("query") and "tabular-memory:internal" in c] + other = [c for c in calls(env) if c.startswith("query") and "tabular-memory:internal" not in c] assert own and not other + + +# -------------------------------------------------------------------- onboarding + + +def fresh(env): + """A machine that has never run a session with the plugin.""" + (env / "data" / "ready").unlink() + + +def test_a_missing_cli_is_reported_to_the_user_with_the_install_command(env): + fresh(env) + out = hook("session_start", {"cwd": str(env / "my-repo")}, + {"PATH": str(Path(sys.executable).parent)}) + assert "hotdata.dev/install" in out["systemMessage"] + assert "cannot run yet" in out["hookSpecificOutput"]["additionalContext"] + assert not (env / "data" / "ready").exists() + + +def test_a_logged_out_cli_is_reported_with_the_login_command(env): + fresh(env) + out = hook("session_start", {"cwd": str(env / "my-repo")}, {"FAKE_LOGGED_OUT": "1"}) + assert "! hotdata auth login" in out["systemMessage"] + assert not (env / "data" / "ready").exists() + + +def test_a_failed_login_check_blocks_nothing_and_is_tried_again(env): + fresh(env) + project.project_database(create=True) + out = hook("session_start", {"cwd": str(env / "my-repo")}, {"FAKE_AUTH_BROKEN": "1"}) + assert "systemMessage" not in out + assert DB in out["hookSpecificOutput"]["additionalContext"] + assert not (env / "data" / "ready").exists() + + +def test_doctor_reports_a_failed_login_check_as_unknown_not_logged_out(env): + out = cli(["doctor"], env, {"FAKE_AUTH_BROKEN": "1"}) + assert out.returncode == 1 + assert "could not check the login" in out.stdout and "auth login" not in out.stdout + + +def test_the_first_ready_session_says_hello_once(env): + fresh(env) + out = hook("session_start", {"cwd": str(env / "my-repo")}) + assert "tabular-memory demo" in out["systemMessage"] + assert (env / "data" / "ready").exists() + assert hook("session_start", {"cwd": str(env / "my-repo")}) == {} + assert len([c for c in calls(env) if c.startswith("auth status")]) == 1 + + +def test_a_ready_machine_does_not_ask_the_cli_about_login(env): + hook("session_start", {"cwd": str(env / "my-repo")}) + assert not [c for c in calls(env) if c.startswith("auth")] + + +def test_large_data_files_not_yet_loaded_are_named_at_session_start(env): + data = env / "my-repo" / "data" + data.mkdir() + (data / "trips.parquet").write_bytes(b"x" * 12 * 1024 * 1024) + (data / "small.csv").write_text("a\n1\n") + deps = env / "my-repo" / "node_modules" + deps.mkdir() + (deps / "huge.csv").write_bytes(b"x" * 12 * 1024 * 1024) + text = hook("session_start", {"cwd": str(env / "my-repo")})["hookSpecificOutput"]["additionalContext"] + assert "tabular-memory load data/trips.parquet" in text and "12MB" in text + assert "small.csv" not in text and "huge.csv" not in text + + +def test_a_hinted_path_with_a_space_is_quoted(env): + (env / "my-repo" / "my trips.csv").write_bytes(b"x" * 12 * 1024 * 1024) + text = hook("session_start", {"cwd": str(env / "my-repo")})["hookSpecificOutput"]["additionalContext"] + assert "tabular-memory load 'my trips.csv'" in text + + +def test_a_project_with_a_database_gets_orientation_not_the_file_hint(env): + (env / "my-repo" / "trips.parquet").write_bytes(b"x" * 12 * 1024 * 1024) + project.project_database(create=True) + text = hook("session_start", {"cwd": str(env / "my-repo")})["hookSpecificOutput"]["additionalContext"] + assert "not yet in Hotdata" not in text and DB in text + + +def test_doctor_passes_on_a_working_setup(env): + out = cli(["doctor"], env) + assert out.returncode == 0, out.stdout + out.stderr + assert "hotdata 0.43.0 installed" in out.stdout and "logged in" in out.stdout + assert "no database yet" in out.stdout and "ready." in out.stdout + + +def test_doctor_stops_at_login_and_names_the_fix(env): + out = cli(["doctor"], env, {"FAKE_LOGGED_OUT": "1"}) + assert out.returncode == 1 + assert "FAIL not logged in" in out.stdout and "! hotdata auth login" in out.stdout + assert "workspace" not in out.stdout + + +def test_doctor_without_the_cli_says_how_to_install_it(env): + out = cli(["doctor"], env, {"PATH": str(Path(sys.executable).parent)}) + assert out.returncode == 1 and "hotdata.dev/install" in out.stdout + + +def test_demo_loads_the_sample_by_url_and_suggests_questions(env): + out = cli(["demo"], env) + assert out.returncode == 0, out.stderr + load = [c for c in calls(env) if c.startswith("databases tables load nyc_taxi_demo")] + assert load and "--url https://" in load[0] + assert "clean_trips" in out.stdout and "demo --remove" in out.stdout + + +def test_demo_remove_drops_the_table(env): + project.project_database(create=True) + out = cli(["demo", "--remove"], env) + assert out.returncode == 0, out.stderr + assert any(c.startswith("databases tables remove nyc_taxi_demo") for c in calls(env))