From d5a7a41fa6eed3bc8326c8e2ba822b1f0378b3aa Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:49:04 -0700 Subject: [PATCH] Prepare for the plugin directory (v4.0.1) - Move bench/ and swarm/ out of the plugin: the directory scans every file in the plugin folder, and their runners read the API key file. - Suggest installing the CLI with Homebrew or the install guide, not curl | sh. - Add the listing icon at .claude-plugin/icon.png. --- .claude-plugin/icon.png | Bin 0 -> 11381 bytes .claude-plugin/plugin.json | 2 +- .gitignore | 4 - README.md | 16 +- bench/.gitignore | 5 - bench/README.md | 100 - bench/aggregate.py | 174 - bench/arms.json | 118 - bench/build_bank.py | 199 - bench/compare.py | 63 - bench/distance.json | 23 - bench/orchestrate.sh | 31 - bench/questions.json | 452 -- bench/redundancy.py | 149 - bench/runner.py | 303 - bench/validate_bank.py | 55 - scripts/hotdata.py | 3 +- scripts/setup.py | 6 +- swarm/analyze.py | 142 - swarm/build_data.py | 119 - swarm/build_tasks.py | 199 - swarm/check_complaints.py | 140 - swarm/gen_complaints.py | 132 - swarm/runner.py | 346 -- swarm/tasks.json | 10002 --------------------------------- tests/test_tabular_memory.py | 4 +- 26 files changed, 17 insertions(+), 12770 deletions(-) create mode 100644 .claude-plugin/icon.png delete mode 100644 bench/.gitignore delete mode 100644 bench/README.md delete mode 100644 bench/aggregate.py delete mode 100644 bench/arms.json delete mode 100644 bench/build_bank.py delete mode 100644 bench/compare.py delete mode 100644 bench/distance.json delete mode 100755 bench/orchestrate.sh delete mode 100644 bench/questions.json delete mode 100644 bench/redundancy.py delete mode 100644 bench/runner.py delete mode 100644 bench/validate_bank.py delete mode 100644 swarm/analyze.py delete mode 100644 swarm/build_data.py delete mode 100644 swarm/build_tasks.py delete mode 100644 swarm/check_complaints.py delete mode 100644 swarm/gen_complaints.py delete mode 100644 swarm/runner.py delete mode 100644 swarm/tasks.json diff --git a/.claude-plugin/icon.png b/.claude-plugin/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..c19fd97b1339150ac7bd8f2a2a0ccd1d5b81c170 GIT binary patch literal 11381 zcmeHt`9IYC*FR&_45w~*2hy_y_%qlklU`xji!k@NpGYg z1)qzmxE9C>KHNCsB7aA?i0561Qb9*tx`}z}RgI%Xo}XW^{F^Mw9#|CBrdT=Z{ABIW z3`lG6blG&Xr!Tc^UWO3J#f?CsnCtU3YvfjKS^nr})@^iO!@a-St-m_0@6a)zz(^AU z%by}1?kLkZS?V{_rs|5r<*m)@@|=Bo{fa1?2T(9jdlz51l`y8)n;yQ1(?eQ0l}xVRe;Tv$WgPCntH>pJFov=W1~cT9H0NN#dNMimh9ZZ>y@Lc@ zqEL|Tl1N-s83scby)2=0c;GeKR=(?OVS f-YV#eVN53ZeFah@QR9|f!8QkrR5&2K@Ag-*zaIM=ihpzL|5ov!s`jl* zA4Yxgo2aQMaJLQi+B?}jR;lja<;LppS{FS!lM`Rw_u*l1QrTQzPsZXO_lnW^&QY79 z5tfP9dMKs@db#njjXvHlTAcGb_I>gHoo92I|BLTvNyXyAaM$KimTy_oNVweN1+(}+ zlVc0({M@dL6buL&pBIMqkS#wp$6vc{$I1DUnFNL!eKav1UVb5EiNfnYNbQcC*yA=C$M8|1C?_Zk< zejI*6+!q1g+YRkNHvW1|S=!fT6$GhDbjEez=$WzyLS)K=7gKTwv9OMaG)5RPke~p~^yucB_D2as`&}nyWeBIXbMv0xe&VN~ z25^qy8pnc6o*=rGTk(Er3j#4jBuA1Pk;aF^rEPVzv4_L5KZ%BMQTGGur^;dVcLQnl zrcIut!1B`wzN*tMod>$@xUqe9di)say3D|9^=|pO4?wr~@%Pba%y9#P?uoY@X6Ta} z$&_@nX&yK&ogas5#?Mj9eL!xXsW_uRf(b}zyFKv-nLr#=_ORSaIF$;Pi>%~FK7j8% zP-pE&ATNPPn(jM~zD1BoZ+qJJa3h+*B0q%}2+|-EO`bmjBb|8%zILZ5Nl2=0*ggak$_D?~T+qVD2}`+PD6roZhMEKw6e@B@2zQJ4(=H z8_dB347Z&k4u%T`-aKy)A+yjpe5>Hjb=(Md;N117ST>}9W_Sb@W!w0 zgpjBh44G^aI87sjgV^CIQBhn}D=_zViH!Bhh6G{?Zrd|KX=MclBTTS>O3(%(?GsUS zE@}r5xkzOQCWF{1+f{gkq}>2OUE&LMU7u_PkF<3sgrpga=`|R;BcPbt@phTI5O5l? zF1IxaJZn4RY0TgSTooidOf6r_jXl5m&O;Q&5OiRMSDr%lg2^-vDXy2!3*H0E-b}^KxTpCPekn8NDTs#Acpo$E-De! zl-sT@WM>9Ss2(elA-@2a=zZ9KDbZV0Z_^_hCqL44Gn1U>W{5Ub(0d;AY}_ z8zJdTI%rSN9>64n6;sqS&I%=i9w~bXUxkkAfj#tERZ*Dy!vx)e!xiypOp5}8>69=I zl}HsVqROzK5mLcrsqfo1QvE=vQX`}N(m4>sdn7B?+W~5}-%1kSk{JWR7?UHv{yW6t z;4bgkt!~Uf+9f$g9tdDY&~1JH^#J59n6}8H_+Vi9PB6+&pUy*pn-T@_Ph5EZiE&Db?jbi zx&P)1YBTWO>}to0&<0OxC%V)kMW7yvtYT>(1mJrDH|+S?cBd@a{TDp(&QeBG`+-rv zY_2W>>I_M=uaLiFvi{J|wcUfrDUf68#wBs;M{x1MeO$GBCyZpZo`*|9d`MDq4G?c3 zFW=52c(Vrfu@>vA&tMRkjI}r*F-q{zcYz#W~M^26C zEKZb(?wfrrWIv!PN_7T@?-uKluY*EfPhJ@H?b@++nV?k`G z`W-2~icR>%6>hpL<3o_qn&#pjPa;JZv~A$#ePi4GIFQzR9TJ+hl+bSMo?dw9Vb@Vg zVOU9&u_C@^zr8HGqTu$SG9YTKU-sOt0?$_=RJSji$Qn%KV_X`3RT1y}&2A%7SU8?) z2H|9O!tcdOYg`t91p;*Ha9P8M{RknuQpJ}(@+}nqJ zG1S8lvGjKWY=fqp8L5?HFwc=HwsCrx&R1Y9c2P;YD3GS2^?WsESq`^ks9ec0tn=V7 z73Qg%957>ZYLrxkcqh3N+OCRtS`a!63dFPhE*jY%?T!QzL6=nc5AgueC^S z0I_dGSlla5hWbYOb`!^q#ggjxI6{g$Ci`Bfg^Z{j%*gb@p6B(ry9R*WE1c4Q{E`i& zU&)~a3&LY_v{N~bIrgQ9mL*eaAf*I_%Gi#3#nkL(6092HITy|CERxPxt$fF7c&!dE=A4Fub!G~#_9Ji|=P@ev`DeN1%5!Pc*PM4h1r zLCEo}$l(Z%a(wUWasyO-Vi(TgoB^gL@oo{jmh&{Z}aykGHMer*_isM@o*t zxE~^dcQ>~0xz5~cx8_(c6aNf#%y&*ehEA2@@k?e%RBkDzx7=5^sA#e|hui4aMY$}`E@KsPxKlfkB&*w2X$`ipHga)* zre}|@S(xgTODL=M?Rb8z-tkxLs3mc5j)y!m70Wy#$rxt=GMYMb$?P^n-0hU97xMD> z`EwmVO*XKx*{CD5{dG_YO(+f~wL@ot@1uLo(ExBC-_0Ks*h946)0wWG#PE;k>ys{QUM5ZhV&_R?r!Pd3&`!@1 zG>6y-Qbg#N&joIId{o2E_DA5mt>1o+KouC#K^*1LAg1goO%FasF1*v#&w|-uD)N%e zK_C`0+u6V&zKrOC81fT#5P|dDeXf2?XG}&FSxEuj+>9D<#v@~x%v&!pzEHt@^$Nma z5z1)*U?QWm+NcfmeeQWbm&e7cM}c+fFW>1nuv{G3Mb-^mt%>jb`53q~(#_*vDB!O4 zyEN;9@iENv=Ih;PS$svoBY5b-=)T>Ec4&9bDy25p%lqZ}Ap1_`|NN-sDB2t*0bNld z$T@WaO29;`g||Nx9sjeYkG3-yM(Pu55lPT#lCq{RV1Y4K_p(~zwfe&?xqHf>{FXhJ zua<{eWb4>B2vT{dj4bE@k6xl8NjlFw)P1ttd$`Kqh9y*p99ueggu)KIILl4 zI#JshaTg-#j(YYh4=4+TQC=7g7oXrN6vg0!66QD#>-kg3E zf@mgHv6lsSying4|62eWZarpUGysijB5Ey<0Y~$@jA(9Qy35ntPcW~M64k0zU@DGM z=wHx>B+_pz4kiSGl!103pFx$+-;2C6uhknj?tXk2rrAfKQB$0_=-LA2+2YWtFAdWw z`gL|=FLa0AZykHdx`SIC#A~)eKQwvkbQYdMQmli8_}ar0l5FIU2jqWgC%9zi4;M)$ z!$T-1ya|9fdmzf=H1$OcH9Ea?u&+kkwP<*%X9^bo4hG##maY<|^X;P&Bc-z=Jr68h z>}}6V41J699T8h-UYwJv$n*OU@cmR?Zp-8HjiqBjJuid({n*hJu^K) zsb%VA!t|);&BqrmXB62^yOvEp=~gzq*-nnP4tIX%jm0Dkaie}2*_8r>ulWbHQJTXvvtW@YL3)lTt zTRTy*$d(t|j+Hok*4(?-r27y6v|hEeZ0N>W*;g!U&0ys+ zaYt)J)PKI&|8oA!h5Ne&2Y)-`@USO%q6f9&RUIvCD4>Rhyf`GPGtn6@IZ`}fdDp6P zHiFNIR5iaYX5wocb4JuL4;aG2B?$(Uq@p9Nj=a*}Z~uvIEpI@^`TzOQykJrlVY*Lq zIDBO5Jp#ljM(?~V4j0Lr>-=T(Px@%*6m3*pE>n}J={vGT9xrgdt-+j5kE`G3pTSo( zeiLs#*7d<~eT9tv^u=14YDpCUY$`%d*FPxW1a3MU8s?`Fikwlt&fLVWz(-T42=a}GhKqijVGGRWdc;n z!sP|?Lgu_=2q+~n)Pxh$CIOpGMqlOEI!3WD-08oLk5rtp0sXHGgZb~Pkta)q;+*~m z7^Z^k6_IqnyN@9~A2-~Nx6%z!&yk@Xg^x#PwC4<=#2S|0>`Y(cioro5+;>ZZ2v6CW zGYwS=GFs3OFbTVst%ho``qi$<-Mk4;l%0dh|5Kh}R3j<+n{$+-fKDmB%GLWJuw{0x7EpU)Lrw zUPoKpCpZAIyS(?NIMmXEft!3z=-T6qEVpE4fPpgTvnn8-m{~|Xk)xpkB(1_3m2V=z zir+}T%bdK4ufZH;1X-IHTA%~IqT$1*PY5rmm`pllM)?@$Yo`&g;CIwr{6YCaXo@U% zC@RruvFlqbyKmdf)w?b73_wwP^W_ndL8ohtrV9xf>&nSyLQi& zuPeqk7+%nXeI~m;p|f#-!xl_Px8F}aPx308wYut*eceg{Cvn5+HZqYNKBS4nIFRoR(JH~{$Kz-h6_atlNPbNX&UT4dmW z;CE9PUF9Y+{D0eQY7KEdw_)sS+`sQ1-6li>wQRw{gwCQFd-SeGMANcnc6|JE(X^jl zeUKpl!%_I=VAHFARKtf}@6O_%))B3ayElW%hDkE}{@1ZjcNa#+hF;X}&Ju5bx1H?& zCUf(wNvfx6w9|K17VVOSlR$UX2=#nLaBuJ3aPJR*FLLRPf#jy*8vV zTZEf)>nLEQGc|6iLA1x!ZpwYSRE*J$u#jzxZ0>B>vxLHz+c0c&0KKTq>AKNi$w|yh zWXcIt9!Dy`!dPLAbJaT&N}^qXIy9URa`q~2ID2(Gw0VgeF;2JHO)gBIp$ArZRkv2G zmB?-&?qM(!g~hQ1VJYS@DF$2*w-9sxValOji7s;;esn*Ux%t^ZNicMe;AC&uIr5Q$ zm)PDc{q-P{9?3Iy1+f7z$A+6CJF_C_r~{5hkS!p{LoPV%Yi(cz{Z56A{ZCRAX-x9a zvIH^)o;1M}`?RZ&GFRze^aBI0^{Ogs5XZM1hYj!vi!T$XMtLy7!`5v=TEwWJ&rh|0 zQbNNTwxBQ|qtf{`hy^A+>-Zt{`E`BQQ28LB`1GUQ&&lK6e^|__L6K40&?O7&rg5=p zFMx>)5+SbD&>5@OIC(G2oBJ9p_JBMUCJsq2zH2TD1b$-+i{A)(nws?^Hb6i(Wv!U^ zJs4tJ%J2yQj&BBF{|@)Zz3$2i-E!Wrp5gdbYM6_*_tQPTve#xk<0{-fN zD{E=|FJS|JgQ2m}Oq!)oy%*@_*D>h-+Zy5I1y{ee4-E!EldOg6B_$4eE1YQq% za$d5PNb2`!i!tWxJlSj98z2CzPrGkHCqjHXzs%IU0!-^*n35(INzeV2Exx3d6S(2R zvz)P|`WWUaluU+=rE>P;gPk!&5&l4hL#Iy$DtW$?0qJb5BuB#?h2`a_0!Yx zk`j{M-}FgUdyCDr4*;g(eUpbe0_ku>Rdq8V&h>k&RXMS0 z=(VI98-#K99ztvYqiqQK{3_W5n%MJnZ5;k`-+7DW+yC2W4I+E-SPHFS8iz@S%9FC= z#n!E!wEF#FN1gSEDGJ*w1Vx2~rLKJjnEda-uHoyEbeCs&Mj7{4b>ayxEr7)!JpyES1X;B^?WX^ti94Q@P9&aw!4Xho`r2R5O;JIpEp^5MMG&Fv6Sp*Ma~0O(Tc z;~I)!N}iQxQ0KLX_R_Dba$(XHd2}%q4Eax3O`SaCUyShh?=L|8;pLyPiqT^x zYXvl0XRC-+LHrwO#!a0Xwfkm=#$M>KCaQ?~p8%=?Li~z|E{pi}(W}(AWv*gddj`9| z-}EY()vid|Hy`EnENTEnH73sVNX=wa*v4P!y>+zs^274}Z{G|8hMX&ACKfuIwkNe& zPsql#397o@Do?eU|It%2I-k-r?CSe-OrmQ%wj$Qc-|=0c*~Zqn^DO~$Zv#a4O+@KP zEYunV*s|}76Swy)gd#hCJy=w6@^73@RKL17*4A9=ncw%oQsMEw+T`enO1Q6mF~k2n zy3GBTLBQl&6O&)B-5cf97j7BoYKGQFFTWIoql)u~N{#PNu?Xh8{DXp30DI@hc}r+N z6rIDgrsX%LUcsoxMopZTh7Lmn(V3T*Ur BENCH_WORKSPACE= - ./orchestrate.sh "$PWD/runs-myrun" # the run -python3 aggregate.py # per-arm figures -> report.json -python3 compare.py # the arms side by side -``` - -One agent alone: - -```sh -python3 runner.py --arm plugin --replicate 1 --out "$PWD/runs-x" --sessions 20 \ - --model claude-opus-5 --tag x -``` - -`aggregate.py` reads every `runs*/` directory it knows about; add a new one to its -`sources` list with a suffix, and its arms appear as `plugin_`. - -## What is measured - -Cost, tokens, turns and latency come from each session's own `claude -p --output-format -json` result — nothing is modelled. The rest comes from the transcripts Claude Code -leaves in `~/.claude/projects`, matched to a run by session id so a rerun in the same -working directory cannot read the previous attempt's sessions. - -`redundancy.py` extracts every query an agent ran, including the several inside an -`tabular-memory batch` call, and reports how many were repeats and how many were orientation — a -schema read or an unranked peek, as opposed to a query that answers something. - -## Three traps this harness now guards against - -Each of these silently invalidated a run before it was caught: - -- **The model.** `orchestrate.sh` once called `runner.py` without `--model`, so every - run used the default while the command line said otherwise. Each session now records - the model that did the work, taken from its own `modelUsage` and ignoring the Haiku - call that titles the session. `aggregate.py` prints it beside every arm. -- **The CLI version.** Claude Code was upgraded mid-run. Its system prompt and tool - definitions are the prefix every turn re-reads, and that prefix is most of the token - cost. Each session records `claude --version`; a run spanning two prints `MIXED CLI`. -- **Pricing.** A model newer than the CLI's catalog is billed at a placeholder rate — - Haiku 5.5 was costed at the Opus rate, about 5× high. Any arm whose model the CLI - cannot price prints `COST UNRELIABLE`, and its dollar figures must not be published. - -## Results so far - -Two agents an arm, twenty sessions each. - -| Model | Arm | Correct | Turns | Tokens / session | -| :- | :- | :- | :- | :- | -| Opus 5 | no Hotdata | 95.0% | 5.0 | 138,345 | -| Opus 5 | with the plugin | 97.5% | **3.4** | **97,408** | -| Haiku 5.5 | no Hotdata | 97.5% | 5.2 | 188,229 | -| Haiku 5.5 | with the plugin | 96.0% | 5.1 | 164,122 | - -On Opus the plugin cuts turns 32% and tokens 30%. On Haiku 5.5 it cuts 2% and 13%, and -loses 1.5 points of accuracy. The transcripts say why: Haiku 5.5 calls `tabular-memory batch` as -often as Opus does in absolute terms, 42 times against 40, but makes 70% more calls -overall — it uses the affordance and keeps working the old way alongside it. The plugin -removes turns an agent would otherwise spend; it cannot remove turns it spends anyway. - -Haiku 5.5 also answers 97.5% unaided, so there was no accuracy headroom to recover. - -**Unexplained:** Haiku 5.5 uses more tokens per session than Opus 5 on the same work. -It may be real, or an artefact of how an unrecognised model is handled. Do not quote it -until someone has checked. diff --git a/bench/aggregate.py b/bench/aggregate.py deleted file mode 100644 index 37b23a4..0000000 --- a/bench/aggregate.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Read every session result and print the per-arm comparison as JSON.""" - -from __future__ import annotations - -import json -import os -import statistics -from collections import defaultdict -from pathlib import Path - -import redundancy - -HERE = Path(__file__).resolve().parent -TAG = os.environ.get("TAG", "") - - -def main() -> None: - sources = [(HERE / "runs", ""), (HERE / "runs-v6", "_fixed"), (HERE / "runs-v21", "_v21"), - (HERE / "runs-local", ""), (HERE / "runs-h55", "_h55")] - arms: dict[str, list] = defaultdict(list) - answers: dict[str, list] = defaultdict(list) - agents = [(p, suffix, runs) for runs, suffix in sources if runs.is_dir() - for p in sorted(runs.iterdir()) if p.is_dir()] - runs_of: dict[str, Path] = {} - for agent, suffix, runs in agents: - arm = agent.name.rsplit("-r", 1)[0] + suffix - runs_of[arm] = runs - for path in sorted(agent.glob("s*.json")): - row = json.loads(path.read_text()) - usage = row.get("usage") or {} - arms[arm].append({ - "agent": agent.name + suffix, - "session": row.get("session_index"), - "pass": row.get("pass", 1), - "cost": row.get("total_cost_usd", 0.0), - "turns": row.get("num_turns", 0), - "wall": row.get("wall_seconds", 0.0), - "api_ms": row.get("duration_api_ms", 0), - "input": usage.get("input_tokens", 0), - "output": usage.get("output_tokens", 0), - "cache_read": usage.get("cache_read_input_tokens", 0), - "cache_write": usage.get("cache_creation_input_tokens", 0), - "error": bool(row.get("is_error")), - # the model that did the work, not the one that titled the session - "cli": row.get("cli_version", ""), - "models": [m for m, u in (row.get("modelUsage") or {}).items() - if (u.get("cacheReadInputTokens") or 0) > 1000], - # "unknown" means this CLI has no price for the model and is guessing, - # so the dollar figures for that arm are not to be published - "cost_basis": [u.get("costBasis") for m, u in (row.get("modelUsage") or {}).items() - if (u.get("cacheReadInputTokens") or 0) > 1000], - }) - path = agent / "answers.jsonl" - if path.exists(): - for line in path.read_text().splitlines(): - row = json.loads(line) - row["agent"] = agent.name + suffix - answers[arm].append(row) - - def stability(graded): - """Count what happened to each question between pass 1 and pass 2. - - A question answered right in pass 1 and wrong in pass 2 is memory doing harm; - the same wrong answer twice is an error that memory preserved. - """ - seen = defaultdict(dict) - for a in graded: - seen[(a["agent"], a["id"])][a.get("pass", 1)] = a - out = {"right_both": 0, "wrong_both_same": 0, "wrong_both_diff": 0, - "right_to_wrong": 0, "wrong_to_right": 0, "pairs": 0} - for v in seen.values(): - if len(v) != 2: - continue - out["pairs"] += 1 - one, two = v[1], v[2] - if one["correct"] and two["correct"]: - out["right_both"] += 1 - elif one["correct"]: - out["right_to_wrong"] += 1 - elif two["correct"]: - out["wrong_to_right"] += 1 - elif str(one["given"]) == str(two["given"]): - out["wrong_both_same"] += 1 - else: - out["wrong_both_diff"] += 1 - return out - - report = {} - for arm, rows in arms.items(): - graded = answers[arm] - by_phase = defaultdict(list) - by_session = defaultdict(list) - by_pass = defaultdict(list) - for a in graded: - by_phase[a["phase"]].append(a["correct"]) - by_session[a["session"]].append(a["correct"]) - by_pass[a.get("pass", 1)].append(a["correct"]) - pass_cost = defaultdict(list) - pass_turns = defaultdict(list) - for r in rows: - pass_cost[r["pass"]].append(r["cost"]) - pass_turns[r["pass"]].append(r["turns"]) - costs = [r["cost"] for r in rows] - report[arm] = { - "agents": len({r["agent"] for r in rows}), - "sessions": len(rows), - "questions": len(graded), - "accuracy": round(sum(a["correct"] for a in graded) / max(len(graded), 1), 4), - "accuracy_by_phase": { - str(p): round(sum(v) / len(v), 4) for p, v in sorted(by_phase.items()) - }, - "accuracy_by_session": { - str(s): round(sum(v) / len(v), 4) for s, v in sorted(by_session.items()) - }, - "accuracy_by_pass": { - str(k): round(sum(v) / len(v), 4) for k, v in sorted(by_pass.items()) - }, - "cost_by_pass": { - str(k): round(sum(v), 4) for k, v in sorted(pass_cost.items()) - }, - "turns_by_pass": { - str(k): sum(v) for k, v in sorted(pass_turns.items()) - }, - "stability": stability(graded), - "models": sorted({m for r in rows for m in r.get("models", [])}), - "cost_basis": sorted({b for r in rows for b in r.get("cost_basis", []) if b}), - "cli": sorted({r["cli"] for r in rows if r.get("cli")}), - "swarm": redundancy.swarm_metrics(runs_of[arm], arm.split("_")[0]), - "tools": redundancy.tool_use(runs_of[arm], arm.split("_")[0]), - "invented_filter_errors": redundancy.invented_filter_errors(graded), - "per_agent": sorted( - [ - { - "agent": name, - "accuracy": round( - sum(a["correct"] for a in graded if a["agent"] == name) - / max(sum(1 for a in graded if a["agent"] == name), 1), 4), - "cost": round( - sum(r["cost"] for r in rows if r["agent"] == name), 4), - "turns": sum(r["turns"] for r in rows if r["agent"] == name), - } - for name in sorted({r["agent"] for r in rows}) - ], - key=lambda d: d["agent"], - ), - "cost_total": round(sum(costs), 4), - "cost_per_session_mean": round(statistics.mean(costs), 4) if costs else 0, - "cost_per_correct": round( - sum(costs) / max(sum(a["correct"] for a in graded), 1), 4), - "tokens": { - k: sum(r[k] for r in rows) - for k in ("input", "output", "cache_read", "cache_write") - }, - "turns_total": sum(r["turns"] for r in rows), - "wall_mean_s": round(statistics.mean([r["wall"] for r in rows]), 1) if rows else 0, - "errors": sum(r["error"] for r in rows), - "sessions_detail": rows, - } - out = HERE / "report.json" - out.write_text(json.dumps(report, indent=1) + "\n") - for arm, r in sorted(report.items()): - s = r.get("swarm") or {} - priced = "" if r["cost_basis"] == ["list"] else " COST UNRELIABLE" - if len(r.get("cli", [])) > 1: - priced += f" MIXED CLI {','.join(v.split()[0] for v in r['cli'])}" - print(f"{arm:9s} [{','.join(r['models']) or '?'}]{priced} acc={r['accuracy']:.1%} cost=${r['cost_total']:.2f} " - f"$/correct={r['cost_per_correct']:.3f} turns={r['turns_total']} " - f"redundant={s.get('redundant_pct', '?')}% orient={s.get('orientation_queries', '?')} " - f"invented={r['invented_filter_errors']}") - print(f"\n-> {out}") - - -if __name__ == "__main__": - main() diff --git a/bench/arms.json b/bench/arms.json deleted file mode 100644 index 560f090..0000000 --- a/bench/arms.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "arms": [ - { - "label": "No Hotdata", - "sessions": 40, - "accuracy": 0.95, - "phase3": 0.9667, - "phase4": 0.85, - "flips": 3, - "turns": 5.0, - "tools": 4.0, - "tool_mix": { - "Bash": 160 - }, - "input": 9, - "output": 1528, - "cache_read": 121775, - "cache_write": 15031, - "tokens": 138345, - "cache_per_turn": 24355, - "tokens_per_correct": 29125, - "cost_session": 0.251, - "cost_correct": 0.053, - "secs": 21, - "swarm": { - "queries": 0, - "distinct": 0, - "redundant_pct": 0.0, - "orientation_queries": 0, - "agents": 2 - }, - "context": { - "n": 160, - "median": 427.5, - "p90": 3094, - "max": 6108, - "total": 173398 - } - }, - { - "label": "Hotdata CLI", - "sessions": 40, - "accuracy": 0.97, - "phase3": 0.95, - "phase4": 0.925, - "flips": 2, - "turns": 7.2, - "tools": 6.2, - "tool_mix": { - "Bash": 210, - "Read": 15, - "Edit": 21, - "Write": 1 - }, - "input": 13, - "output": 2228, - "cache_read": 173904, - "cache_write": 15014, - "tokens": 191160, - "cache_per_turn": 24238, - "tokens_per_correct": 39414, - "cost_session": 0.294, - "cost_correct": 0.061, - "secs": 33, - "swarm": { - "queries": 211, - "distinct": 143, - "redundant_pct": 32.2, - "orientation_queries": 84, - "agents": 2 - }, - "context": { - "n": 247, - "median": 171, - "p90": 2985, - "max": 10398, - "total": 220899 - } - }, - { - "label": "Plugin v2.1", - "sessions": 40, - "accuracy": 0.975, - "phase3": 0.9833, - "phase4": 0.9, - "flips": 0, - "turns": 3.4, - "tools": 2.4, - "tool_mix": { - "Bash": 96 - }, - "input": 6, - "output": 764, - "cache_read": 80606, - "cache_write": 16031, - "tokens": 97408, - "cache_per_turn": 23708, - "tokens_per_correct": 19981, - "cost_session": 0.221, - "cost_correct": 0.045, - "secs": 15, - "swarm": { - "queries": 161, - "distinct": 116, - "redundant_pct": 28.0, - "orientation_queries": 3, - "agents": 2 - }, - "context": { - "n": 96, - "median": 240.0, - "p90": 375, - "max": 474, - "total": 19775 - } - } - ] -} diff --git a/bench/build_bank.py b/bench/build_bank.py deleted file mode 100644 index a8368bc..0000000 --- a/bench/build_bank.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Build the benchmark question bank and compute its ground truth with DuckDB. - -Each question has one deterministic answer. The same parquet that was loaded into -Hotdata is read here, so a correct agent answer and the truth come from the same rows. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import duckdb - -HERE = Path(__file__).resolve().parent -TRIPS = str(HERE / "yellow_2024-01.parquet") -ZONES = str(HERE / "taxi_zone_lookup.csv") - -# The filter every sound answer needs. Discovering it is the point of phase 1. -CLEAN = """ - fare_amount > 0 AND total_amount > 0 AND trip_distance > 0 - AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01' - AND tpep_pickup_datetime < TIMESTAMP '2024-02-01' - AND tpep_dropoff_datetime > tpep_pickup_datetime -""" - -Q: list[dict] = [] - - -def add(phase: int, ask: str, sql: str, kind: str = "number", tol: float = 0.01) -> None: - Q.append({"id": f"q{len(Q) + 1:02d}", "phase": phase, "ask": ask, "sql": sql, - "kind": kind, "tol": tol}) - - -# ---- phase 1: discovery. Schema, scale, and the traps. ----------------------------- - -add(1, "How many rows are in the trips table?", "SELECT count(*) FROM t", "number", 0) -add(1, "How many columns does the trips table have?", - "SELECT count(*) FROM (DESCRIBE SELECT * FROM t)", "number", 0) -add(1, "How many trips have a fare_amount of zero or less?", - "SELECT count(*) FROM t WHERE fare_amount <= 0", "number", 0) -add(1, "How many trips have a tip_amount greater than their fare_amount?", - "SELECT count(*) FROM t WHERE tip_amount > fare_amount", "number", 0) -add(1, "How many trips have a pickup datetime outside January 2024?", - "SELECT count(*) FROM t WHERE tpep_pickup_datetime < TIMESTAMP '2024-01-01'" - " OR tpep_pickup_datetime >= TIMESTAMP '2024-02-01'", "number", 0) -add(1, "How many trips have a trip_distance of zero?", - "SELECT count(*) FROM t WHERE trip_distance = 0", "number", 0) -add(1, "How many trips remain after removing rows with a non-positive fare_amount, " - "non-positive total_amount, non-positive trip_distance, a pickup outside " - "January 2024, or a dropoff at or before the pickup? Call this the clean set; " - "every later question means the clean set when it says trips.", - f"SELECT count(*) FROM t WHERE {CLEAN}", "number", 0) -add(1, "How many distinct PULocationID values appear in the clean set?", - f"SELECT count(DISTINCT \"PULocationID\") FROM t WHERE {CLEAN}", "number", 0) -add(1, "How many rows does the taxi zone lookup table have?", - "SELECT count(*) FROM z", "number", 0) -add(1, "Which borough does LocationID 132 belong to?", - "SELECT \"Borough\" FROM z WHERE \"LocationID\" = 132", "string") - -# ---- phase 2: one route at a time. ------------------------------------------------- - -ROUTES = [(132, 230), (138, 161), (161, 237), (237, 236), (186, 79)] -for pu, do in ROUTES: - add(2, f"In the clean set, how many trips went from LocationID {pu} to {do}?", - f"SELECT count(*) FROM t WHERE {CLEAN} AND \"PULocationID\"={pu} " - f"AND \"DOLocationID\"={do}", "number", 0) - add(2, f"In the clean set, what is the mean total_amount for trips from LocationID " - f"{pu} to {do}? Give dollars to two decimals.", - f"SELECT round(avg(total_amount), 2) FROM t WHERE {CLEAN} " - f"AND \"PULocationID\"={pu} AND \"DOLocationID\"={do}") - add(2, f"In the clean set, what is the median trip duration in minutes for trips " - f"from LocationID {pu} to {do}? Give minutes to two decimals.", - f"SELECT round(median(EXTRACT(EPOCH FROM (tpep_dropoff_datetime " - f"- tpep_pickup_datetime)) / 60.0), 2) FROM t WHERE {CLEAN} " - f"AND \"PULocationID\"={pu} AND \"DOLocationID\"={do}") - -# ---- phase 3: comparisons across routes. ------------------------------------------- - -BUSY = f""" - SELECT "PULocationID" AS pu, "DOLocationID" AS do_id, count(*) AS n, - avg(total_amount) AS fare, - avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins, - avg(trip_distance) AS miles, - avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate - FROM t WHERE {CLEAN} - GROUP BY 1, 2 HAVING count(*) >= 500 -""" -add(3, "Among pickup-dropoff LocationID pairs in the clean set with at least 500 trips, " - "which pair has the highest mean total_amount? Answer as PU-DO, e.g. 132-230.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) ORDER BY fare DESC, pu, do_id LIMIT 1", "string") -add(3, "Among pairs with at least 500 trips, which has the lowest mean total_amount? " - "Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) ORDER BY fare ASC, pu, do_id LIMIT 1", "string") -add(3, "Among pairs with at least 500 trips, which has the highest mean trip duration " - "in minutes? Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) ORDER BY mins DESC, pu, do_id LIMIT 1", "string") -add(3, "Among pairs with at least 500 trips, which has the highest mean tip as a share " - "of fare_amount? Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) ORDER BY tip_rate DESC, pu, do_id LIMIT 1", "string") -add(3, "How many pickup-dropoff pairs in the clean set have at least 500 trips?", - f"SELECT count(*) FROM ({BUSY})", "number", 0) -add(3, "Which pickup zone name has the most clean-set trips?", - f"SELECT z.\"Zone\" FROM t JOIN z ON t.\"PULocationID\" = z.\"LocationID\" " - f"WHERE {CLEAN} GROUP BY 1 ORDER BY count(*) DESC LIMIT 1", "string") -add(3, "Which borough has the most clean-set pickups?", - f"SELECT z.\"Borough\" FROM t JOIN z ON t.\"PULocationID\" = z.\"LocationID\" " - f"WHERE {CLEAN} GROUP BY 1 ORDER BY count(*) DESC LIMIT 1", "string") -add(3, "What is the mean total_amount of clean-set trips picked up in Manhattan? " - "Two decimals.", - f"SELECT round(avg(total_amount), 2) FROM t JOIN z ON t.\"PULocationID\" = " - f"z.\"LocationID\" WHERE {CLEAN} AND z.\"Borough\" = 'Manhattan'") -add(3, "In the clean set, which hour of the day (0-23, by pickup time) has the most " - "trips?", - f"SELECT EXTRACT(HOUR FROM tpep_pickup_datetime) FROM t WHERE {CLEAN} GROUP BY 1 " - f"ORDER BY count(*) DESC LIMIT 1", "number", 0) -add(3, "In the clean set, which hour of the day has the highest mean total_amount?", - f"SELECT EXTRACT(HOUR FROM tpep_pickup_datetime) FROM t WHERE {CLEAN} GROUP BY 1 " - f"ORDER BY avg(total_amount) DESC LIMIT 1", "number", 0) -add(3, "In the clean set, what is the mean fare per mile (total_amount / trip_distance) " - "over all trips? Two decimals.", - f"SELECT round(avg(total_amount / trip_distance), 2) FROM t WHERE {CLEAN}") -add(3, "Among pairs with at least 500 trips, which has the highest mean trip_distance? " - "Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) ORDER BY miles DESC, pu, do_id LIMIT 1", "string") -add(3, "How many clean-set trips were paid by credit card (payment_type = 1)?", - f"SELECT count(*) FROM t WHERE {CLEAN} AND payment_type = 1", "number", 0) -add(3, "What share of clean-set trips were paid by credit card? Give a fraction to four " - "decimals.", - f"SELECT round(sum(CASE WHEN payment_type = 1 THEN 1 ELSE 0 END)::DOUBLE / count(*), 4)" - f" FROM t WHERE {CLEAN}", "number", 0.0001) -add(3, "In the clean set, what is the mean Airport_fee over trips where it is greater " - "than zero? Two decimals.", - f"SELECT round(avg(\"Airport_fee\"), 2) FROM t WHERE {CLEAN} AND \"Airport_fee\" > 0") - -# ---- phase 4: synthesis. The best route under a stated rule. ----------------------- - -add(4, "Define route efficiency as mean total_amount divided by mean duration in minutes. " - "Among pairs with at least 500 clean-set trips, which pair has the highest " - "efficiency? Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) ORDER BY fare / mins DESC, pu, do_id LIMIT 1", - "string") -add(4, "Using that same efficiency measure, what is its value for the winning pair? " - "Two decimals.", - f"SELECT round(max(fare / mins), 2) FROM ({BUSY})") -add(4, "Among pairs with at least 500 clean-set trips and a mean trip_distance of at " - "least 3 miles, which has the highest mean total_amount per mile? Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) WHERE miles >= 3 " - f"ORDER BY fare / miles DESC, pu, do_id LIMIT 1", "string") -add(4, "Among pairs with at least 1000 clean-set trips, which has the best combination " - "of high mean total_amount and low mean duration, scored as mean total_amount " - "minus mean duration in minutes? Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM ({BUSY}) WHERE n >= 1000 " - f"ORDER BY fare - mins DESC, pu, do_id LIMIT 1", "string") -add(4, "Name the pickup and dropoff zones of that last winning pair, as " - "' to '.", - f"SELECT zp.\"Zone\" || ' to ' || zd.\"Zone\" FROM ({BUSY}) b " - f"JOIN z zp ON b.pu = zp.\"LocationID\" JOIN z zd ON b.do_id = zd.\"LocationID\" " - f"WHERE b.n >= 1000 ORDER BY b.fare - b.mins DESC, b.pu, b.do_id LIMIT 1", "string") -add(4, "Restricting to clean-set trips picked up between 17:00 and 19:59 inclusive, and " - "to pairs with at least 200 such trips, which pair has the highest mean " - "total_amount? Answer as PU-DO.", - f"SELECT \"PULocationID\" || '-' || \"DOLocationID\" FROM t WHERE {CLEAN} " - f"AND EXTRACT(HOUR FROM tpep_pickup_datetime) BETWEEN 17 AND 19 " - f"GROUP BY 1, \"PULocationID\", \"DOLocationID\" HAVING count(*) >= 200 " - f"ORDER BY avg(total_amount) DESC, \"PULocationID\", \"DOLocationID\" LIMIT 1", - "string") -add(4, "For that evening-peak winner, what is its mean total_amount? Two decimals.", - f"SELECT round(avg(total_amount), 2) FROM t WHERE {CLEAN} " - f"AND EXTRACT(HOUR FROM tpep_pickup_datetime) BETWEEN 17 AND 19 " - f"GROUP BY \"PULocationID\", \"DOLocationID\" HAVING count(*) >= 200 " - f"ORDER BY avg(total_amount) DESC, \"PULocationID\", \"DOLocationID\" LIMIT 1") -add(4, "Among pairs with at least 500 clean-set trips, how many have a mean total_amount " - "above 50 dollars?", - f"SELECT count(*) FROM ({BUSY}) WHERE fare > 50", "number", 0) -add(4, "Among pairs with at least 500 clean-set trips, what is the mean of their mean " - "fares per mile? Two decimals.", - f"SELECT round(avg(fare / miles), 2) FROM ({BUSY})") -add(4, "Taking the ten pairs with the most clean-set trips, which of those ten has the " - "highest mean total_amount? Answer as PU-DO.", - f"SELECT pu || '-' || do_id FROM (SELECT * FROM ({BUSY}) ORDER BY n DESC, pu, do_id LIMIT 10)" - f" ORDER BY fare DESC, pu, do_id LIMIT 1", "string") - - -def main() -> None: - con = duckdb.connect() - con.execute(f"CREATE VIEW t AS SELECT * FROM read_parquet('{TRIPS}')") - con.execute(f"CREATE VIEW z AS SELECT * FROM read_csv_auto('{ZONES}')") - for q in Q: - value = con.execute(q["sql"]).fetchone()[0] - q["answer"] = float(value) if q["kind"] == "number" else str(value) - out = HERE / "questions.json" - out.write_text(json.dumps(Q, indent=1) + "\n") - print(f"{len(Q)} questions -> {out}") - for q in Q: - print(f" {q['id']} p{q['phase']} {q['answer']}") - - -if __name__ == "__main__": - main() diff --git a/bench/compare.py b/bench/compare.py deleted file mode 100644 index ca9db38..0000000 --- a/bench/compare.py +++ /dev/null @@ -1,63 +0,0 @@ -"""The arms side by side: what each one was, and what it cost. - -local no Hotdata at all — the files on disk, whatever the machine has -bare the Hotdata CLI from a brief, no plugin -plugin_v21 the plugin, v2.1: one database, orientation, definitions, tabular-memory batch -""" - -from __future__ import annotations - -import json -from pathlib import Path - -HERE = Path(__file__).resolve().parent -ARMS = [("local", "No Hotdata"), ("bare", "Hotdata CLI"), ("plugin_v21", "Plugin v2.1")] - - -def pct(x: float) -> str: - return f"{x:.1%}" - - -def main() -> None: - r = json.loads((HERE / "report.json").read_text()) - arms = [(k, label, r[k]) for k, label in ARMS if k in r] - rows = [ - ("sessions", lambda d: str(d["sessions"])), - ("agents", lambda d: " ".join(f"{a['accuracy']:.0%}" for a in d["per_agent"])), - ("correct", lambda d: pct(d["accuracy"])), - (" phase 3", lambda d: pct(d["accuracy_by_phase"].get("3", 0))), - (" phase 4", lambda d: pct(d["accuracy_by_phase"].get("4", 0))), - ("right then wrong", lambda d: str(d["stability"]["right_to_wrong"])), - ("turns / session", lambda d: f"{d['turns_total'] / d['sessions']:.1f}"), - ("tool calls / session", - lambda d: f"{sum(d.get('tools', {}).values()) / d['sessions']:.1f}"), - ("tokens / session", lambda d: f"{sum(d['tokens'].values()) // d['sessions']:,}"), - (" of which output", - lambda d: f"{d['tokens']['output'] // d['sessions']:,}"), - ("tokens / correct", - lambda d: f"{sum(d['tokens'].values()) // max(round(d['accuracy'] * d['questions']), 1):,}"), - ("$ / session", lambda d: f"{d['cost_total'] / d['sessions']:.3f}"), - ("$ / correct", lambda d: f"{d['cost_per_correct']:.3f}"), - ("seconds / session", lambda d: f"{d['wall_mean_s']:.0f}"), - ] - width = max(len(name) for name, _ in rows) + 2 - print(" " * width + "".join(f"{label:>14s}" for _, label, _ in arms)) - for name, get in rows: - print(f"{name:{width}s}" + "".join(f"{get(d):>14s}" for _, _, d in arms)) - - print("\ntools used") - names = sorted({n for _, _, d in arms for n in d.get("tools", {})}) - for n in names: - print(f" {n:{width - 2}s}" + "".join(f"{d.get('tools', {}).get(n, 0):>14d}" - for _, _, d in arms)) - - print("\nqueries against Hotdata (none by design for the first arm)") - for key, label, d in arms: - s = d.get("swarm") or {} - print(f" {label:{width - 2}s}{s.get('queries', 0):>14d}" - f" redundant {s.get('redundant_pct', 0):.0f}%" - f" orientation {s.get('orientation_queries', 0)}") - - -if __name__ == "__main__": - main() diff --git a/bench/distance.json b/bench/distance.json deleted file mode 100644 index 71fe212..0000000 --- a/bench/distance.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "No Hotdata": { - "n": 160, - "median": 427.5, - "p90": 3094, - "max": 6108, - "total": 173398 - }, - "Hotdata CLI": { - "n": 247, - "median": 171, - "p90": 2985, - "max": 10398, - "total": 220899 - }, - "Plugin v2.1": { - "n": 96, - "median": 240.0, - "p90": 375, - "max": 474, - "total": 19775 - } -} diff --git a/bench/orchestrate.sh b/bench/orchestrate.sh deleted file mode 100755 index 15715dd..0000000 --- a/bench/orchestrate.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Run the benchmark: four replicates of each arm, the three arms side by side. -# -# The arms run together in each round so a slow period on the API hits all three -# equally. Sessions inside one agent stay sequential, because memory has to cross a -# session boundary to be worth anything. -set -uo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUT="${1:-$HERE/runs}" -REPLICATES="${REPLICATES:-4}" -SESSIONS="${SESSIONS:-10}" -MODEL="${MODEL:-claude-opus-5}" -TAG="${TAG:-bench}" - -mkdir -p "$OUT" -for r in $(seq 1 "$REPLICATES"); do - echo "=== replicate $r of $REPLICATES ===" - pids=() - for arm in ${ARMS:-bare plugin}; do - python3 "$HERE/runner.py" --arm "$arm" --replicate "$r" --out "$OUT" \ - --sessions "$SESSIONS" --model "$MODEL" --tag "$TAG" \ - > "$OUT/$arm-r$r.log" 2>&1 & - pids+=($!) - done - for pid in "${pids[@]}"; do wait "$pid"; done - for arm in ${ARMS:-bare plugin}; do - tail -1 "$OUT/$arm-r$r.log" - done -done -echo "=== all agents done: $MODEL ===" diff --git a/bench/questions.json b/bench/questions.json deleted file mode 100644 index 5d123ca..0000000 --- a/bench/questions.json +++ /dev/null @@ -1,452 +0,0 @@ -[ - { - "id": "q01", - "phase": 1, - "ask": "How many rows are in the trips table?", - "sql": "SELECT count(*) FROM t", - "kind": "number", - "tol": 0, - "answer": 2964624.0 - }, - { - "id": "q02", - "phase": 1, - "ask": "How many columns does the trips table have?", - "sql": "SELECT count(*) FROM (DESCRIBE SELECT * FROM t)", - "kind": "number", - "tol": 0, - "answer": 19.0 - }, - { - "id": "q03", - "phase": 1, - "ask": "How many trips have a fare_amount of zero or less?", - "sql": "SELECT count(*) FROM t WHERE fare_amount <= 0", - "kind": "number", - "tol": 0, - "answer": 38341.0 - }, - { - "id": "q04", - "phase": 1, - "ask": "How many trips have a tip_amount greater than their fare_amount?", - "sql": "SELECT count(*) FROM t WHERE tip_amount > fare_amount", - "kind": "number", - "tol": 0, - "answer": 41391.0 - }, - { - "id": "q05", - "phase": 1, - "ask": "How many trips have a pickup datetime outside January 2024?", - "sql": "SELECT count(*) FROM t WHERE tpep_pickup_datetime < TIMESTAMP '2024-01-01' OR tpep_pickup_datetime >= TIMESTAMP '2024-02-01'", - "kind": "number", - "tol": 0, - "answer": 18.0 - }, - { - "id": "q06", - "phase": 1, - "ask": "How many trips have a trip_distance of zero?", - "sql": "SELECT count(*) FROM t WHERE trip_distance = 0", - "kind": "number", - "tol": 0, - "answer": 60371.0 - }, - { - "id": "q07", - "phase": 1, - "ask": "How many trips remain after removing rows with a non-positive fare_amount, non-positive total_amount, non-positive trip_distance, a pickup outside January 2024, or a dropoff at or before the pickup? Call this the clean set; every later question means the clean set when it says trips.", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n", - "kind": "number", - "tol": 0, - "answer": 2869585.0 - }, - { - "id": "q08", - "phase": 1, - "ask": "How many distinct PULocationID values appear in the clean set?", - "sql": "SELECT count(DISTINCT \"PULocationID\") FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n", - "kind": "number", - "tol": 0, - "answer": 257.0 - }, - { - "id": "q09", - "phase": 1, - "ask": "How many rows does the taxi zone lookup table have?", - "sql": "SELECT count(*) FROM z", - "kind": "number", - "tol": 0, - "answer": 265.0 - }, - { - "id": "q10", - "phase": 1, - "ask": "Which borough does LocationID 132 belong to?", - "sql": "SELECT \"Borough\" FROM z WHERE \"LocationID\" = 132", - "kind": "string", - "tol": 0.01, - "answer": "Queens" - }, - { - "id": "q11", - "phase": 2, - "ask": "In the clean set, how many trips went from LocationID 132 to 230?", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=132 AND \"DOLocationID\"=230", - "kind": "number", - "tol": 0, - "answer": 5759.0 - }, - { - "id": "q12", - "phase": 2, - "ask": "In the clean set, what is the mean total_amount for trips from LocationID 132 to 230? Give dollars to two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=132 AND \"DOLocationID\"=230", - "kind": "number", - "tol": 0.01, - "answer": 94.13 - }, - { - "id": "q13", - "phase": 2, - "ask": "In the clean set, what is the median trip duration in minutes for trips from LocationID 132 to 230? Give minutes to two decimals.", - "sql": "SELECT round(median(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=132 AND \"DOLocationID\"=230", - "kind": "number", - "tol": 0.01, - "answer": 49.53 - }, - { - "id": "q14", - "phase": 2, - "ask": "In the clean set, how many trips went from LocationID 138 to 161?", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=138 AND \"DOLocationID\"=161", - "kind": "number", - "tol": 0, - "answer": 3257.0 - }, - { - "id": "q15", - "phase": 2, - "ask": "In the clean set, what is the mean total_amount for trips from LocationID 138 to 161? Give dollars to two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=138 AND \"DOLocationID\"=161", - "kind": "number", - "tol": 0.01, - "answer": 73.65 - }, - { - "id": "q16", - "phase": 2, - "ask": "In the clean set, what is the median trip duration in minutes for trips from LocationID 138 to 161? Give minutes to two decimals.", - "sql": "SELECT round(median(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=138 AND \"DOLocationID\"=161", - "kind": "number", - "tol": 0.01, - "answer": 30.17 - }, - { - "id": "q17", - "phase": 2, - "ask": "In the clean set, how many trips went from LocationID 161 to 237?", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=161 AND \"DOLocationID\"=237", - "kind": "number", - "tol": 0, - "answer": 10139.0 - }, - { - "id": "q18", - "phase": 2, - "ask": "In the clean set, what is the mean total_amount for trips from LocationID 161 to 237? Give dollars to two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=161 AND \"DOLocationID\"=237", - "kind": "number", - "tol": 0.01, - "answer": 17.06 - }, - { - "id": "q19", - "phase": 2, - "ask": "In the clean set, what is the median trip duration in minutes for trips from LocationID 161 to 237? Give minutes to two decimals.", - "sql": "SELECT round(median(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=161 AND \"DOLocationID\"=237", - "kind": "number", - "tol": 0.01, - "answer": 8.37 - }, - { - "id": "q20", - "phase": 2, - "ask": "In the clean set, how many trips went from LocationID 237 to 236?", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=237 AND \"DOLocationID\"=236", - "kind": "number", - "tol": 0, - "answer": 21641.0 - }, - { - "id": "q21", - "phase": 2, - "ask": "In the clean set, what is the mean total_amount for trips from LocationID 237 to 236? Give dollars to two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=237 AND \"DOLocationID\"=236", - "kind": "number", - "tol": 0.01, - "answer": 15.74 - }, - { - "id": "q22", - "phase": 2, - "ask": "In the clean set, what is the median trip duration in minutes for trips from LocationID 237 to 236? Give minutes to two decimals.", - "sql": "SELECT round(median(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=237 AND \"DOLocationID\"=236", - "kind": "number", - "tol": 0.01, - "answer": 6.8 - }, - { - "id": "q23", - "phase": 2, - "ask": "In the clean set, how many trips went from LocationID 186 to 79?", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=186 AND \"DOLocationID\"=79", - "kind": "number", - "tol": 0, - "answer": 2631.0 - }, - { - "id": "q24", - "phase": 2, - "ask": "In the clean set, what is the mean total_amount for trips from LocationID 186 to 79? Give dollars to two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=186 AND \"DOLocationID\"=79", - "kind": "number", - "tol": 0.01, - "answer": 23.77 - }, - { - "id": "q25", - "phase": 2, - "ask": "In the clean set, what is the median trip duration in minutes for trips from LocationID 186 to 79? Give minutes to two decimals.", - "sql": "SELECT round(median(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"PULocationID\"=186 AND \"DOLocationID\"=79", - "kind": "number", - "tol": 0.01, - "answer": 15.2 - }, - { - "id": "q26", - "phase": 3, - "ask": "Among pickup-dropoff LocationID pairs in the clean set with at least 500 trips, which pair has the highest mean total_amount? Answer as PU-DO, e.g. 132-230.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY fare DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "138-265" - }, - { - "id": "q27", - "phase": 3, - "ask": "Among pairs with at least 500 trips, which has the lowest mean total_amount? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY fare ASC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "166-151" - }, - { - "id": "q28", - "phase": 3, - "ask": "Among pairs with at least 500 trips, which has the highest mean trip duration in minutes? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY mins DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "68-132" - }, - { - "id": "q29", - "phase": 3, - "ask": "Among pairs with at least 500 trips, which has the highest mean tip as a share of fare_amount? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY tip_rate DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "148-68" - }, - { - "id": "q30", - "phase": 3, - "ask": "How many pickup-dropoff pairs in the clean set have at least 500 trips?", - "sql": "SELECT count(*) FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n)", - "kind": "number", - "tol": 0, - "answer": 1380.0 - }, - { - "id": "q31", - "phase": 3, - "ask": "Which pickup zone name has the most clean-set trips?", - "sql": "SELECT z.\"Zone\" FROM t JOIN z ON t.\"PULocationID\" = z.\"LocationID\" WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n GROUP BY 1 ORDER BY count(*) DESC LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "Midtown Center" - }, - { - "id": "q32", - "phase": 3, - "ask": "Which borough has the most clean-set pickups?", - "sql": "SELECT z.\"Borough\" FROM t JOIN z ON t.\"PULocationID\" = z.\"LocationID\" WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n GROUP BY 1 ORDER BY count(*) DESC LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "Manhattan" - }, - { - "id": "q33", - "phase": 3, - "ask": "What is the mean total_amount of clean-set trips picked up in Manhattan? Two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t JOIN z ON t.\"PULocationID\" = z.\"LocationID\" WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND z.\"Borough\" = 'Manhattan'", - "kind": "number", - "tol": 0.01, - "answer": 22.76 - }, - { - "id": "q34", - "phase": 3, - "ask": "In the clean set, which hour of the day (0-23, by pickup time) has the most trips?", - "sql": "SELECT EXTRACT(HOUR FROM tpep_pickup_datetime) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n GROUP BY 1 ORDER BY count(*) DESC LIMIT 1", - "kind": "number", - "tol": 0, - "answer": 18.0 - }, - { - "id": "q35", - "phase": 3, - "ask": "In the clean set, which hour of the day has the highest mean total_amount?", - "sql": "SELECT EXTRACT(HOUR FROM tpep_pickup_datetime) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n GROUP BY 1 ORDER BY avg(total_amount) DESC LIMIT 1", - "kind": "number", - "tol": 0, - "answer": 5.0 - }, - { - "id": "q36", - "phase": 3, - "ask": "In the clean set, what is the mean fare per mile (total_amount / trip_distance) over all trips? Two decimals.", - "sql": "SELECT round(avg(total_amount / trip_distance), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n", - "kind": "number", - "tol": 0.01, - "answer": 17.49 - }, - { - "id": "q37", - "phase": 3, - "ask": "Among pairs with at least 500 trips, which has the highest mean trip_distance? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY miles DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "233-138" - }, - { - "id": "q38", - "phase": 3, - "ask": "How many clean-set trips were paid by credit card (payment_type = 1)?", - "sql": "SELECT count(*) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND payment_type = 1", - "kind": "number", - "tol": 0, - "answer": 2298339.0 - }, - { - "id": "q39", - "phase": 3, - "ask": "What share of clean-set trips were paid by credit card? Give a fraction to four decimals.", - "sql": "SELECT round(sum(CASE WHEN payment_type = 1 THEN 1 ELSE 0 END)::DOUBLE / count(*), 4) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n", - "kind": "number", - "tol": 0.0001, - "answer": 0.8009 - }, - { - "id": "q40", - "phase": 3, - "ask": "In the clean set, what is the mean Airport_fee over trips where it is greater than zero? Two decimals.", - "sql": "SELECT round(avg(\"Airport_fee\"), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND \"Airport_fee\" > 0", - "kind": "number", - "tol": 0.01, - "answer": 1.75 - }, - { - "id": "q41", - "phase": 4, - "ask": "Define route efficiency as mean total_amount divided by mean duration in minutes. Among pairs with at least 500 clean-set trips, which pair has the highest efficiency? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY fare / mins DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "138-138" - }, - { - "id": "q42", - "phase": 4, - "ask": "Using that same efficiency measure, what is its value for the winning pair? Two decimals.", - "sql": "SELECT round(max(fare / mins), 2) FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n)", - "kind": "number", - "tol": 0.01, - "answer": 3.54 - }, - { - "id": "q43", - "phase": 4, - "ask": "Among pairs with at least 500 clean-set trips and a mean trip_distance of at least 3 miles, which has the highest mean total_amount per mile? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) WHERE miles >= 3 ORDER BY fare / miles DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "138-138" - }, - { - "id": "q44", - "phase": 4, - "ask": "Among pairs with at least 1000 clean-set trips, which has the best combination of high mean total_amount and low mean duration, scored as mean total_amount minus mean duration in minutes? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) WHERE n >= 1000 ORDER BY fare - mins DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "138-265" - }, - { - "id": "q45", - "phase": 4, - "ask": "Name the pickup and dropoff zones of that last winning pair, as ' to '.", - "sql": "SELECT zp.\"Zone\" || ' to ' || zd.\"Zone\" FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) b JOIN z zp ON b.pu = zp.\"LocationID\" JOIN z zd ON b.do_id = zd.\"LocationID\" WHERE b.n >= 1000 ORDER BY b.fare - b.mins DESC, b.pu, b.do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "LaGuardia Airport to Outside of NYC" - }, - { - "id": "q46", - "phase": 4, - "ask": "Restricting to clean-set trips picked up between 17:00 and 19:59 inclusive, and to pairs with at least 200 such trips, which pair has the highest mean total_amount? Answer as PU-DO.", - "sql": "SELECT \"PULocationID\" || '-' || \"DOLocationID\" FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND EXTRACT(HOUR FROM tpep_pickup_datetime) BETWEEN 17 AND 19 GROUP BY 1, \"PULocationID\", \"DOLocationID\" HAVING count(*) >= 200 ORDER BY avg(total_amount) DESC, \"PULocationID\", \"DOLocationID\" LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "138-265" - }, - { - "id": "q47", - "phase": 4, - "ask": "For that evening-peak winner, what is its mean total_amount? Two decimals.", - "sql": "SELECT round(avg(total_amount), 2) FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n AND EXTRACT(HOUR FROM tpep_pickup_datetime) BETWEEN 17 AND 19 GROUP BY \"PULocationID\", \"DOLocationID\" HAVING count(*) >= 200 ORDER BY avg(total_amount) DESC, \"PULocationID\", \"DOLocationID\" LIMIT 1", - "kind": "number", - "tol": 0.01, - "answer": 147.1 - }, - { - "id": "q48", - "phase": 4, - "ask": "Among pairs with at least 500 clean-set trips, how many have a mean total_amount above 50 dollars?", - "sql": "SELECT count(*) FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) WHERE fare > 50", - "kind": "number", - "tol": 0, - "answer": 157.0 - }, - { - "id": "q49", - "phase": 4, - "ask": "Among pairs with at least 500 clean-set trips, what is the mean of their mean fares per mile? Two decimals.", - "sql": "SELECT round(avg(fare / miles), 2) FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n)", - "kind": "number", - "tol": 0.01, - "answer": 11.59 - }, - { - "id": "q50", - "phase": 4, - "ask": "Taking the ten pairs with the most clean-set trips, which of those ten has the highest mean total_amount? Answer as PU-DO.", - "sql": "SELECT pu || '-' || do_id FROM (SELECT * FROM (\n SELECT \"PULocationID\" AS pu, \"DOLocationID\" AS do_id, count(*) AS n,\n avg(total_amount) AS fare,\n avg(EXTRACT(EPOCH FROM (tpep_dropoff_datetime - tpep_pickup_datetime)) / 60.0) AS mins,\n avg(trip_distance) AS miles,\n avg(tip_amount / nullif(fare_amount, 0)) AS tip_rate\n FROM t WHERE \n fare_amount > 0 AND total_amount > 0 AND trip_distance > 0\n AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01'\n AND tpep_pickup_datetime < TIMESTAMP '2024-02-01'\n AND tpep_dropoff_datetime > tpep_pickup_datetime\n\n GROUP BY 1, 2 HAVING count(*) >= 500\n) ORDER BY n DESC, pu, do_id LIMIT 10) ORDER BY fare DESC, pu, do_id LIMIT 1", - "kind": "string", - "tol": 0.01, - "answer": "161-236" - } -] diff --git a/bench/redundancy.py b/bench/redundancy.py deleted file mode 100644 index 3ed2fac..0000000 --- a/bench/redundancy.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Swarm metrics read from the transcripts a run leaves behind. - -For each arm: the share of data queries some agent had already run, how many were -orientation (`LIMIT 1/2`, information_schema), and how many answers carry the -invented-filter signature (wrong, but within 5% of the truth on the route questions). -""" - -from __future__ import annotations - -import glob -import json -import re -from collections import defaultdict -from pathlib import Path - -HERE = Path(__file__).resolve().parent -PROJECTS = Path.home() / ".claude" / "projects" -SCHEMA = re.compile(r"information_schema|\bdescribe\b|\bshow\s+(tables|columns)", re.I) -PEEK = re.compile(r"\bselect\s+\*|\blimit\s+[1-5]\b", re.I) -RANKING = re.compile(r"\border\s+by\b|\bgroup\s+by\b|\b(count|sum|avg|min|max|median)\s*\(", re.I) - - -def is_orientation(sql: str) -> bool: - """A look at the shape of the data, as opposed to a question about it. - - Schema reads always count. A `SELECT *` or a tiny LIMIT counts only when the query - is not ranking or aggregating — `ORDER BY fare DESC LIMIT 1` is an answer, not a peek. - """ - if SCHEMA.search(sql): - return True - return bool(PEEK.search(sql)) and not RANKING.search(sql) -# A shell double-quoted string: anything but a quote or backslash, or a backslash and -# whatever follows it. SQL in these commands carries \"PULocationID\" inside the quotes. -DQ = r'"((?:[^"\\]|\\.)*)"' -QUERY = re.compile(r"(?:hotdata query|tabular-memory query)\s+" + DQ, re.S) -BATCH_ARG = re.compile(DQ, re.S) -LABELLED = re.compile(r"^\s*(?:[A-Za-z_][A-Za-z0-9_]*\s*:\s*)?((?:select|with)\b.*)$", re.I | re.S) - - -def unescape(text: str) -> str: - return text.replace('\\"', '"').replace("\\\\", "\\") -BATCH = re.compile(r"\btabular-memory batch\b") -BATCH_HEREDOC = re.compile(r"tabular-memory batch\s*<<\s*'?(\w+)'?\n(.*?)\n\1", re.S) -LABEL_LINE = re.compile(r"^\s*--\s*\w+\s*$") - - -def norm(sql: str) -> str: - return re.sub(r"\s+", " ", sql.strip().lower()).rstrip(";") - - -def transcripts(runs: Path, arm: str) -> list[Path]: - """Transcript files for every agent of `arm` under `runs`. - - Claude Code names a project's folder from its working directory, with every `/` - turned into `-`, so the agent's work dir is the key. - """ - found = [] - for agent in sorted(runs.glob(f"{arm}-r*")): - # Only the sessions this run recorded. A rerun in the same work directory shares - # the project folder with whatever ran there before, and those must not count. - ids = set() - for result in agent.glob("s*.json"): - try: - ids.add(json.loads(result.read_text()).get("session_id")) - except (OSError, json.JSONDecodeError): - pass - name = str((agent / "work").resolve()).replace("/", "-") - found.extend(path for path in (PROJECTS / name).glob("*.jsonl") if path.stem in ids) - return found - - -def queries_of(path: Path) -> list[str]: - out = [] - for line in open(path): - try: - d = json.loads(line) - except json.JSONDecodeError: - continue - if d.get("type") != "assistant": - continue - for c in d.get("message", {}).get("content", []): - if c.get("type") == "tool_use" and c.get("name") == "Bash": - cmd = (c.get("input") or {}).get("command", "") - for m in QUERY.finditer(cmd): - out.append(norm(unescape(m.group(1)))) - if BATCH.search(cmd): - for m in BATCH_HEREDOC.finditer(cmd): - for stmt in m.group(2).split(";"): - body = "\n".join(l for l in stmt.splitlines() if not LABEL_LINE.match(l)).strip() - if body: - out.append(norm(body)) - head = cmd.split("<<", 1)[0] if "<<" in cmd else cmd - head = BATCH.split(head, 1)[1] - for m in BATCH_ARG.finditer(head): - body = LABELLED.match(unescape(m.group(1))) - if body: - out.append(norm(body.group(1))) - return out - - -def swarm_metrics(runs: Path, arm: str) -> dict: - all_q, per_agent = [], defaultdict(list) - for path in transcripts(runs, arm): - agent = path.parent.name - qs = queries_of(path) - per_agent[agent].extend(qs) - all_q.extend(qs) - distinct = len(set(all_q)) - orient = sum(1 for q in all_q if is_orientation(q)) - return { - "queries": len(all_q), - "distinct": distinct, - "redundant_pct": round(100 * (1 - distinct / len(all_q)), 1) if all_q else 0.0, - "orientation_queries": orient, - "agents": len(per_agent), - } - - -def invented_filter_errors(answers: list[dict]) -> int: - """Wrong answers on the per-route questions that sit within 5% of the truth.""" - count = 0 - for a in answers: - if a["correct"] or a["phase"] != 2: - continue - try: - given, truth = float(str(a["given"]).replace(",", "")), float(a["answer"]) - except (TypeError, ValueError): - continue - if truth and abs(given - truth) / abs(truth) < 0.05: - count += 1 - return count - - -def tool_use(runs: Path, arm: str) -> dict: - """How the arm worked: tool calls by name, and the data files it opened.""" - import collections - tools = collections.Counter() - for path in transcripts(runs, arm): - for line in open(path): - try: - d = json.loads(line) - except json.JSONDecodeError: - continue - if d.get("type") != "assistant": - continue - for c in d.get("message", {}).get("content", []): - if c.get("type") == "tool_use": - tools[c["name"]] += 1 - return dict(tools) diff --git a/bench/runner.py b/bench/runner.py deleted file mode 100644 index feb7510..0000000 --- a/bench/runner.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Run one benchmark agent: ten sessions of five questions, in one arm. - -Every session is a separate `claude -p` process, so memory has to cross a session -boundary to be worth anything. The arms differ only in how memory is kept; the data, -the questions, and their order are identical. - -Each session's JSON result is written to //sNN.json, and the graded answers -to //answers.jsonl. -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import shutil -import subprocess -import sys -import time -from pathlib import Path - -HERE = Path(__file__).resolve().parent -# The benchmark database (taxi.public.trips and zones, built by build_bank.py) and the -# workspace it lives in. Set these to your own before a run. -DB = os.environ.get("BENCH_DB") or sys.exit("set BENCH_DB to the id of the taxi database") -MEMORY_DB = os.environ.get("BENCH_MEMORY_DB", "") -WORKSPACE = os.environ.get("BENCH_WORKSPACE") or sys.exit("set BENCH_WORKSPACE to the workspace id") -PER_SESSION = 5 - -BRIEF = """You are answering questions about New York yellow taxi trips for January 2024. - -The data is in a Hotdata managed database. Query it with: - - hotdata query "" -d {db} -o csv - -Two tables: `taxi.public.trips` (one row per trip) and `taxi.public.zones` (the -LocationID lookup, with Borough and Zone). Column names are case-sensitive; wrap the -uppercase ones in double quotes, for example "PULocationID". - -Answer each question below. Work however you like. When you are done, print one final -line and nothing after it: - -ANSWERS {{"q01": , "q02": , ...}} - -A value is a bare number (no dollar sign, no thousands separator) or a string, matching -what the question asks for. Use the question ids exactly as given. - -You will be asked more questions about this same data in later sessions, and you will -not remember this one. Before you finish, record anything you had to work out that -would save you the work next time: a filter you had to define, a quirk of the schema or -the SQL dialect, a figure you may be asked for again. Use whatever memory this session -gives you. If it gives you none, skip this. - -Questions: -{questions} -""" - - -DATA_DIR = "/private/tmp/claude-501/-Users-eddie-Code-claude-hotdata-plugin/6bc8e054-526c-4223-8763-8182b4f473d2/scratchpad/bench" - -LOCAL_BRIEF = """You are answering questions about New York yellow taxi trips for January 2024. - -The data is in two files: - - {data}/yellow_2024-01.parquet one row per trip - {data}/taxi_zone_lookup.csv the LocationID lookup, with Borough and Zone - -Answer each question below. Work however you like, with whatever is on this machine. - -When you are done, print one final line and nothing after it: - -ANSWERS {{"q01": , "q02": , ...}} - -A value is a bare number (no dollar sign, no thousands separator) or a string, matching -what the question asks for. Use the question ids exactly as given. - -Questions: -{questions} -""" - -PLUGIN_BRIEF = """You are answering questions about New York yellow taxi trips for January 2024. - -The project's data is in Hotdata and the `tabular-memory` command is on your PATH. Your session -started with the database, its tables, a sample, any definitions and the SQL dialect. -Use `tabular-memory query ""`, or `tabular-memory batch "label: " "label: " ...` to ask several -questions in one call. In SQL, {{{{name}}}} expands to a definition; `tabular-memory defs` lists them. -If a question names a set (such as "the clean set") and no definition exists, define it -from the question's own words with `tabular-memory define` before answering, so later questions use -the same one. - -Answer each question below. When you are done, print one final line and nothing after it: - -ANSWERS {{"q01": , "q02": , ...}} - -A value is a bare number (no dollar sign, no thousands separator) or a string, matching -what the question asks for. Use the question ids exactly as given. - -Questions: -{questions} -""" - - -def arm_settings(arm: str, home: Path) -> dict: - """Return the settings.json contents for `arm`.""" - base = { - "permissions": { - "allow": ["Bash(hotdata query:*)", "Bash(hotdata databases:*)", - "Bash(tabular-memory:*)"], - }, - "autoMemoryDirectory": str(home / "memory"), - } - if arm == "local": - # Claude as it would be if Hotdata did not exist: the files are on disk, every - # tool on the machine is fair game, and the CLI is denied so a stray mention in - # the skill listing cannot turn into a call. - base["permissions"] = {"allow": ["Bash", "Read", "Write", "Edit", "Glob", "Grep"], - "deny": ["Bash(hotdata:*)", "Bash(tabular-memory:*)"]} - base["autoMemoryEnabled"] = False - base["enabledPlugins"] = {"hotdata-tabular-memory@hotdata": False} - elif arm in ("bare", "plugin"): - base["autoMemoryEnabled"] = False - base["enabledPlugins"] = {"hotdata-tabular-memory@hotdata": False} - elif arm == "markdown": - base["autoMemoryEnabled"] = True - base["enabledPlugins"] = {"hotdata-tabular-memory@hotdata": False} - elif arm == "hotdata": - base["autoMemoryEnabled"] = False - base["enabledPlugins"] = {"hotdata-tabular-memory@hotdata": True} - elif arm == "none": - base["autoMemoryEnabled"] = False - base["enabledPlugins"] = {"hotdata-tabular-memory@hotdata": False} - else: - raise SystemExit(f"unknown arm {arm!r}") - return base - - -PLUGIN_DIR = str(HERE.parent) - - -def plugin_env(arm: str, scope: str) -> dict: - """Return the plugin option environment for `arm`.""" - if arm == "plugin": - return {"CLAUDE_PLUGIN_OPTION_DATABASE": DB, - "CLAUDE_PLUGIN_OPTION_PROJECT": "taxi", - "CLAUDE_PLUGIN_OPTION_READ_GUARD": "on"} - if arm != "hotdata": - return {} - return { - "CLAUDE_PLUGIN_OPTION_STORE": "hotdata", - "CLAUDE_PLUGIN_OPTION_MODE": "standalone", - "CLAUDE_PLUGIN_OPTION_DATABASE_ID": MEMORY_DB, - "CLAUDE_PLUGIN_OPTION_WORKSPACE_ID": WORKSPACE, - "CLAUDE_PLUGIN_OPTION_SCOPE_PREFIX": scope, - # Pin the project. The work directory is not a repository, so without this the - # scope would follow the shell and split when the agent changes directory. - "CLAUDE_PLUGIN_OPTION_PROJECT": "taxi", - # Attach memory to the database being queried, so a session can read both in - # one statement instead of shelling out to the memory command. - "CLAUDE_PLUGIN_OPTION_WORK_DATABASE": DB, - "CLAUDE_PLUGIN_OPTION_EMBEDDER": "hashed", - } - - -ANSWER_LINE = re.compile(r"ANSWERS\s*(\{.*\})", re.DOTALL) - - -def parse_answers(text: str) -> dict: - match = None - for match in ANSWER_LINE.finditer(text): - pass - if match is None: - return {} - raw = match.group(1).strip() - # A model that takes the brief literally can wrap the object in a second pair of - # braces. Read what it meant rather than scoring the session zero. - candidates = [raw] - if raw.startswith("{{") and raw.endswith("}}"): - candidates.insert(0, raw[1:-1]) - for text in candidates: - for end in range(len(text), 0, -1): - try: - found = json.loads(text[:end]) - except json.JSONDecodeError: - continue - if isinstance(found, dict): - return found - return {} - - -def grade(question: dict, given) -> bool: - if given is None: - return False - if question["kind"] == "number": - try: - value = float(str(given).replace("$", "").replace(",", "").strip()) - except (TypeError, ValueError): - return False - return abs(value - question["answer"]) <= max(question["tol"], 0.005) - return str(given).strip().lower() == str(question["answer"]).strip().lower() - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--arm", required=True, - choices=["local", "bare", "plugin", "none", "markdown", "hotdata"]) - parser.add_argument("--replicate", type=int, required=True) - parser.add_argument("--out", type=Path, default=HERE / "runs") - parser.add_argument("--sessions", type=int, default=20) - parser.add_argument("--model", default="claude-opus-5") - parser.add_argument("--plugin-dir", default="", - help="load the plugin from this directory for the run") - parser.add_argument("--tag", default="bench", - help="run label; keeps one run's memory out of the next one's") - args = parser.parse_args() - - agent = f"{args.arm}-r{args.replicate}" - root = args.out.resolve() / agent - if root.exists(): - shutil.rmtree(root) - (root / "home").mkdir(parents=True) - home = root / "home" - work = root / "work" - work.mkdir() - - cli_version = subprocess.run(["claude", "--version"], capture_output=True, - text=True).stdout.strip() - questions = json.loads((HERE / "questions.json").read_text()) - scope = f"{args.tag}-{agent}" - - settings = home / "settings.json" - settings.write_text(json.dumps(arm_settings(args.arm, home), indent=1)) - - env = {**os.environ, **plugin_env(args.arm, scope)} - env["HOTDATA_API_KEY"] = (Path.home() / ".hotdata/api-key").read_text().strip() - env["CLAUDE_CODE_PROJECT_DIR_NAME"] = scope - # A session started by another Claude Code session has auto memory off by default, - # and `autoMemoryEnabled` cannot turn it back on there. Only "0" forces it on, so - # every arm states what it wants rather than inheriting it. - env["CLAUDE_CODE_DISABLE_AUTO_MEMORY"] = "0" if args.arm == "markdown" else "1" - if args.arm != "local": - env["PATH"] = f"{PLUGIN_DIR}/bin:" + env.get("PATH", "") - # One plugin data dir per run, shared by every agent in it: a definition one agent - # makes reaches the next one's orientation, which is the behaviour under test. - env["CLAUDE_PLUGIN_DATA"] = str(args.out.resolve() / "plugin-data") - - answers_path = root / "answers.jsonl" - graded = [] - per_pass = (len(questions) + PER_SESSION - 1) // PER_SESSION - for index in range(args.sessions): - # Pass 2 repeats pass 1's questions in the same order. An agent whose memory - # works should answer them without querying the data again. - slot = index % per_pass - pass_no = index // per_pass + 1 - batch = questions[slot * PER_SESSION : (slot + 1) * PER_SESSION] - if not batch: - break - listing = "\n".join(f"{q['id']}. {q['ask']}" for q in batch) - brief = {"plugin": PLUGIN_BRIEF, "local": LOCAL_BRIEF}.get(args.arm, BRIEF) - prompt = brief.format(db=DB, data=DATA_DIR, questions=listing) - started = time.time() - proc = subprocess.run( - ["claude", "-p", prompt, "--output-format", "json", - "--model", args.model, "--settings", str(settings), - "--permission-mode", "acceptEdits", - *(["--plugin-dir", args.plugin_dir or PLUGIN_DIR] - if (args.plugin_dir or args.arm == "plugin") else [])], - capture_output=True, text=True, cwd=work, env=env, timeout=3600, - ) - wall = time.time() - started - try: - result = json.loads(proc.stdout) - except json.JSONDecodeError: - result = {"is_error": True, "result": proc.stdout[-2000:], - "stderr": proc.stderr[-2000:]} - result["wall_seconds"] = wall - result["cli_version"] = cli_version - result["session_index"] = index - result["pass"] = pass_no - result["question_ids"] = [q["id"] for q in batch] - (root / f"s{index:02d}.json").write_text(json.dumps(result, indent=1)) - - given = parse_answers(result.get("result", "") or "") - for q in batch: - ok = grade(q, given.get(q["id"])) - graded.append(ok) - with answers_path.open("a") as handle: - handle.write(json.dumps({ - "session": index, "pass": pass_no, "id": q["id"], - "phase": q["phase"], - "given": given.get(q["id"]), "answer": q["answer"], "correct": ok, - }) + "\n") - score = sum(1 for q in batch if grade(q, given.get(q["id"]))) - print(f"{agent} s{index:02d} p{pass_no} {score}/{len(batch)} " - f"${result.get('total_cost_usd', 0):.2f} {wall:.0f}s " - f"turns={result.get('num_turns', '?')}", flush=True) - - print(f"{agent} DONE {sum(graded)}/{len(graded)} correct", flush=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bench/validate_bank.py b/bench/validate_bank.py deleted file mode 100644 index 9a2ef12..0000000 --- a/bench/validate_bank.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Run every question's ground-truth SQL against Hotdata and compare with DuckDB. - -The agents query Hotdata, so a truth that only DuckDB agrees with would mark a correct -answer wrong. Any mismatch here is a bug in the bank, not in the agent. -""" - -from __future__ import annotations - -import json -import os -import subprocess -import sys -from pathlib import Path - -HERE = Path(__file__).resolve().parent -DB = os.environ.get("BENCH_DB") or sys.exit("set BENCH_DB to the id of the taxi database") -PREFIX = ("WITH t AS (SELECT * FROM taxi.public.trips), " - "z AS (SELECT * FROM taxi.public.zones) ") - - -def ask(sql: str) -> str: - out = subprocess.run( - ["hotdata", "query", PREFIX + sql, "-d", DB, "-o", "csv"], - capture_output=True, text=True, timeout=600, - ) - rows = [r for r in out.stdout.strip().splitlines() if r] - if len(rows) < 2: - return f"ERROR {out.stderr.strip()[:120] or out.stdout.strip()[:120]}" - return rows[1].strip().strip('"') - - -def main() -> None: - questions = json.loads((HERE / "questions.json").read_text()) - bad = [] - for q in questions: - if "DESCRIBE" in q["sql"]: - print(f" {q['id']} skipped (DESCRIBE)") - continue - got = ask(q["sql"]) - if q["kind"] == "number": - try: - ok = abs(float(got) - q["answer"]) <= max(q["tol"], 0.005) - except ValueError: - ok = False - else: - ok = got == q["answer"] - print(f" {q['id']} {'ok ' if ok else 'MISMATCH'} duckdb={q['answer']} hotdata={got}") - if not ok: - bad.append(q["id"]) - print(f"\n{len(questions) - len(bad) - 1}/{len(questions) - 1} agree" - + (f"; mismatched: {', '.join(bad)}" if bad else "")) - - -if __name__ == "__main__": - main() diff --git a/scripts/hotdata.py b/scripts/hotdata.py index ac38221..6389c0d 100644 --- a/scripts/hotdata.py +++ b/scripts/hotdata.py @@ -47,7 +47,8 @@ def run(args: Sequence[str], timeout: int = TIMEOUT) -> str: ) except FileNotFoundError: raise HotdataError("the `hotdata` CLI is not on PATH: install it with " - "`curl -fsSL https://hotdata.dev/install | sh`, then " + "`brew install hotdata-dev/tap/cli` (other ways: " + "https://github.com/hotdata-dev/hotdata-cli#install), then " "`hotdata auth login`") from None except subprocess.SubprocessError as error: raise HotdataError(str(error)) from None diff --git a/scripts/setup.py b/scripts/setup.py index bdc0124..61a1e9c 100644 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -17,7 +17,9 @@ import hotdata import project -INSTALL = "curl -fsSL https://hotdata.dev/install | sh" +# Advice only: the plugin never installs the CLI itself. +INSTALL = "brew install hotdata-dev/tap/cli" +INSTALL_GUIDE = "https://github.com/hotdata-dev/hotdata-cli#install" LOGIN = "hotdata auth login" SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", ".tox", "dist", "build", "target", ".cache"} @@ -54,7 +56,7 @@ def logged_in(timeout: int = 5) -> tuple[bool | None, str]: def problem() -> tuple[str, bool]: """Return (the one thing stopping the plugin, whether the check was conclusive).""" if not installed(): - return f"the `hotdata` CLI is not installed. Install it with `{INSTALL}`, then run `{LOGIN}`.", True + return f"the `hotdata` CLI is not installed. Install it with `{INSTALL}` (other ways: {INSTALL_GUIDE}), then run `{LOGIN}`.", True ok, _ = logged_in() if ok is False: return f"the `hotdata` CLI is not logged in. Type `! {LOGIN}` (it opens a browser).", True diff --git a/swarm/analyze.py b/swarm/analyze.py deleted file mode 100644 index 9d61f3d..0000000 --- a/swarm/analyze.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Summarise swarm runs: cost, accuracy, and what happened to the classification work. - -For every run directory under runs*/ it reports: - - cost dollars, tokens, turns and seconds per task, and the cost of the first - and last third of tasks — falling cost is the shared store paying off - accuracy answers within tolerance, and recorded labels that match the truth - reasoning label rows recorded per distinct complaint: 1.0 means nothing was - classified twice - agreement complaints two agents labelled differently - integrity label rows that could not be read, or name no known category - - python3 analyze.py # every run - python3 analyze.py runs-pilot # one directory of runs -""" - -from __future__ import annotations - -import json -import re -import statistics -import sys -from collections import defaultdict -from pathlib import Path - -from build_data import CATEGORIES - -HERE = Path(__file__).resolve().parent -BENCH = Path.home() / "swarm-bench" -PROJECTS = Path.home() / ".claude" / "projects" -# Files only the harness may read, and the shared scratch space agents must not use. -SPILL = re.compile(r"/private/tmp/claude-\d+/[^\s'\"]*") -FORBIDDEN = ("truth.jsonl", "complaint_specs", "checks.jsonl", "complaints_raw", "/tmp/") - - -def leaks(session: dict) -> list[str]: - """Forbidden paths this session's tool calls mention.""" - sid = session.get("session_id") - found = set() - for path in PROJECTS.glob(f"*/{sid}.jsonl") if sid else []: - for line in path.read_text().splitlines(): - e = json.loads(line) - if e.get("type") != "assistant": - continue - for c in e["message"].get("content", []): - if c.get("type") == "tool_use": - # Claude Code saves a large tool result under /private/tmp/claude-/ - # in a directory of the session's own; reading it back is not a leak. - text = SPILL.sub("", json.dumps(c["input"])) - found.update(f for f in FORBIDDEN if f in text) - return sorted(found) - - -def load_jsonl(path: Path) -> list[dict]: - return [json.loads(l) for l in path.read_text().splitlines()] if path.exists() else [] - - -def summarise(run: Path, truth: dict, tasks: dict) -> dict: - meta = json.loads((run / "run.json").read_text()) - sessions = sorted((json.loads(p.read_text()) for p in (run / "sessions").glob("t*.json")), - key=lambda s: s["task"]) - order = {t: i for i, t in enumerate(meta["tasks"])} - sessions.sort(key=lambda s: order[s["task"]]) - - def tokens(s: dict) -> int: - u = s.get("usage") or {} - return sum(u.get(k, 0) for k in ("input_tokens", "output_tokens", - "cache_read_input_tokens", "cache_creation_input_tokens")) - - # A session that failed before working (a spend limit, a crash) is not a cheap - # task. It is left out of every average and counted under `errors`. - failed = [s for s in sessions if s.get("is_error")] - sessions = [s for s in sessions if not s.get("is_error")] - costs = [s.get("total_cost_usd") or 0 for s in sessions] - third = max(1, len(sessions) // 3) - labels = load_jsonl(run / "labels.jsonl") - by_complaint: dict[int, set] = defaultdict(set) - for l in labels: - by_complaint[l["complaint_id"]].add(l["category"]) - known = [l for l in labels if l["category"] in CATEGORIES] - scored = [l for l in known if l["complaint_id"] in truth] - needed = {c for t in meta["tasks"] for c in tasks[t]["needs"]} - models = sorted({m for s in sessions for m in (s.get("modelUsage") or {}) - if "haiku" not in m or "haiku" in meta["model"]}) - - return { - "run": run.name, "arm": meta["arm"], "concurrency": meta["concurrency"], - "tasks": len(sessions), "models": models, - "cli": sorted({s.get("cli_version", "") for s in sessions}), - "correct": sum(s.get("correct", False) for s in sessions) / max(1, len(sessions)), - "cost_task": statistics.mean(costs) if costs else 0, - "cost_first_third": statistics.mean(costs[:third]) if costs else 0, - "cost_last_third": statistics.mean(costs[-third:]) if costs else 0, - "tokens_task": statistics.mean(map(tokens, sessions)) if sessions else 0, - "turns_task": statistics.mean(s.get("num_turns") or 0 for s in sessions) if sessions else 0, - "secs_task": statistics.mean(s["end"] - s["start"] for s in sessions) if sessions else 0, - "wall_run": max((s["end"] for s in sessions), default=0), - "label_rows": len(labels), - "distinct_labelled": len(by_complaint), - "rows_per_complaint": len(labels) / max(1, len(by_complaint)), - "needed_distinct": len(needed), - "label_accuracy": sum(truth[l["complaint_id"]] == l["category"] for l in scored) - / max(1, len(scored)), - "disagreements": sum(len(c) > 1 for c in by_complaint.values()), - "unknown_category": len(labels) - len(known), - "unreadable": len(json.loads((run / "unreadable.json").read_text())) - if (run / "unreadable.json").exists() else 0, - "errors": len(failed), - "leaks": {s["task"]: l for s in sessions if (l := leaks(s))}, - } - - -def main() -> None: - roots = [BENCH / a for a in sys.argv[1:]] or sorted(BENCH.glob("runs*")) - truth = {t["complaint_id"]: t["category"] for t in load_jsonl(HERE / "data" / "truth.jsonl")} - tasks = {t["id"]: t for t in json.loads((HERE / "tasks.json").read_text())} - rows = [summarise(run, truth, tasks) - for root in roots for run in sorted(root.iterdir()) - if (run / "run.json").exists() and (run / "sessions").exists()] - (HERE / "report.json").write_text(json.dumps(rows, indent=1) + "\n") - - cols = [("run", "{:<22}"), ("correct", "{:>7.0%}"), ("cost_task", "${:>5.2f}"), - ("cost_first_third", "${:>5.2f}"), ("cost_last_third", "${:>5.2f}"), - ("turns_task", "{:>5.1f}"), ("secs_task", "{:>5.0f}"), - ("rows_per_complaint", "{:>5.2f}"), ("label_accuracy", "{:>6.1%}"), - ("disagreements", "{:>5}"), ("unreadable", "{:>5}"), ("errors", "{:>4}")] - print(" ".join(c for c, _ in cols)) - for r in rows: - print(" ".join(f.format(r[c]) for c, f in cols)) - flags = [] - if len(r["cli"]) > 1: - flags.append("MIXED CLI") - if len(r["models"]) != 1: - flags.append(f"MODELS {r['models']}") - if r["leaks"]: - flags.append(f"LEAKS {r['leaks']}") - if flags: - print(" " + "; ".join(flags)) - - -if __name__ == "__main__": - main() diff --git a/swarm/build_data.py b/swarm/build_data.py deleted file mode 100644 index 9dc921c..0000000 --- a/swarm/build_data.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Build the swarm benchmark's data: trips with a key, and which trips get a complaint. - -The trips are the January 2024 TLC file with `trip_id` added (its row number in the -file), so a complaint can name the trip it is about and a label can be joined back. - -Each complaint is assigned its true category here, before any text exists. The -category depends on the trip — airport rides draw fare disputes, late-night rides -draw safety complaints — so the share of each category differs by zone and time, -and a question about one slice has a different answer from a question about another. - -Run once: python3 build_data.py -> data/trips.parquet, data/complaint_specs.jsonl -""" - -from __future__ import annotations - -import json -import random -from pathlib import Path - -import duckdb - -HERE = Path(__file__).resolve().parent -DATA = HERE / "data" -SOURCE = DATA / "yellow_tripdata_2024-01.parquet" -ZONES = DATA / "taxi_zone_lookup.csv" -TRIPS = DATA / "trips.parquet" -SPECS = DATA / "complaint_specs.jsonl" - -CATEGORIES = ["fare_dispute", "detour", "driver_conduct", "vehicle_condition", - "lost_item", "safety"] - -AIRPORTS = (1, 132, 138) # Newark, JFK, LaGuardia -MIDTOWN = (100, 161, 162, 163, 164, 170, 186, 230, 234, 48) - -CLEAN = """ - fare_amount > 0 AND total_amount > 0 AND trip_distance > 0 - AND tpep_pickup_datetime >= TIMESTAMP '2024-01-01' - AND tpep_pickup_datetime < TIMESTAMP '2024-02-01' - AND tpep_dropoff_datetime > tpep_pickup_datetime -""" - -# How many complaints each slice gets. Tasks filter on these slices, so they overlap. -STRATA = { - "airport": (900, f'("PULocationID" IN {AIRPORTS} OR "DOLocationID" IN {AIRPORTS})'), - "midtown": (700, f'"PULocationID" IN {MIDTOWN}'), - "night": (400, "(EXTRACT(HOUR FROM tpep_pickup_datetime) >= 22 " - "OR EXTRACT(HOUR FROM tpep_pickup_datetime) < 5)"), - "elsewhere": (400, "TRUE"), -} - - -def weights(trip: dict) -> list[float]: - """The odds of each category for this trip.""" - w = dict.fromkeys(CATEGORIES, 1.0) - airport = trip["pu"] in AIRPORTS or trip["do"] in AIRPORTS - hour = trip["pickup"].hour - if airport: - w["fare_dispute"] *= 3 - w["lost_item"] *= 2 - if hour >= 22 or hour < 5: - w["safety"] *= 3 - w["driver_conduct"] *= 2 - if trip["miles"] > 8 and not airport: - w["detour"] *= 3 - if 15 <= trip["pickup"].day <= 21: # the week of the cold snap - w["vehicle_condition"] *= 2 - return [w[c] for c in CATEGORIES] - - -def main() -> None: - con = duckdb.connect() - con.execute(f""" - COPY (SELECT file_row_number + 1 AS trip_id, * EXCLUDE (file_row_number) - FROM read_parquet('{SOURCE}', file_row_number = true) - ORDER BY trip_id) - TO '{TRIPS}' (FORMAT parquet)""") - con.execute(f"CREATE VIEW t AS SELECT * FROM read_parquet('{TRIPS}')") - con.execute(f"CREATE VIEW z AS SELECT * FROM read_csv_auto('{ZONES}')") - - taken: set[int] = set() - picked: list[tuple[str, dict]] = [] - for stratum, (n, where) in STRATA.items(): - # hash() orders deterministically, so a rebuild picks the same trips. - rows = con.execute(f""" - SELECT t.trip_id, t."PULocationID", t."DOLocationID", pz."Zone", dz."Zone", - t.tpep_pickup_datetime, t.trip_distance, t.fare_amount, t.total_amount, - t.tip_amount, t.passenger_count, t.payment_type - FROM t JOIN z pz ON pz."LocationID" = t."PULocationID" - JOIN z dz ON dz."LocationID" = t."DOLocationID" - WHERE {CLEAN} AND {where} - ORDER BY hash(t.trip_id) LIMIT {n + len(taken)}""").fetchall() - fresh = [r for r in rows if r[0] not in taken][:n] - for r in fresh: - taken.add(r[0]) - picked.append((stratum, { - "trip_id": r[0], "pu": r[1], "do": r[2], "pickup_zone": r[3], - "dropoff_zone": r[4], "pickup": r[5], "miles": r[6], "fare": r[7], - "total": r[8], "tip": r[9], "passengers": r[10], "card": r[11] == 1})) - - rng = random.Random(42) - with SPECS.open("w") as out: - for i, (stratum, trip) in enumerate(picked, 1): - category = rng.choices(CATEGORIES, weights(trip))[0] - out.write(json.dumps({ - "complaint_id": i, "category": category, "stratum": stratum, - **{k: v for k, v in trip.items() if k != "pickup"}, - "pickup": trip["pickup"].isoformat(sep=" ")}) + "\n") - - counts = dict.fromkeys(CATEGORIES, 0) - for line in SPECS.read_text().splitlines(): - counts[json.loads(line)["category"]] += 1 - print(f"{con.execute('SELECT count(*) FROM t').fetchone()[0]:,} trips -> {TRIPS.name}") - print(f"{len(picked)} complaint specs -> {SPECS.name}") - for c, n in counts.items(): - print(f" {c:18} {n}") - - -if __name__ == "__main__": - main() diff --git a/swarm/build_tasks.py b/swarm/build_tasks.py deleted file mode 100644 index 86b0be1..0000000 --- a/swarm/build_tasks.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Build the swarm task bank and its truth. - -Each task asks about one slice of trips — JFK pickups, Midtown in the cold-snap week, -rides at night — and needs the complaints on those trips classified to answer. Slices -overlap, so tasks share classification work: that overlap is what a shared label table -can save and an isolated agent cannot. - -A task is kept only if its answer survives a careless reader: with 5% of labels -flipped at random, the answer must still be within tolerance in 95% of 200 trials. -Otherwise a task would grade luck, not understanding. - - python3 build_tasks.py -> tasks.json -""" - -from __future__ import annotations - -import json -import random -import statistics -from pathlib import Path - -import duckdb - -from build_data import AIRPORTS, CATEGORIES, CLEAN, MIDTOWN - -HERE = Path(__file__).resolve().parent -DATA = HERE / "data" -TASKS = HERE / "tasks.json" -N_TASKS = 60 -MIN_SLICE, MAX_SLICE = 40, 260 -PER_SLICE = 4 -FLIP, TRIALS, SURVIVE = 0.05, 200, 0.95 - -WEEKS = {1: (1, 7), 2: (8, 14), 3: (15, 21), 4: (22, 31)} -NAMES = {"fare_dispute": "fare disputes", "detour": "detours or bad routes", - "driver_conduct": "driver conduct", "vehicle_condition": "vehicle condition", - "lost_item": "lost items", "safety": "safety"} - - -def slices() -> list[tuple[str, str, callable]]: - """(key, how a task describes it, predicate over a complaint's trip).""" - night = lambda r: r["hour"] >= 22 or r["hour"] < 5 - air = lambda r: r["pu"] in AIRPORTS or r["do"] in AIRPORTS - out = [ - ("jfk_pu", "trips picked up at JFK Airport (zone 132)", lambda r: r["pu"] == 132), - ("jfk_do", "trips dropped off at JFK Airport (zone 132)", lambda r: r["do"] == 132), - ("jfk", "trips to or from JFK Airport (zone 132)", lambda r: 132 in (r["pu"], r["do"])), - ("lga", "trips to or from LaGuardia Airport (zone 138)", lambda r: 138 in (r["pu"], r["do"])), - ("ewr", "trips to or from Newark Airport (zone 1)", lambda r: 1 in (r["pu"], r["do"])), - ("air", "trips to or from any of the three airports (zones 1, 132, 138)", air), - ("tsq", "trips picked up in Times Sq/Theatre District (zone 230)", lambda r: r["pu"] == 230), - ("midtown", f"trips picked up in Midtown (zones {', '.join(map(str, MIDTOWN))})", - lambda r: r["pu"] in MIDTOWN), - ("night", "trips picked up between 22:00 and 04:59", night), - ("night_midtown", f"trips picked up in Midtown (zones {', '.join(map(str, MIDTOWN))}) " - "between 22:00 and 04:59", lambda r: night(r) and r["pu"] in MIDTOWN), - ("night_air", "airport trips (to or from zones 1, 132, 138) picked up between 22:00 " - "and 04:59", lambda r: night(r) and air(r)), - ("long", "trips longer than 8 miles that neither start nor end at an airport " - "(zones 1, 132, 138)", lambda r: r["miles"] > 8 and not air(r)), - ] - for w, (a, b) in WEEKS.items(): - span = f"picked up from January {a} to January {b} inclusive" - out.append((f"wk{w}", f"trips {span}", lambda r, a=a, b=b: a <= r["day"] <= b)) - out.append((f"air_wk{w}", f"airport trips (to or from zones 1, 132, 138) {span}", - lambda r, a=a, b=b: air(r) and a <= r["day"] <= b)) - out.append((f"jfk_pu_wk{w}", f"trips picked up at JFK Airport (zone 132) {span}", - lambda r, a=a, b=b: r["pu"] == 132 and a <= r["day"] <= b)) - out.append((f"night_wk{w}", f"trips {span}, picked up between 22:00 and 04:59", - lambda r, a=a, b=b: night(r) and a <= r["day"] <= b)) - out.append((f"midtown_wk{w}", f"Midtown pickups (zones {', '.join(map(str, MIDTOWN))}) " - f"{span}", lambda r, a=a, b=b: r["pu"] in MIDTOWN - and a <= r["day"] <= b)) - for a, b in ((1, 15), (16, 31)): - out.append((f"lga_{a}_{b}", f"trips to or from LaGuardia Airport (zone 138) picked up " - f"from January {a} to January {b} inclusive", - lambda r, a=a, b=b: 138 in (r["pu"], r["do"]) and a <= r["day"] <= b)) - return out - - -# Each template returns (ask, answer function over labels, kind, tolerance) or None. -def share(desc, cat): - def f(rows, lab): - return round(100 * sum(lab[r["cid"]] == cat for r in rows) / len(rows), 1) - return (f"Among complaints about {desc}, what percentage are mainly about " - f"{NAMES[cat]}? One decimal.", f, "number", 3.0) - - -def count(desc, cat): - def f(rows, lab): - return sum(lab[r["cid"]] == cat for r in rows) - return (f"How many complaints about {desc} are mainly about {NAMES[cat]}?", - f, "count", 0.10) - - -def top(desc, _): - def f(rows, lab): - n = {c: 0 for c in CATEGORIES} - for r in rows: - n[lab[r["cid"]]] += 1 - return max(CATEGORIES, key=lambda c: (n[c], -CATEGORIES.index(c))) - return (f"Which category of complaint is most common among complaints about {desc}? " - f"Answer with one of: {', '.join(CATEGORIES)}.", f, "string", 0) - - -def median_total(desc, cat): - def f(rows, lab): - xs = [r["total"] for r in rows if lab[r["cid"]] == cat] - return round(statistics.median(xs), 2) if xs else None - return (f"What is the median total_amount of {desc} that drew a complaint mainly " - f"about {NAMES[cat]}? Two decimals.", f, "number_rel", 0.05) - - -TEMPLATES = [share, count, top, median_total] - - -def within(kind, tol, truth, got) -> bool: - if truth is None or got is None: - return truth == got - if kind == "string": - return truth == got - if kind == "number": - return abs(got - truth) <= tol - if kind == "count": - return abs(got - truth) <= max(2, tol * truth) - return abs(got - truth) <= tol * abs(truth) # number_rel - - -def main() -> None: - con = duckdb.connect() - rows = con.execute(f""" - SELECT c.complaint_id, t."PULocationID", t."DOLocationID", - EXTRACT(HOUR FROM t.tpep_pickup_datetime), EXTRACT(DAY FROM t.tpep_pickup_datetime), - t.trip_distance, t.total_amount - FROM read_parquet('{DATA / "complaints.parquet"}') c - JOIN read_parquet('{DATA / "trips.parquet"}') t USING (trip_id) - WHERE {CLEAN}""").fetchall() - rows = [dict(zip(["cid", "pu", "do", "hour", "day", "miles", "total"], r)) for r in rows] - truth = {json.loads(l)["complaint_id"]: json.loads(l)["category"] - for l in (DATA / "truth.jsonl").read_text().splitlines()} - rng = random.Random(7) - - candidates = [] - for key, desc, pred in slices(): - sl = [r for r in rows if pred(r)] - if not MIN_SLICE <= len(sl) <= MAX_SLICE: - continue - for template in TEMPLATES: - for cat in (CATEGORIES if template is not top else [None]): - ask, f, kind, tol = template(desc, cat) - answer = f(sl, truth) - if answer is None or (kind in ("count", "number") and answer < 5): - continue # too rare to ask about fairly - ok = 0 - for _ in range(TRIALS): - noisy = dict(truth) - for r in sl: - if rng.random() < FLIP: - noisy[r["cid"]] = rng.choice([c for c in CATEGORIES if c != truth[r["cid"]]]) - ok += within(kind, tol, answer, f(sl, noisy)) - if ok / TRIALS >= SURVIVE: - candidates.append({"slice": key, "ask": ask, "kind": kind, "tol": tol, - "answer": answer, "needs": sorted(r["cid"] for r in sl), - "survival": ok / TRIALS}) - - # One task per slice-and-template first, so the bank spreads across slices, then fill. - rng.shuffle(candidates) - seen, picked = set(), [] - for c in candidates: - k = (c["slice"], c["ask"].split()[0]) - if k not in seen: - seen.add(k) - picked.append(c) - picked += [c for c in candidates if c not in picked] - per: dict[str, int] = {} - capped = [] - for c in picked: - if per.get(c["slice"], 0) < PER_SLICE: - per[c["slice"]] = per.get(c["slice"], 0) + 1 - capped.append(c) - picked = capped[:N_TASKS] - rng.shuffle(picked) - for i, t in enumerate(picked, 1): - t["id"] = f"t{i:02d}" - - TASKS.write_text(json.dumps(picked, indent=1) + "\n") - needed = [len(t["needs"]) for t in picked] - distinct = len({c for t in picked for c in t["needs"]}) - print(f"{len(candidates)} robust candidates, {len(picked)} tasks over " - f"{len({t['slice'] for t in picked})} slices -> {TASKS.name}") - print(f"complaints per task: min {min(needed)}, median {statistics.median(needed)}, " - f"max {max(needed)}; {sum(needed)} needed in all, {distinct} distinct " - f"({sum(needed) / distinct:.1f}x reuse available)") - for kind in ("number", "count", "string", "number_rel"): - print(f" {kind:10} {sum(t['kind'] == kind for t in picked)}") - - -if __name__ == "__main__": - main() diff --git a/swarm/check_complaints.py b/swarm/check_complaints.py deleted file mode 100644 index 2cdb683..0000000 --- a/swarm/check_complaints.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Keep only the complaints a second model classifies the way they were written. - -A complaint a careful reader could fairly put in two categories would grade an agent -wrong for a defensible answer. So a different model reads each one blind, with the -same category definitions the agents get, and only agreements are kept. - -Writes what the agents see and what grading uses, separately: - data/complaints.parquet complaint_id, trip_id, text - data/truth.jsonl complaint_id, trip_id, category, stratum - - python3 check_complaints.py # classify what is not yet checked - python3 check_complaints.py --finish # write the two files from the checks -""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import threading -from collections import Counter -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -import duckdb - -from gen_complaints import MEANING - -HERE = Path(__file__).resolve().parent -DATA = HERE / "data" -SPECS = DATA / "complaint_specs.jsonl" -RAW = DATA / "complaints_raw.jsonl" -CHECKS = DATA / "checks.jsonl" -OUT = DATA / "complaints.parquet" -TRUTH = DATA / "truth.jsonl" - -SYSTEM = "You classify passenger complaints for a city taxi commission." - -BRIEF = """Give each complaint the one category that is the passenger's main reason for -complaining. A minor annoyance mentioned in passing is not the main reason. - -Categories: -""" + "\n".join(f"- {name}: {meaning}" for name, meaning in MEANING.items()) + """ - -Return only a JSON array: [{"complaint_id": , "category": ""}, ...] - -Complaints: -""" - -lock = threading.Lock() - - -def classify(batch: list[dict], model: str) -> list[dict]: - prompt = BRIEF + "\n".join(json.dumps({"complaint_id": c["complaint_id"], "text": c["text"]}) - for c in batch) - run = subprocess.run( - ["claude", "-p", prompt, "--model", model, "--system-prompt", SYSTEM, - "--setting-sources", "", "--strict-mcp-config", "--tools", "", - "--disable-slash-commands", "--no-session-persistence", "--output-format", "json"], - capture_output=True, text=True, timeout=600) - result = json.loads(run.stdout) - if result.get("is_error"): - raise RuntimeError(result.get("result")) - text = re.sub(r"^```(?:json)?\s*|\s*```\s*$", "", result["result"].strip()) - return json.loads(text) - - -def check(args) -> None: - done = set() - if CHECKS.exists(): - done = {json.loads(line)["complaint_id"] for line in CHECKS.read_text().splitlines()} - raw = [json.loads(line) for line in RAW.read_text().splitlines()] - todo = [c for c in raw if c["complaint_id"] not in done] - batches = [todo[i:i + args.batch] for i in range(0, len(todo), args.batch)] - print(f"{len(done)} checked, {len(todo)} to check in {len(batches)} batches") - - def work(batch: list[dict]) -> None: - try: - labels = classify(batch, args.model) - except Exception as error: - print(f" batch from {batch[0]['complaint_id']} failed: {error}") - return - wanted = {c["complaint_id"] for c in batch} - with lock, CHECKS.open("a") as out: - for item in labels: - if item.get("complaint_id") in wanted: - out.write(json.dumps({"complaint_id": item["complaint_id"], - "category": item.get("category")}) + "\n") - print(f" batch from {batch[0]['complaint_id']}: {len(labels)}/{len(batch)}") - - with ThreadPoolExecutor(args.workers) as pool: - list(pool.map(work, batches)) - - -def finish() -> None: - specs = {s["complaint_id"]: s for s in map(json.loads, SPECS.read_text().splitlines())} - raw = {c["complaint_id"]: c for c in map(json.loads, RAW.read_text().splitlines())} - checks = {c["complaint_id"]: c["category"] for c in map(json.loads, CHECKS.read_text().splitlines())} - - kept, confusion = [], Counter() - for cid, text in sorted(raw.items()): - if cid not in checks: - continue - written, read = specs[cid]["category"], checks[cid] - confusion[(written, read)] += 1 - if written == read: - kept.append(cid) - - print(f"{len(kept)} of {len(checks)} kept ({len(kept) / len(checks):.1%} agreement)") - print("disagreements, written -> read:") - for (w, r), n in confusion.most_common(): - if w != r: - print(f" {w:18} -> {r:18} {n}") - - with TRUTH.open("w") as out: - for cid in kept: - s = specs[cid] - out.write(json.dumps({"complaint_id": cid, "trip_id": s["trip_id"], - "category": s["category"], "stratum": s["stratum"]}) + "\n") - rows = [(cid, specs[cid]["trip_id"], raw[cid]["text"]) for cid in kept] - con = duckdb.connect() - con.execute("CREATE TABLE c (complaint_id INTEGER, trip_id BIGINT, text VARCHAR)") - con.executemany("INSERT INTO c VALUES (?, ?, ?)", rows) - con.execute(f"COPY (SELECT * FROM c ORDER BY complaint_id) TO '{OUT}' (FORMAT parquet)") - print(f"-> {OUT.name}, {TRUTH.name}") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--finish", action="store_true") - parser.add_argument("--batch", type=int, default=25) - parser.add_argument("--workers", type=int, default=8) - parser.add_argument("--model", default="claude-sonnet-5-5") - args = parser.parse_args() - finish() if args.finish else check(args) - - -if __name__ == "__main__": - main() diff --git a/swarm/gen_complaints.py b/swarm/gen_complaints.py deleted file mode 100644 index 7ad1f87..0000000 --- a/swarm/gen_complaints.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Write the text of each complaint from its spec, with Claude Haiku. - -The category is fixed before the text exists (build_data.py), so it is the truth the -benchmark grades against. The writer is told the category and the trip, and told not -to name the category, so a reader has to understand the complaint to classify it. - -Resumable: complaints already in data/complaints_raw.jsonl are skipped. - - python3 gen_complaints.py --limit 20 # a pilot batch - python3 gen_complaints.py # the rest -""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import threading -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -HERE = Path(__file__).resolve().parent -SPECS = HERE / "data" / "complaint_specs.jsonl" -RAW = HERE / "data" / "complaints_raw.jsonl" - -MEANING = { - "fare_dispute": "charged more than they should have been: the meter, a surcharge, " - "a flat airport fare not honoured, a card charged twice or wrongly", - "detour": "the driver took a longer, slower or wrong route, or ignored the route " - "the passenger asked for", - "driver_conduct": "the driver was rude, dismissive, on the phone, refused a stop, " - "argued, or made inappropriate remarks, without physical danger", - "vehicle_condition": "the cab itself was the problem: dirty, smelly, no heat, a " - "broken door, window, seat or card reader", - "lost_item": "the passenger left something in the cab and wants it back", - "safety": "the passenger was put in physical danger: speeding, running lights, " - "phone use while weaving, a near collision, drowsy or impaired driving", -} - -SYSTEM = "You write realistic passenger complaints for a city taxi commission dataset." - -BRIEF = """Write one complaint for each trip below. Each complaint is about the issue -described for it, and only that issue is the reason for complaining. - -Rules: -- 150 to 300 words, first person, as a real passenger would write to the commission. -- Vary the voice: some angry, some calm, some rambling, some terse and formal, some - with typos. No two should open the same way. -- Use the trip facts loosely and consistently: the zones, roughly when, the fare if - it matters. Never quote a trip id. -- About a third may mention a minor second annoyance in passing, but the main issue - must be clear to a careful reader. -- Never use these words for the issue: fare dispute, detour, conduct, vehicle - condition, lost item, safety. - -Return only a JSON array: [{"complaint_id": , "text": ""}, ...] - -Trips: -""" - -lock = threading.Lock() - - -def describe(spec: dict) -> str: - return json.dumps({ - "complaint_id": spec["complaint_id"], - "issue": MEANING[spec["category"]], - "pickup": spec["pickup_zone"], "dropoff": spec["dropoff_zone"], - "when": spec["pickup"], "miles": spec["miles"], "fare": spec["fare"], - "total_paid": spec["total"], "tip": spec["tip"], "passengers": spec["passengers"], - "paid_by_card": spec["card"]}) - - -def generate(batch: list[dict], model: str) -> tuple[list[dict], float]: - prompt = BRIEF + "\n".join(describe(s) for s in batch) - run = subprocess.run( - ["claude", "-p", prompt, "--model", model, "--system-prompt", SYSTEM, - "--setting-sources", "", "--strict-mcp-config", "--tools", "", - "--disable-slash-commands", "--no-session-persistence", "--output-format", "json"], - capture_output=True, text=True, timeout=600) - result = json.loads(run.stdout) - if result.get("is_error"): - raise RuntimeError(result.get("result")) - text = re.sub(r"^```(?:json)?\s*|\s*```\s*$", "", result["result"].strip()) - items = json.loads(text) - wanted = {s["complaint_id"] for s in batch} - return [i for i in items if i.get("complaint_id") in wanted and i.get("text")], \ - float(result.get("total_cost_usd") or 0) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--limit", type=int, default=0) - parser.add_argument("--batch", type=int, default=20) - parser.add_argument("--workers", type=int, default=8) - parser.add_argument("--model", default="claude-haiku-5-5") - args = parser.parse_args() - - done = set() - if RAW.exists(): - done = {json.loads(line)["complaint_id"] for line in RAW.read_text().splitlines()} - specs = [json.loads(line) for line in SPECS.read_text().splitlines()] - todo = [s for s in specs if s["complaint_id"] not in done] - if args.limit: - todo = todo[:args.limit] - batches = [todo[i:i + args.batch] for i in range(0, len(todo), args.batch)] - print(f"{len(done)} done, {len(todo)} to write in {len(batches)} batches") - - spent = 0.0 - - def work(batch: list[dict]) -> None: - nonlocal spent - try: - items, cost = generate(batch, args.model) - except Exception as error: # one bad batch is retried next run - print(f" batch from {batch[0]['complaint_id']} failed: {error}") - return - with lock, RAW.open("a") as out: - for item in items: - out.write(json.dumps({"complaint_id": item["complaint_id"], - "text": item["text"].strip()}) + "\n") - spent += cost - print(f" batch from {batch[0]['complaint_id']}: {len(items)}/{len(batch)}") - - with ThreadPoolExecutor(args.workers) as pool: - list(pool.map(work, batches)) - print(f"spent ${spent:.2f} (as the CLI prices it; Haiku 5.5 may be priced as Opus)") - - -if __name__ == "__main__": - main() diff --git a/swarm/runner.py b/swarm/runner.py deleted file mode 100644 index 1dd5383..0000000 --- a/swarm/runner.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Run the swarm: one agent per task, C agents at a time, in one arm. - -Every task needs complaints classified. What differs between arms is whether an -agent can find and reuse the classifications other agents already made: - - isolated files on disk, nothing shared. How agents work today. - folder the same files, plus one folder every agent can read and write. - hotdata the plugin, one project database with a `labels` table every agent shares. - -Each session is a separate `claude -p` process with a fresh working directory. Its -JSON result goes to /sessions/.json; after the run, every label any agent -recorded is collected into /labels.jsonl, and the graded answers into -/answers.jsonl. - - python3 runner.py --arm hotdata --concurrency 4 --replicate 1 --tasks 20 -""" - -from __future__ import annotations - -import argparse -import csv -import io -import json -import os -import re -import secrets -import shutil -import subprocess -import sys -import threading -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -import duckdb - -from gen_complaints import MEANING - -HERE = Path(__file__).resolve().parent -DATA = HERE / "data" -PLUGIN_DIR = str(HERE.parent) -# Agents work outside the repository, where the truth and every other run live. Their -# data directory holds only the three files they are meant to see. -BENCH = Path.home() / "swarm-bench" -AGENT_DATA = BENCH / "data" -SOURCES = {"trips": "trips.parquet", "complaints": "complaints.parquet", - "zones": "taxi_zone_lookup.csv"} -FILES = {k: AGENT_DATA / v for k, v in SOURCES.items()} - -BRIEF = """You are one of many agents working for a city taxi commission. Passengers file -complaints about taxi trips, one trip per complaint. Your job is to answer one question -that needs the complaints on some trips classified by their main issue. - -Categories, by these exact names: -{categories} - -A complaint's category is the passenger's main reason for complaining. A minor -annoyance mentioned in passing is not. - -{store} - -Keep any scratch files in your working directory. - -Record every classification you make, with your agent id `{agent}`, so the work is not -lost: {record} -{claiming} -Question: {ask} - -When you are done, print one final line and nothing after it: - -ANSWER - -The value is a bare number (no units, no thousands separator) or a category name. -""" - -FILES_STORE = """The data is in files: - - {trips} one row per trip, keyed by trip_id - {complaints} complaint_id, trip_id, text - {zones} the LocationID lookup, with Borough and Zone - -Work however you like, with whatever is on this machine.""" - -STORES = { - "isolated": ( - FILES_STORE, - "append them to labels.csv in your working directory, with the columns " - "complaint_id,category,agent."), - "folder": ( - FILES_STORE + """ - -Other agents working on related questions share the folder {shared}. The -classifications they have made are CSV files in {shared}/labels/ with the columns -complaint_id, category, agent. Use them rather than classifying a complaint again.""", - "write them as CSV into {shared}/labels/ with the columns complaint_id,category," - "agent. Other agents read that folder, possibly while you write to it."), - "hotdata": ( - """The data is in the project's Hotdata database, and `tabular-memory` is on your PATH. Your -session started with its tables, a sample of each and the SQL dialect. The tables: - - {cat}.public.trips one row per trip, keyed by trip_id - {cat}.public.complaints complaint_id, trip_id, text - {cat}.public.zones the LocationID lookup, with Borough and Zone - {cat}.public.labels complaint_id, category, agent - {cat}.public.claims complaint_id, agent: work in progress - -`labels` holds the classifications other agents working on related questions have -made. Use them rather than classifying a complaint again.""", - "INSERT them into {cat}.public.labels with `tabular-memory query`, all of them in one " - "statement. If the insert is refused with \"retry shortly\", another agent is " - "writing; run the same statement again after a few seconds."), -} - -CLAIMING = """ -Other agents are working at the same time, often on the same complaints. Before you -classify, claim the work so two agents never classify the same complaint: - -1. Find the complaints your question needs that have no label yet. -2. Claim the ones nobody has claimed: {claim} -3. Classify exactly the complaints you claimed, and record them as above. -4. A complaint another agent claimed but has not labelled yet is being classified - now. Wait about 30 seconds and check the labels again. If one is still missing after - five minutes, classify it yourself. -""" - -CLAIMS = { - "folder": "create the empty file {shared}/claims/ for each one, " - "exclusively, so creating it fails if it already exists (in Python, " - "open(path, 'x')). The files you created are your claims.", - "hotdata": "in one statement, INSERT INTO {cat}.public.claims (complaint_id, agent) " - "SELECT the unlabelled, unclaimed ids with '{agent}' as the agent, excluding " - "complaint_id IN (SELECT complaint_id FROM {cat}.public.claims). Your claims " - "are the rows with agent = '{agent}'.", -} - -ANSWER = re.compile(r"^ANSWER\s+(.+?)\s*$", re.M) - - -def settings_for(arm: str) -> dict: - """Every arm gets every tool on the machine; only where the data lives differs. - - The arms without Hotdata have the CLI denied, so a mention of it in a skill listing - cannot turn into a call. - """ - permissions = {"allow": ["Bash", "Read", "Write", "Edit", "Glob", "Grep"]} - if arm != "hotdata": - permissions["deny"] = ["Bash(hotdata:*)", "Bash(tabular-memory:*)"] - return {"permissions": permissions, "autoMemoryEnabled": False, - "enabledPlugins": {"hotdata-tabular-memory@hotdata": False}} - - -def hotdata(*args: str) -> str: - run = subprocess.run(["hotdata", *args, "--no-input"], capture_output=True, text=True) - if run.returncode: - raise SystemExit(f"hotdata {' '.join(args)}: {run.stderr.strip()}") - return run.stdout - - -def make_database(name: str, scratch: Path) -> tuple[str, str]: - """A fresh database for one run: the three data tables and an empty labels table.""" - catalog = f"swarm_{name.replace('-', '_')}_{secrets.token_hex(3)}" - made = json.loads(hotdata("databases", "create", "--name", f"swarm {name}", - "--catalog", catalog, "--expires-at", "7d", "-o", "json")) - db = made["id"] - for table, path in FILES.items(): - hotdata("databases", "load", "--catalog", catalog, "--table", table, "--file", str(path)) - # A typed parquet with no rows: the server has no DELETE yet, so the table cannot - # be seeded with a row and emptied. - seed = scratch / "labels_empty.parquet" - duckdb.execute(f"COPY (SELECT 0::INTEGER AS complaint_id, ''::VARCHAR AS category, " - f"''::VARCHAR AS agent WHERE false) TO '{seed}' (FORMAT parquet)") - hotdata("databases", "load", "--catalog", catalog, "--table", "labels", "--file", str(seed)) - claims = scratch / "claims_empty.parquet" - duckdb.execute(f"COPY (SELECT 0::INTEGER AS complaint_id, ''::VARCHAR AS agent " - f"WHERE false) TO '{claims}' (FORMAT parquet)") - hotdata("databases", "load", "--catalog", catalog, "--table", "claims", "--file", str(claims)) - return db, catalog - - -def labels_from_csv(text: str, source: str, bad: list) -> list[dict]: - out = [] - try: - for row in csv.DictReader(io.StringIO(text)): - try: - out.append({"complaint_id": int(row["complaint_id"]), - "category": (row.get("category") or "").strip(), - "agent": (row.get("agent") or "").strip(), "source": source}) - except (KeyError, TypeError, ValueError): - bad.append({"source": source, "row": row}) - except csv.Error as error: - bad.append({"source": source, "error": str(error)}) - return out - - -def collect_labels(arm: str, root: Path, db: str, catalog: str) -> tuple[list, list]: - bad: list = [] - if arm == "hotdata": - text = hotdata("query", f"SELECT complaint_id, category, agent FROM " - f"{catalog}.public.labels", "-d", db, "-o", "csv") - return labels_from_csv(text, "labels", bad), bad - files = (sorted((root / "sessions").glob("*/labels.csv")) if arm == "isolated" - else sorted((root / "shared" / "labels").rglob("*"))) - labels = [] - for f in files: - if f.is_file(): - labels += labels_from_csv(f.read_text(errors="replace"), str(f.relative_to(root)), bad) - return labels, bad - - -def grade(task: dict, given: str | None) -> bool: - if given is None: - return False - if task["kind"] == "string": - return given.strip().strip("`'\"").lower() == task["answer"] - try: - value = float(given.replace(",", "").replace("%", "").replace("$", "").strip()) - except ValueError: - return False - truth, tol = task["answer"], task["tol"] - if task["kind"] == "number": - return abs(value - truth) <= tol - if task["kind"] == "count": - return abs(value - truth) <= max(2, tol * truth) - return abs(value - truth) <= tol * abs(truth) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--arm", required=True, choices=list(STORES)) - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--replicate", type=int, default=1) - parser.add_argument("--tasks", type=int, default=0, help="run only the first N tasks") - parser.add_argument("--pick", default="", help="run only these task ids, in this order") - parser.add_argument("--model", default="claude-opus-5") - parser.add_argument("--out", type=Path, default=BENCH / "runs") - parser.add_argument("--claim", action="store_true", - help="shared arms claim complaints before classifying them") - args = parser.parse_args() - - AGENT_DATA.mkdir(parents=True, exist_ok=True) - for f in SOURCES.values(): - if not (AGENT_DATA / f).exists(): - shutil.copy(DATA / f, AGENT_DATA / f) - name = f"{args.arm}-c{args.concurrency}-r{args.replicate}" + ("-claim" if args.claim else "") - root = args.out.resolve() / name - if root.exists(): - shutil.rmtree(root) - (root / "sessions").mkdir(parents=True) - tasks = json.loads((HERE / "tasks.json").read_text()) - if args.pick: - by_id = {t["id"]: t for t in tasks} - tasks = [by_id[i] for i in args.pick.split(",")] - if args.tasks: - tasks = tasks[:args.tasks] - - settings = root / "settings.json" - settings.write_text(json.dumps(settings_for(args.arm), indent=1)) - env = {**os.environ, "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"} - db = catalog = "" - shared = root / "shared" - if args.arm == "folder": - (shared / "labels").mkdir(parents=True) - (shared / "claims").mkdir() - if args.arm == "hotdata": - db, catalog = make_database(name, root) - env["HOTDATA_API_KEY"] = (Path.home() / ".hotdata/api-key").read_text().strip() - env["PATH"] = f"{PLUGIN_DIR}/bin:" + env.get("PATH", "") - env["CLAUDE_PLUGIN_OPTION_DATABASE"] = db - env["CLAUDE_PLUGIN_OPTION_PROJECT"] = f"swarm-{name}" - env["CLAUDE_PLUGIN_DATA"] = str(root / "plugin-data") - cli_version = subprocess.run(["claude", "--version"], capture_output=True, - text=True).stdout.strip() - (root / "run.json").write_text(json.dumps({ - "arm": args.arm, "concurrency": args.concurrency, "replicate": args.replicate, - "model": args.model, "cli": cli_version, "database": db, "catalog": catalog, - "claim": args.claim, - "tasks": [t["id"] for t in tasks]}, indent=1)) - - categories = "\n".join(f"- {k}: {v}" for k, v in MEANING.items()) - store, record = STORES[args.arm] - paths = {k: str(v) for k, v in FILES.items()} - lock = threading.Lock() - run_started = time.time() - - halted = threading.Event() - - def session(task: dict) -> None: - if halted.is_set(): - return - work = root / "sessions" / task["id"] - work.mkdir() - fill = {**paths, "shared": str(shared), "cat": catalog} - fill["agent"] = task["id"] - claiming = (CLAIMING.format(claim=CLAIMS[args.arm].format(**fill)) - if args.claim and args.arm in CLAIMS else "") - prompt = BRIEF.format(categories=categories, store=store.format(**fill), - record=record.format(**fill), claiming=claiming, - agent=task["id"], ask=task["ask"]) - started = time.time() - (work / "tmp").mkdir() - proc = subprocess.run( - ["claude", "-p", prompt, "--output-format", "json", "--model", args.model, - "--settings", str(settings), "--permission-mode", "acceptEdits", - *(["--plugin-dir", PLUGIN_DIR] if args.arm == "hotdata" else [])], - capture_output=True, text=True, cwd=work, env={**env, "TMPDIR": str(work / "tmp")}, - timeout=3600) - ended = time.time() - try: - result = json.loads(proc.stdout) - except json.JSONDecodeError: - result = {"is_error": True, "result": proc.stdout[-2000:], "stderr": proc.stderr[-2000:]} - if result.get("is_error") and "limit" in str(result.get("result", "")).lower(): - # Every later session would fail the same way; a run with holes in it - # cannot be compared, so stop and say so. - halted.set() - print(f"{name} HALTED at {task['id']}: {str(result.get('result'))[:120]}", flush=True) - found = ANSWER.findall(result.get("result") or "") - given = found[-1] if found else None - result.update({"task": task["id"], "start": started - run_started, - "end": ended - run_started, "given": given, - "correct": grade(task, given), "cli_version": cli_version}) - (root / "sessions" / f"{task['id']}.json").write_text(json.dumps(result, indent=1)) - with lock: - with (root / "answers.jsonl").open("a") as out: - out.write(json.dumps({"task": task["id"], "given": given, - "answer": task["answer"], "correct": result["correct"]}) + "\n") - print(f"{name} {task['id']} {'ok ' if result['correct'] else 'BAD'} " - f"given={given} truth={task['answer']} ${result.get('total_cost_usd', 0):.2f} " - f"{ended - started:.0f}s turns={result.get('num_turns', '?')}", flush=True) - - with ThreadPoolExecutor(args.concurrency) as pool: - list(pool.map(session, tasks)) - - labels, bad = collect_labels(args.arm, root, db, catalog) - with (root / "labels.jsonl").open("w") as out: - for label in labels: - out.write(json.dumps(label) + "\n") - (root / "unreadable.json").write_text(json.dumps(bad, indent=1)) - correct = sum(json.loads(l)["correct"] for l in (root / "answers.jsonl").read_text().splitlines()) - print(f"{name} DONE {correct}/{len(tasks)} correct, {len(labels)} labels recorded, " - f"{len(bad)} unreadable", flush=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/swarm/tasks.json b/swarm/tasks.json deleted file mode 100644 index 2e825a4..0000000 --- a/swarm/tasks.json +++ /dev/null @@ -1,10002 +0,0 @@ -[ - { - "slice": "night_wk3", - "ask": "How many complaints about trips picked up from January 15 to January 21 inclusive, picked up between 22:00 and 04:59 are mainly about safety?", - "kind": "count", - "tol": 0.1, - "answer": 41, - "needs": [ - 12, - 23, - 94, - 102, - 118, - 131, - 167, - 178, - 197, - 217, - 241, - 251, - 264, - 270, - 277, - 342, - 365, - 380, - 404, - 442, - 445, - 467, - 510, - 664, - 675, - 703, - 707, - 709, - 749, - 768, - 805, - 834, - 845, - 874, - 886, - 887, - 904, - 923, - 928, - 972, - 1013, - 1075, - 1146, - 1165, - 1170, - 1208, - 1225, - 1267, - 1339, - 1343, - 1437, - 1463, - 1472, - 1521, - 1536, - 1561, - 1616, - 1637, - 1641, - 1642, - 1643, - 1644, - 1647, - 1649, - 1651, - 1652, - 1653, - 1656, - 1662, - 1663, - 1669, - 1670, - 1671, - 1672, - 1691, - 1703, - 1706, - 1707, - 1709, - 1715, - 1720, - 1728, - 1729, - 1733, - 1737, - 1741, - 1743, - 1752, - 1753, - 1755, - 1758, - 1759, - 1766, - 1776, - 1779, - 1800, - 1804, - 1809, - 1811, - 1815, - 1818, - 1821, - 1824, - 1827, - 1833, - 1836, - 1846, - 1851, - 1860, - 1861, - 1868, - 1880, - 1883, - 1884, - 1890, - 1891, - 1893, - 1894, - 1896, - 1899, - 1904, - 1908, - 1918, - 1927, - 1935, - 1936, - 1937, - 1942, - 1944, - 1946, - 1947, - 1952, - 1969, - 1984, - 1997, - 1998 - ], - "survival": 0.97, - "id": "t01" - }, - { - "slice": "jfk_do", - "ask": "Among complaints about trips dropped off at JFK Airport (zone 132), what percentage are mainly about vehicle condition? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 7.7, - "needs": [ - 2, - 5, - 7, - 13, - 35, - 41, - 46, - 73, - 79, - 80, - 87, - 105, - 107, - 109, - 128, - 137, - 152, - 154, - 162, - 163, - 179, - 186, - 189, - 192, - 195, - 199, - 208, - 221, - 235, - 246, - 281, - 282, - 287, - 293, - 312, - 314, - 331, - 337, - 339, - 344, - 353, - 358, - 364, - 376, - 381, - 383, - 390, - 396, - 403, - 432, - 434, - 458, - 478, - 483, - 500, - 515, - 518, - 526, - 578, - 582, - 583, - 593, - 606, - 611, - 671, - 688, - 690, - 700, - 729, - 753, - 756, - 757, - 759, - 776, - 777, - 780, - 788, - 790, - 795, - 824, - 837, - 844, - 848, - 857, - 874, - 878, - 882, - 884, - 885, - 894, - 898 - ], - "survival": 0.955, - "id": "t02" - }, - { - "slice": "night_air", - "ask": "Among complaints about airport trips (to or from zones 1, 132, 138) picked up between 22:00 and 04:59, what percentage are mainly about vehicle condition? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 12.2, - "needs": [ - 8, - 10, - 12, - 14, - 16, - 17, - 23, - 24, - 27, - 36, - 44, - 50, - 58, - 60, - 66, - 68, - 76, - 85, - 92, - 94, - 98, - 102, - 110, - 114, - 116, - 118, - 120, - 131, - 141, - 157, - 163, - 166, - 167, - 176, - 177, - 178, - 181, - 197, - 217, - 222, - 224, - 236, - 241, - 245, - 249, - 251, - 263, - 264, - 270, - 275, - 277, - 284, - 298, - 303, - 304, - 326, - 342, - 348, - 352, - 359, - 363, - 365, - 366, - 367, - 374, - 380, - 394, - 398, - 404, - 407, - 409, - 417, - 418, - 442, - 445, - 459, - 462, - 463, - 467, - 472, - 473, - 478, - 495, - 497, - 508, - 509, - 510, - 515, - 516, - 520, - 535, - 536, - 545, - 551, - 581, - 585, - 591, - 602, - 621, - 628, - 635, - 651, - 654, - 660, - 662, - 664, - 675, - 691, - 693, - 697, - 703, - 707, - 709, - 711, - 713, - 724, - 726, - 747, - 749, - 750, - 752, - 758, - 764, - 768, - 782, - 783, - 805, - 827, - 830, - 834, - 845, - 849, - 851, - 867, - 871, - 874, - 886, - 887, - 889 - ], - "survival": 0.995, - "id": "t03" - }, - { - "slice": "jfk_do", - "ask": "Which category of complaint is most common among complaints about trips dropped off at JFK Airport (zone 132)? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "fare_dispute", - "needs": [ - 2, - 5, - 7, - 13, - 35, - 41, - 46, - 73, - 79, - 80, - 87, - 105, - 107, - 109, - 128, - 137, - 152, - 154, - 162, - 163, - 179, - 186, - 189, - 192, - 195, - 199, - 208, - 221, - 235, - 246, - 281, - 282, - 287, - 293, - 312, - 314, - 331, - 337, - 339, - 344, - 353, - 358, - 364, - 376, - 381, - 383, - 390, - 396, - 403, - 432, - 434, - 458, - 478, - 483, - 500, - 515, - 518, - 526, - 578, - 582, - 583, - 593, - 606, - 611, - 671, - 688, - 690, - 700, - 729, - 753, - 756, - 757, - 759, - 776, - 777, - 780, - 788, - 790, - 795, - 824, - 837, - 844, - 848, - 857, - 874, - 878, - 882, - 884, - 885, - 894, - 898 - ], - "survival": 1.0, - "id": "t04" - }, - { - "slice": "jfk_do", - "ask": "How many complaints about trips dropped off at JFK Airport (zone 132) are mainly about vehicle condition?", - "kind": "count", - "tol": 0.1, - "answer": 7, - "needs": [ - 2, - 5, - 7, - 13, - 35, - 41, - 46, - 73, - 79, - 80, - 87, - 105, - 107, - 109, - 128, - 137, - 152, - 154, - 162, - 163, - 179, - 186, - 189, - 192, - 195, - 199, - 208, - 221, - 235, - 246, - 281, - 282, - 287, - 293, - 312, - 314, - 331, - 337, - 339, - 344, - 353, - 358, - 364, - 376, - 381, - 383, - 390, - 396, - 403, - 432, - 434, - 458, - 478, - 483, - 500, - 515, - 518, - 526, - 578, - 582, - 583, - 593, - 606, - 611, - 671, - 688, - 690, - 700, - 729, - 753, - 756, - 757, - 759, - 776, - 777, - 780, - 788, - 790, - 795, - 824, - 837, - 844, - 848, - 857, - 874, - 878, - 882, - 884, - 885, - 894, - 898 - ], - "survival": 0.97, - "id": "t05" - }, - { - "slice": "air_wk2", - "ask": "Which category of complaint is most common among complaints about airport trips (to or from zones 1, 132, 138) picked up from January 8 to January 14 inclusive? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "fare_dispute", - "needs": [ - 1, - 3, - 7, - 10, - 16, - 18, - 19, - 21, - 27, - 30, - 31, - 34, - 37, - 46, - 48, - 52, - 53, - 57, - 68, - 71, - 77, - 78, - 83, - 95, - 96, - 101, - 110, - 111, - 113, - 122, - 123, - 124, - 133, - 135, - 137, - 142, - 146, - 160, - 163, - 166, - 169, - 172, - 174, - 176, - 179, - 180, - 181, - 182, - 183, - 193, - 206, - 208, - 209, - 240, - 242, - 245, - 247, - 250, - 265, - 286, - 291, - 299, - 305, - 308, - 315, - 317, - 318, - 325, - 331, - 335, - 343, - 348, - 352, - 353, - 354, - 356, - 362, - 363, - 366, - 369, - 370, - 371, - 377, - 383, - 391, - 398, - 406, - 412, - 414, - 416, - 418, - 420, - 425, - 429, - 431, - 436, - 437, - 443, - 448, - 453, - 454, - 456, - 463, - 472, - 473, - 474, - 479, - 495, - 497, - 498, - 500, - 501, - 503, - 506, - 509, - 511, - 514, - 515, - 518, - 534, - 536, - 546, - 550, - 554, - 559, - 560, - 576, - 580, - 590, - 595, - 599, - 600, - 619, - 620, - 628, - 632, - 640, - 652, - 666, - 669, - 677, - 685, - 690, - 701, - 705, - 726, - 734, - 744, - 752, - 754, - 758, - 767, - 769, - 779, - 780, - 783, - 787, - 792, - 793, - 796, - 798, - 799, - 800, - 801, - 812, - 814, - 816, - 820, - 822, - 825, - 828, - 829, - 830, - 831, - 833, - 842, - 846, - 854, - 856, - 861, - 862, - 864, - 866, - 869, - 873, - 876, - 880, - 881, - 896, - 900 - ], - "survival": 1.0, - "id": "t06" - }, - { - "slice": "lga_1_15", - "ask": "What is the median total_amount of trips to or from LaGuardia Airport (zone 138) picked up from January 1 to January 15 inclusive that drew a complaint mainly about driver conduct? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 64.32, - "needs": [ - 1, - 8, - 14, - 16, - 31, - 43, - 53, - 54, - 57, - 64, - 72, - 74, - 86, - 88, - 91, - 100, - 104, - 106, - 113, - 114, - 115, - 122, - 124, - 135, - 137, - 139, - 144, - 145, - 157, - 164, - 169, - 172, - 180, - 181, - 182, - 205, - 206, - 214, - 224, - 226, - 237, - 240, - 247, - 262, - 263, - 266, - 279, - 283, - 285, - 288, - 289, - 291, - 296, - 297, - 305, - 308, - 313, - 315, - 331, - 338, - 351, - 352, - 354, - 360, - 364, - 369, - 370, - 371, - 388, - 389, - 391, - 410, - 412, - 414, - 415, - 416, - 418, - 423, - 425, - 429, - 432, - 437, - 443, - 447, - 448, - 449, - 451, - 452, - 453, - 454, - 463, - 475, - 480, - 497, - 498, - 499, - 501, - 503, - 505, - 511, - 514, - 516, - 517, - 519, - 525, - 546, - 549, - 559, - 576, - 580, - 587, - 597, - 601, - 610, - 617, - 619, - 635, - 639, - 640, - 648, - 654, - 658, - 666, - 669, - 685, - 687, - 697, - 701, - 705, - 711, - 718, - 736, - 752, - 761, - 763, - 769, - 778, - 787, - 792, - 798, - 799, - 800, - 801, - 802, - 807, - 814, - 816, - 817, - 820, - 822, - 825, - 827, - 829, - 830, - 833, - 854, - 859, - 862, - 864, - 865, - 866, - 867, - 869, - 873, - 881, - 892, - 893, - 900 - ], - "survival": 1.0, - "id": "t07" - }, - { - "slice": "jfk_pu_wk1", - "ask": "Which category of complaint is most common among complaints about trips picked up at JFK Airport (zone 132) picked up from January 1 to January 7 inclusive? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "fare_dispute", - "needs": [ - 4, - 11, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 44, - 50, - 51, - 58, - 65, - 67, - 75, - 76, - 81, - 84, - 85, - 92, - 103, - 112, - 127, - 129, - 134, - 136, - 141, - 148, - 152, - 177, - 184, - 187, - 191, - 194, - 196, - 200, - 201, - 202, - 207, - 211, - 213, - 216, - 222, - 227, - 229, - 230, - 238, - 239, - 249, - 259, - 260, - 267, - 271, - 275, - 294, - 303, - 311, - 321, - 334, - 337, - 347, - 357, - 359, - 367, - 374, - 375, - 382, - 417, - 430, - 440, - 459, - 470, - 485, - 494, - 508, - 513, - 522, - 527, - 529, - 535, - 545, - 548, - 552, - 563, - 588, - 591, - 612, - 634, - 649, - 650, - 651, - 655, - 662, - 665, - 672, - 681, - 694, - 702, - 713, - 727, - 728, - 732, - 740, - 746, - 755, - 761, - 765, - 786, - 811, - 818, - 823, - 838, - 851, - 853, - 860, - 870, - 875, - 883, - 889, - 890, - 891 - ], - "survival": 0.965, - "id": "t08" - }, - { - "slice": "air_wk1", - "ask": "What is the median total_amount of airport trips (to or from zones 1, 132, 138) picked up from January 1 to January 7 inclusive that drew a complaint mainly about fare disputes? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 82.69, - "needs": [ - 4, - 5, - 8, - 11, - 13, - 14, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 43, - 44, - 50, - 51, - 54, - 58, - 64, - 65, - 67, - 72, - 74, - 75, - 76, - 81, - 84, - 85, - 86, - 88, - 92, - 103, - 104, - 106, - 112, - 114, - 115, - 127, - 129, - 134, - 136, - 139, - 141, - 144, - 145, - 148, - 152, - 157, - 162, - 164, - 177, - 184, - 187, - 189, - 191, - 192, - 194, - 196, - 200, - 201, - 202, - 205, - 207, - 211, - 213, - 214, - 216, - 222, - 224, - 226, - 227, - 229, - 230, - 235, - 237, - 238, - 239, - 249, - 259, - 260, - 262, - 263, - 266, - 267, - 271, - 275, - 279, - 283, - 285, - 288, - 289, - 293, - 294, - 296, - 297, - 303, - 311, - 312, - 313, - 321, - 329, - 334, - 337, - 338, - 339, - 347, - 351, - 357, - 358, - 359, - 364, - 367, - 374, - 375, - 381, - 382, - 388, - 389, - 400, - 410, - 415, - 417, - 423, - 430, - 432, - 434, - 440, - 447, - 449, - 451, - 452, - 458, - 459, - 470, - 475, - 480, - 485, - 494, - 499, - 505, - 508, - 513, - 516, - 517, - 519, - 522, - 525, - 526, - 527, - 529, - 535, - 545, - 548, - 552, - 555, - 563, - 583, - 588, - 591, - 597, - 601, - 605, - 610, - 611, - 612, - 617, - 634, - 635, - 639, - 648, - 649, - 650, - 651, - 654, - 655, - 658, - 662, - 665, - 672, - 674, - 681, - 687, - 694, - 697, - 702, - 711, - 713, - 727, - 728, - 732, - 736, - 740, - 746, - 755, - 761, - 763, - 765, - 776, - 778, - 786, - 802, - 807, - 811, - 817, - 818, - 823, - 827, - 837, - 838, - 844, - 851, - 853, - 857, - 859, - 860, - 865, - 867, - 870, - 875, - 883, - 889, - 890, - 891, - 892, - 893 - ], - "survival": 1.0, - "id": "t09" - }, - { - "slice": "jfk_pu_wk4", - "ask": "How many complaints about trips picked up at JFK Airport (zone 132) picked up from January 22 to January 31 inclusive are mainly about lost items?", - "kind": "count", - "tol": 0.1, - "answer": 32, - "needs": [ - 25, - 38, - 56, - 59, - 61, - 63, - 66, - 70, - 73, - 89, - 107, - 117, - 120, - 125, - 143, - 149, - 171, - 173, - 175, - 188, - 190, - 203, - 218, - 220, - 231, - 236, - 243, - 252, - 253, - 261, - 284, - 295, - 298, - 302, - 304, - 316, - 322, - 323, - 324, - 326, - 327, - 328, - 333, - 361, - 368, - 378, - 379, - 386, - 387, - 392, - 395, - 397, - 399, - 407, - 409, - 413, - 426, - 428, - 444, - 455, - 461, - 462, - 464, - 471, - 477, - 482, - 486, - 491, - 492, - 504, - 512, - 520, - 532, - 533, - 539, - 543, - 547, - 551, - 556, - 557, - 558, - 567, - 570, - 571, - 584, - 585, - 592, - 602, - 607, - 613, - 614, - 616, - 621, - 623, - 624, - 631, - 637, - 643, - 647, - 656, - 659, - 676, - 678, - 689, - 691, - 693, - 696, - 698, - 699, - 706, - 716, - 721, - 723, - 724, - 725, - 731, - 741, - 745, - 750, - 764, - 775, - 791, - 794, - 797, - 808, - 813, - 821, - 841, - 843, - 849, - 858, - 863, - 877 - ], - "survival": 0.965, - "id": "t10" - }, - { - "slice": "midtown_wk3", - "ask": "How many complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 15 to January 21 inclusive are mainly about safety?", - "kind": "count", - "tol": 0.1, - "answer": 34, - "needs": [ - 2, - 40, - 80, - 94, - 99, - 228, - 346, - 350, - 360, - 411, - 523, - 566, - 582, - 593, - 671, - 695, - 756, - 805, - 809, - 878, - 884, - 904, - 909, - 911, - 914, - 916, - 923, - 924, - 926, - 928, - 936, - 937, - 950, - 951, - 957, - 961, - 969, - 972, - 974, - 991, - 996, - 1000, - 1013, - 1016, - 1022, - 1026, - 1027, - 1039, - 1042, - 1043, - 1045, - 1055, - 1063, - 1069, - 1070, - 1071, - 1074, - 1075, - 1078, - 1093, - 1095, - 1103, - 1111, - 1116, - 1119, - 1121, - 1122, - 1126, - 1136, - 1139, - 1144, - 1146, - 1150, - 1151, - 1154, - 1165, - 1168, - 1170, - 1171, - 1181, - 1183, - 1188, - 1192, - 1193, - 1208, - 1209, - 1221, - 1224, - 1225, - 1229, - 1233, - 1241, - 1251, - 1255, - 1258, - 1259, - 1260, - 1261, - 1267, - 1269, - 1270, - 1279, - 1282, - 1283, - 1285, - 1293, - 1294, - 1309, - 1310, - 1313, - 1324, - 1330, - 1334, - 1338, - 1339, - 1343, - 1349, - 1357, - 1360, - 1364, - 1369, - 1370, - 1373, - 1374, - 1379, - 1380, - 1385, - 1386, - 1388, - 1390, - 1391, - 1395, - 1398, - 1409, - 1417, - 1427, - 1432, - 1433, - 1434, - 1437, - 1441, - 1452, - 1454, - 1457, - 1463, - 1468, - 1472, - 1474, - 1481, - 1484, - 1486, - 1489, - 1493, - 1496, - 1502, - 1505, - 1509, - 1520, - 1521, - 1522, - 1530, - 1534, - 1535, - 1536, - 1540, - 1541, - 1546, - 1547, - 1548, - 1550, - 1553, - 1561, - 1572, - 1585, - 1592, - 1596, - 1600, - 1836, - 1860, - 1884, - 1896, - 1918, - 1927, - 1936, - 1944, - 1997, - 1998 - ], - "survival": 0.96, - "id": "t11" - }, - { - "slice": "tsq", - "ask": "How many complaints about trips picked up in Times Sq/Theatre District (zone 230) are mainly about driver conduct?", - "kind": "count", - "tol": 0.1, - "answer": 18, - "needs": [ - 1, - 42, - 82, - 94, - 122, - 126, - 162, - 189, - 219, - 228, - 246, - 360, - 412, - 429, - 434, - 449, - 478, - 518, - 526, - 589, - 671, - 695, - 759, - 776, - 809, - 866, - 882, - 898, - 901, - 907, - 917, - 920, - 925, - 932, - 949, - 959, - 985, - 997, - 1006, - 1007, - 1015, - 1035, - 1047, - 1055, - 1057, - 1067, - 1104, - 1112, - 1119, - 1126, - 1135, - 1169, - 1181, - 1210, - 1216, - 1235, - 1245, - 1248, - 1258, - 1266, - 1270, - 1278, - 1283, - 1307, - 1322, - 1332, - 1344, - 1346, - 1358, - 1362, - 1363, - 1365, - 1372, - 1376, - 1377, - 1400, - 1414, - 1415, - 1425, - 1427, - 1431, - 1435, - 1465, - 1470, - 1479, - 1496, - 1502, - 1519, - 1521, - 1553, - 1561, - 1563, - 1573, - 1581, - 1864, - 1882, - 1898, - 1930, - 1949, - 1956, - 1977, - 1979 - ], - "survival": 0.965, - "id": "t12" - }, - { - "slice": "jfk_pu_wk1", - "ask": "How many complaints about trips picked up at JFK Airport (zone 132) picked up from January 1 to January 7 inclusive are mainly about fare disputes?", - "kind": "count", - "tol": 0.1, - "answer": 32, - "needs": [ - 4, - 11, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 44, - 50, - 51, - 58, - 65, - 67, - 75, - 76, - 81, - 84, - 85, - 92, - 103, - 112, - 127, - 129, - 134, - 136, - 141, - 148, - 152, - 177, - 184, - 187, - 191, - 194, - 196, - 200, - 201, - 202, - 207, - 211, - 213, - 216, - 222, - 227, - 229, - 230, - 238, - 239, - 249, - 259, - 260, - 267, - 271, - 275, - 294, - 303, - 311, - 321, - 334, - 337, - 347, - 357, - 359, - 367, - 374, - 375, - 382, - 417, - 430, - 440, - 459, - 470, - 485, - 494, - 508, - 513, - 522, - 527, - 529, - 535, - 545, - 548, - 552, - 563, - 588, - 591, - 612, - 634, - 649, - 650, - 651, - 655, - 662, - 665, - 672, - 681, - 694, - 702, - 713, - 727, - 728, - 732, - 740, - 746, - 755, - 761, - 765, - 786, - 811, - 818, - 823, - 838, - 851, - 853, - 860, - 870, - 875, - 883, - 889, - 890, - 891 - ], - "survival": 0.965, - "id": "t13" - }, - { - "slice": "tsq", - "ask": "Among complaints about trips picked up in Times Sq/Theatre District (zone 230), what percentage are mainly about fare disputes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 19.6, - "needs": [ - 1, - 42, - 82, - 94, - 122, - 126, - 162, - 189, - 219, - 228, - 246, - 360, - 412, - 429, - 434, - 449, - 478, - 518, - 526, - 589, - 671, - 695, - 759, - 776, - 809, - 866, - 882, - 898, - 901, - 907, - 917, - 920, - 925, - 932, - 949, - 959, - 985, - 997, - 1006, - 1007, - 1015, - 1035, - 1047, - 1055, - 1057, - 1067, - 1104, - 1112, - 1119, - 1126, - 1135, - 1169, - 1181, - 1210, - 1216, - 1235, - 1245, - 1248, - 1258, - 1266, - 1270, - 1278, - 1283, - 1307, - 1322, - 1332, - 1344, - 1346, - 1358, - 1362, - 1363, - 1365, - 1372, - 1376, - 1377, - 1400, - 1414, - 1415, - 1425, - 1427, - 1431, - 1435, - 1465, - 1470, - 1479, - 1496, - 1502, - 1519, - 1521, - 1553, - 1561, - 1563, - 1573, - 1581, - 1864, - 1882, - 1898, - 1930, - 1949, - 1956, - 1977, - 1979 - ], - "survival": 0.99, - "id": "t14" - }, - { - "slice": "jfk_pu_wk2", - "ask": "Among complaints about trips picked up at JFK Airport (zone 132) picked up from January 8 to January 14 inclusive, what percentage are mainly about driver conduct? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 9.3, - "needs": [ - 3, - 10, - 18, - 19, - 21, - 27, - 30, - 31, - 34, - 37, - 48, - 52, - 68, - 71, - 77, - 78, - 83, - 95, - 96, - 101, - 110, - 111, - 123, - 133, - 142, - 146, - 160, - 166, - 174, - 176, - 183, - 193, - 209, - 242, - 245, - 250, - 265, - 286, - 291, - 299, - 317, - 318, - 325, - 335, - 343, - 348, - 356, - 363, - 366, - 377, - 383, - 398, - 406, - 420, - 436, - 456, - 472, - 473, - 474, - 479, - 495, - 506, - 509, - 534, - 536, - 550, - 554, - 560, - 595, - 599, - 600, - 620, - 628, - 652, - 666, - 677, - 726, - 734, - 744, - 754, - 758, - 767, - 779, - 780, - 793, - 796, - 812, - 820, - 828, - 831, - 842, - 846, - 856, - 861, - 876, - 880, - 896 - ], - "survival": 0.98, - "id": "t15" - }, - { - "slice": "jfk_pu_wk4", - "ask": "Among complaints about trips picked up at JFK Airport (zone 132) picked up from January 22 to January 31 inclusive, what percentage are mainly about lost items? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 24.1, - "needs": [ - 25, - 38, - 56, - 59, - 61, - 63, - 66, - 70, - 73, - 89, - 107, - 117, - 120, - 125, - 143, - 149, - 171, - 173, - 175, - 188, - 190, - 203, - 218, - 220, - 231, - 236, - 243, - 252, - 253, - 261, - 284, - 295, - 298, - 302, - 304, - 316, - 322, - 323, - 324, - 326, - 327, - 328, - 333, - 361, - 368, - 378, - 379, - 386, - 387, - 392, - 395, - 397, - 399, - 407, - 409, - 413, - 426, - 428, - 444, - 455, - 461, - 462, - 464, - 471, - 477, - 482, - 486, - 491, - 492, - 504, - 512, - 520, - 532, - 533, - 539, - 543, - 547, - 551, - 556, - 557, - 558, - 567, - 570, - 571, - 584, - 585, - 592, - 602, - 607, - 613, - 614, - 616, - 621, - 623, - 624, - 631, - 637, - 643, - 647, - 656, - 659, - 676, - 678, - 689, - 691, - 693, - 696, - 698, - 699, - 706, - 716, - 721, - 723, - 724, - 725, - 731, - 741, - 745, - 750, - 764, - 775, - 791, - 794, - 797, - 808, - 813, - 821, - 841, - 843, - 849, - 858, - 863, - 877 - ], - "survival": 1.0, - "id": "t16" - }, - { - "slice": "night_wk1", - "ask": "Among complaints about trips picked up from January 1 to January 7 inclusive, picked up between 22:00 and 04:59, what percentage are mainly about detours or bad routes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 10.3, - "needs": [ - 8, - 14, - 17, - 36, - 44, - 50, - 58, - 76, - 85, - 92, - 114, - 141, - 157, - 177, - 222, - 224, - 249, - 263, - 275, - 303, - 359, - 367, - 374, - 417, - 459, - 508, - 516, - 535, - 545, - 591, - 635, - 651, - 654, - 662, - 697, - 711, - 713, - 827, - 851, - 867, - 889, - 912, - 947, - 1004, - 1015, - 1092, - 1099, - 1100, - 1115, - 1159, - 1186, - 1194, - 1207, - 1352, - 1405, - 1421, - 1519, - 1527, - 1577, - 1605, - 1612, - 1613, - 1614, - 1618, - 1623, - 1624, - 1630, - 1631, - 1632, - 1636, - 1639, - 1646, - 1648, - 1650, - 1658, - 1659, - 1665, - 1674, - 1682, - 1689, - 1690, - 1692, - 1694, - 1701, - 1702, - 1711, - 1712, - 1716, - 1724, - 1738, - 1747, - 1749, - 1751, - 1756, - 1762, - 1763, - 1764, - 1767, - 1768, - 1769, - 1774, - 1778, - 1780, - 1781, - 1782, - 1785, - 1787, - 1792, - 1796, - 1799, - 1806, - 1807, - 1808, - 1816, - 1819, - 1829, - 1831, - 1837, - 1845, - 1848, - 1850, - 1853, - 1854, - 1857, - 1859, - 1862, - 1864, - 1871, - 1872, - 1877, - 1887, - 1888, - 1897, - 1909, - 1912, - 1919, - 1921, - 1922, - 1923, - 1924, - 1929, - 1931, - 1932, - 1945, - 1948, - 1955, - 1961, - 1974, - 1975, - 1977, - 1978, - 1979, - 1980, - 1988, - 1991, - 1999 - ], - "survival": 0.995, - "id": "t17" - }, - { - "slice": "night_wk4", - "ask": "Among complaints about trips picked up from January 22 to January 31 inclusive, picked up between 22:00 and 04:59, what percentage are mainly about lost items? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 10.8, - "needs": [ - 24, - 60, - 66, - 98, - 116, - 120, - 236, - 284, - 298, - 304, - 326, - 394, - 407, - 409, - 462, - 478, - 520, - 551, - 581, - 585, - 602, - 621, - 660, - 691, - 693, - 724, - 747, - 750, - 764, - 782, - 849, - 871, - 902, - 933, - 939, - 955, - 956, - 1018, - 1023, - 1029, - 1061, - 1084, - 1090, - 1166, - 1167, - 1185, - 1217, - 1218, - 1223, - 1234, - 1249, - 1263, - 1303, - 1311, - 1321, - 1354, - 1372, - 1375, - 1392, - 1424, - 1436, - 1443, - 1464, - 1508, - 1513, - 1554, - 1558, - 1587, - 1594, - 1597, - 1606, - 1607, - 1608, - 1609, - 1615, - 1617, - 1619, - 1621, - 1627, - 1628, - 1640, - 1657, - 1660, - 1667, - 1668, - 1673, - 1675, - 1676, - 1677, - 1681, - 1684, - 1685, - 1686, - 1687, - 1695, - 1697, - 1700, - 1705, - 1708, - 1713, - 1714, - 1717, - 1721, - 1722, - 1723, - 1727, - 1730, - 1731, - 1732, - 1735, - 1739, - 1740, - 1742, - 1744, - 1745, - 1746, - 1748, - 1754, - 1765, - 1770, - 1771, - 1772, - 1775, - 1786, - 1789, - 1790, - 1791, - 1793, - 1794, - 1795, - 1797, - 1798, - 1805, - 1810, - 1812, - 1813, - 1814, - 1817, - 1820, - 1822, - 1826, - 1828, - 1830, - 1832, - 1834, - 1839, - 1840, - 1841, - 1849, - 1855, - 1863, - 1865, - 1873, - 1875, - 1878, - 1882, - 1885, - 1892, - 1895, - 1898, - 1900, - 1902, - 1905, - 1906, - 1910, - 1911, - 1913, - 1914, - 1915, - 1916, - 1920, - 1928, - 1930, - 1938, - 1949, - 1954, - 1956, - 1957, - 1958, - 1960, - 1963, - 1964, - 1965, - 1967, - 1968, - 1970, - 1972, - 1973, - 1976, - 1981, - 1982, - 1983, - 1989, - 1994 - ], - "survival": 1.0, - "id": "t18" - }, - { - "slice": "night_wk2", - "ask": "Among complaints about trips picked up from January 8 to January 14 inclusive, picked up between 22:00 and 04:59, what percentage are mainly about safety? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 26.0, - "needs": [ - 10, - 16, - 27, - 68, - 110, - 163, - 166, - 176, - 181, - 245, - 348, - 352, - 363, - 366, - 398, - 418, - 463, - 472, - 473, - 495, - 497, - 509, - 515, - 536, - 628, - 726, - 752, - 758, - 783, - 830, - 934, - 1024, - 1031, - 1080, - 1134, - 1140, - 1198, - 1213, - 1244, - 1248, - 1327, - 1346, - 1359, - 1377, - 1381, - 1425, - 1439, - 1477, - 1511, - 1528, - 1581, - 1601, - 1602, - 1603, - 1604, - 1610, - 1611, - 1620, - 1622, - 1625, - 1626, - 1629, - 1633, - 1634, - 1635, - 1638, - 1645, - 1654, - 1655, - 1661, - 1664, - 1666, - 1678, - 1679, - 1680, - 1683, - 1688, - 1693, - 1696, - 1698, - 1699, - 1704, - 1710, - 1718, - 1719, - 1725, - 1726, - 1734, - 1736, - 1750, - 1757, - 1760, - 1761, - 1773, - 1777, - 1783, - 1784, - 1788, - 1801, - 1802, - 1803, - 1823, - 1825, - 1835, - 1838, - 1842, - 1843, - 1844, - 1847, - 1852, - 1856, - 1858, - 1866, - 1867, - 1869, - 1870, - 1874, - 1876, - 1879, - 1881, - 1886, - 1889, - 1901, - 1903, - 1907, - 1917, - 1925, - 1926, - 1933, - 1934, - 1939, - 1940, - 1941, - 1943, - 1950, - 1951, - 1953, - 1959, - 1962, - 1966, - 1971, - 1985, - 1986, - 1987, - 1990, - 1992, - 1993, - 1995, - 1996, - 2000 - ], - "survival": 0.98, - "id": "t19" - }, - { - "slice": "night_air", - "ask": "Which category of complaint is most common among complaints about airport trips (to or from zones 1, 132, 138) picked up between 22:00 and 04:59? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "fare_dispute", - "needs": [ - 8, - 10, - 12, - 14, - 16, - 17, - 23, - 24, - 27, - 36, - 44, - 50, - 58, - 60, - 66, - 68, - 76, - 85, - 92, - 94, - 98, - 102, - 110, - 114, - 116, - 118, - 120, - 131, - 141, - 157, - 163, - 166, - 167, - 176, - 177, - 178, - 181, - 197, - 217, - 222, - 224, - 236, - 241, - 245, - 249, - 251, - 263, - 264, - 270, - 275, - 277, - 284, - 298, - 303, - 304, - 326, - 342, - 348, - 352, - 359, - 363, - 365, - 366, - 367, - 374, - 380, - 394, - 398, - 404, - 407, - 409, - 417, - 418, - 442, - 445, - 459, - 462, - 463, - 467, - 472, - 473, - 478, - 495, - 497, - 508, - 509, - 510, - 515, - 516, - 520, - 535, - 536, - 545, - 551, - 581, - 585, - 591, - 602, - 621, - 628, - 635, - 651, - 654, - 660, - 662, - 664, - 675, - 691, - 693, - 697, - 703, - 707, - 709, - 711, - 713, - 724, - 726, - 747, - 749, - 750, - 752, - 758, - 764, - 768, - 782, - 783, - 805, - 827, - 830, - 834, - 845, - 849, - 851, - 867, - 871, - 874, - 886, - 887, - 889 - ], - "survival": 1.0, - "id": "t20" - }, - { - "slice": "night_wk3", - "ask": "Which category of complaint is most common among complaints about trips picked up from January 15 to January 21 inclusive, picked up between 22:00 and 04:59? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "safety", - "needs": [ - 12, - 23, - 94, - 102, - 118, - 131, - 167, - 178, - 197, - 217, - 241, - 251, - 264, - 270, - 277, - 342, - 365, - 380, - 404, - 442, - 445, - 467, - 510, - 664, - 675, - 703, - 707, - 709, - 749, - 768, - 805, - 834, - 845, - 874, - 886, - 887, - 904, - 923, - 928, - 972, - 1013, - 1075, - 1146, - 1165, - 1170, - 1208, - 1225, - 1267, - 1339, - 1343, - 1437, - 1463, - 1472, - 1521, - 1536, - 1561, - 1616, - 1637, - 1641, - 1642, - 1643, - 1644, - 1647, - 1649, - 1651, - 1652, - 1653, - 1656, - 1662, - 1663, - 1669, - 1670, - 1671, - 1672, - 1691, - 1703, - 1706, - 1707, - 1709, - 1715, - 1720, - 1728, - 1729, - 1733, - 1737, - 1741, - 1743, - 1752, - 1753, - 1755, - 1758, - 1759, - 1766, - 1776, - 1779, - 1800, - 1804, - 1809, - 1811, - 1815, - 1818, - 1821, - 1824, - 1827, - 1833, - 1836, - 1846, - 1851, - 1860, - 1861, - 1868, - 1880, - 1883, - 1884, - 1890, - 1891, - 1893, - 1894, - 1896, - 1899, - 1904, - 1908, - 1918, - 1927, - 1935, - 1936, - 1937, - 1942, - 1944, - 1946, - 1947, - 1952, - 1969, - 1984, - 1997, - 1998 - ], - "survival": 1.0, - "id": "t21" - }, - { - "slice": "lga_1_15", - "ask": "Among complaints about trips to or from LaGuardia Airport (zone 138) picked up from January 1 to January 15 inclusive, what percentage are mainly about safety? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 11.3, - "needs": [ - 1, - 8, - 14, - 16, - 31, - 43, - 53, - 54, - 57, - 64, - 72, - 74, - 86, - 88, - 91, - 100, - 104, - 106, - 113, - 114, - 115, - 122, - 124, - 135, - 137, - 139, - 144, - 145, - 157, - 164, - 169, - 172, - 180, - 181, - 182, - 205, - 206, - 214, - 224, - 226, - 237, - 240, - 247, - 262, - 263, - 266, - 279, - 283, - 285, - 288, - 289, - 291, - 296, - 297, - 305, - 308, - 313, - 315, - 331, - 338, - 351, - 352, - 354, - 360, - 364, - 369, - 370, - 371, - 388, - 389, - 391, - 410, - 412, - 414, - 415, - 416, - 418, - 423, - 425, - 429, - 432, - 437, - 443, - 447, - 448, - 449, - 451, - 452, - 453, - 454, - 463, - 475, - 480, - 497, - 498, - 499, - 501, - 503, - 505, - 511, - 514, - 516, - 517, - 519, - 525, - 546, - 549, - 559, - 576, - 580, - 587, - 597, - 601, - 610, - 617, - 619, - 635, - 639, - 640, - 648, - 654, - 658, - 666, - 669, - 685, - 687, - 697, - 701, - 705, - 711, - 718, - 736, - 752, - 761, - 763, - 769, - 778, - 787, - 792, - 798, - 799, - 800, - 801, - 802, - 807, - 814, - 816, - 817, - 820, - 822, - 825, - 827, - 829, - 830, - 833, - 854, - 859, - 862, - 864, - 865, - 866, - 867, - 869, - 873, - 881, - 892, - 893, - 900 - ], - "survival": 0.995, - "id": "t22" - }, - { - "slice": "midtown_wk2", - "ask": "Among complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 8 to January 14 inclusive, what percentage are mainly about lost items? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 21.0, - "needs": [ - 1, - 46, - 122, - 124, - 163, - 169, - 247, - 315, - 412, - 414, - 429, - 431, - 437, - 454, - 463, - 515, - 518, - 576, - 632, - 799, - 866, - 873, - 901, - 905, - 906, - 907, - 919, - 921, - 929, - 934, - 946, - 952, - 953, - 960, - 965, - 975, - 976, - 977, - 979, - 982, - 987, - 989, - 990, - 993, - 997, - 1002, - 1003, - 1007, - 1008, - 1014, - 1024, - 1031, - 1040, - 1046, - 1064, - 1065, - 1066, - 1073, - 1076, - 1079, - 1080, - 1081, - 1082, - 1083, - 1089, - 1096, - 1097, - 1101, - 1107, - 1108, - 1113, - 1114, - 1130, - 1134, - 1140, - 1158, - 1169, - 1172, - 1175, - 1176, - 1179, - 1189, - 1195, - 1196, - 1198, - 1199, - 1201, - 1212, - 1213, - 1214, - 1216, - 1226, - 1227, - 1231, - 1237, - 1243, - 1244, - 1248, - 1250, - 1252, - 1272, - 1274, - 1290, - 1298, - 1302, - 1305, - 1314, - 1319, - 1320, - 1322, - 1325, - 1327, - 1328, - 1331, - 1332, - 1340, - 1341, - 1342, - 1344, - 1345, - 1346, - 1350, - 1359, - 1363, - 1377, - 1381, - 1383, - 1401, - 1402, - 1406, - 1408, - 1412, - 1413, - 1416, - 1418, - 1419, - 1425, - 1431, - 1435, - 1439, - 1440, - 1442, - 1444, - 1447, - 1471, - 1473, - 1477, - 1479, - 1482, - 1497, - 1498, - 1511, - 1514, - 1523, - 1524, - 1525, - 1528, - 1529, - 1533, - 1537, - 1538, - 1556, - 1560, - 1571, - 1573, - 1574, - 1575, - 1578, - 1579, - 1580, - 1581, - 1582, - 1589, - 1838, - 1867, - 1874, - 1907, - 1950, - 1959, - 1966, - 1992 - ], - "survival": 0.995, - "id": "t23" - }, - { - "slice": "midtown_wk3", - "ask": "Among complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 15 to January 21 inclusive, what percentage are mainly about driver conduct? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 13.4, - "needs": [ - 2, - 40, - 80, - 94, - 99, - 228, - 346, - 350, - 360, - 411, - 523, - 566, - 582, - 593, - 671, - 695, - 756, - 805, - 809, - 878, - 884, - 904, - 909, - 911, - 914, - 916, - 923, - 924, - 926, - 928, - 936, - 937, - 950, - 951, - 957, - 961, - 969, - 972, - 974, - 991, - 996, - 1000, - 1013, - 1016, - 1022, - 1026, - 1027, - 1039, - 1042, - 1043, - 1045, - 1055, - 1063, - 1069, - 1070, - 1071, - 1074, - 1075, - 1078, - 1093, - 1095, - 1103, - 1111, - 1116, - 1119, - 1121, - 1122, - 1126, - 1136, - 1139, - 1144, - 1146, - 1150, - 1151, - 1154, - 1165, - 1168, - 1170, - 1171, - 1181, - 1183, - 1188, - 1192, - 1193, - 1208, - 1209, - 1221, - 1224, - 1225, - 1229, - 1233, - 1241, - 1251, - 1255, - 1258, - 1259, - 1260, - 1261, - 1267, - 1269, - 1270, - 1279, - 1282, - 1283, - 1285, - 1293, - 1294, - 1309, - 1310, - 1313, - 1324, - 1330, - 1334, - 1338, - 1339, - 1343, - 1349, - 1357, - 1360, - 1364, - 1369, - 1370, - 1373, - 1374, - 1379, - 1380, - 1385, - 1386, - 1388, - 1390, - 1391, - 1395, - 1398, - 1409, - 1417, - 1427, - 1432, - 1433, - 1434, - 1437, - 1441, - 1452, - 1454, - 1457, - 1463, - 1468, - 1472, - 1474, - 1481, - 1484, - 1486, - 1489, - 1493, - 1496, - 1502, - 1505, - 1509, - 1520, - 1521, - 1522, - 1530, - 1534, - 1535, - 1536, - 1540, - 1541, - 1546, - 1547, - 1548, - 1550, - 1553, - 1561, - 1572, - 1585, - 1592, - 1596, - 1600, - 1836, - 1860, - 1884, - 1896, - 1918, - 1927, - 1936, - 1944, - 1997, - 1998 - ], - "survival": 1.0, - "id": "t24" - }, - { - "slice": "night_midtown", - "ask": "Among complaints about trips picked up in Midtown (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) between 22:00 and 04:59, what percentage are mainly about fare disputes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 11.8, - "needs": [ - 94, - 163, - 394, - 463, - 478, - 515, - 805, - 902, - 904, - 912, - 923, - 928, - 933, - 934, - 939, - 947, - 955, - 956, - 972, - 1004, - 1013, - 1015, - 1018, - 1023, - 1024, - 1029, - 1031, - 1061, - 1075, - 1080, - 1084, - 1090, - 1092, - 1099, - 1100, - 1115, - 1134, - 1140, - 1146, - 1159, - 1165, - 1166, - 1167, - 1170, - 1185, - 1186, - 1194, - 1198, - 1207, - 1208, - 1213, - 1217, - 1218, - 1223, - 1225, - 1234, - 1244, - 1248, - 1249, - 1263, - 1267, - 1303, - 1311, - 1321, - 1327, - 1339, - 1343, - 1346, - 1352, - 1354, - 1359, - 1372, - 1375, - 1377, - 1381, - 1392, - 1405, - 1421, - 1424, - 1425, - 1436, - 1437, - 1439, - 1443, - 1463, - 1464, - 1472, - 1477, - 1508, - 1511, - 1513, - 1519, - 1521, - 1527, - 1528, - 1536, - 1554, - 1558, - 1561, - 1577, - 1581, - 1587, - 1594, - 1597, - 1836, - 1838, - 1840, - 1848, - 1850, - 1857, - 1860, - 1862, - 1864, - 1867, - 1874, - 1877, - 1878, - 1882, - 1884, - 1896, - 1898, - 1907, - 1913, - 1918, - 1919, - 1920, - 1921, - 1927, - 1930, - 1932, - 1936, - 1944, - 1945, - 1949, - 1950, - 1955, - 1956, - 1959, - 1960, - 1963, - 1966, - 1968, - 1970, - 1972, - 1977, - 1979, - 1988, - 1989, - 1992, - 1997, - 1998, - 1999 - ], - "survival": 1.0, - "id": "t25" - }, - { - "slice": "night_wk2", - "ask": "What is the median total_amount of trips picked up from January 8 to January 14 inclusive, picked up between 22:00 and 04:59 that drew a complaint mainly about safety? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 22.88, - "needs": [ - 10, - 16, - 27, - 68, - 110, - 163, - 166, - 176, - 181, - 245, - 348, - 352, - 363, - 366, - 398, - 418, - 463, - 472, - 473, - 495, - 497, - 509, - 515, - 536, - 628, - 726, - 752, - 758, - 783, - 830, - 934, - 1024, - 1031, - 1080, - 1134, - 1140, - 1198, - 1213, - 1244, - 1248, - 1327, - 1346, - 1359, - 1377, - 1381, - 1425, - 1439, - 1477, - 1511, - 1528, - 1581, - 1601, - 1602, - 1603, - 1604, - 1610, - 1611, - 1620, - 1622, - 1625, - 1626, - 1629, - 1633, - 1634, - 1635, - 1638, - 1645, - 1654, - 1655, - 1661, - 1664, - 1666, - 1678, - 1679, - 1680, - 1683, - 1688, - 1693, - 1696, - 1698, - 1699, - 1704, - 1710, - 1718, - 1719, - 1725, - 1726, - 1734, - 1736, - 1750, - 1757, - 1760, - 1761, - 1773, - 1777, - 1783, - 1784, - 1788, - 1801, - 1802, - 1803, - 1823, - 1825, - 1835, - 1838, - 1842, - 1843, - 1844, - 1847, - 1852, - 1856, - 1858, - 1866, - 1867, - 1869, - 1870, - 1874, - 1876, - 1879, - 1881, - 1886, - 1889, - 1901, - 1903, - 1907, - 1917, - 1925, - 1926, - 1933, - 1934, - 1939, - 1940, - 1941, - 1943, - 1950, - 1951, - 1953, - 1959, - 1962, - 1966, - 1971, - 1985, - 1986, - 1987, - 1990, - 1992, - 1993, - 1995, - 1996, - 2000 - ], - "survival": 0.99, - "id": "t26" - }, - { - "slice": "air_wk3", - "ask": "What is the median total_amount of airport trips (to or from zones 1, 132, 138) picked up from January 15 to January 21 inclusive that drew a complaint mainly about vehicle condition? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 79.91, - "needs": [ - 2, - 6, - 12, - 22, - 23, - 28, - 29, - 40, - 49, - 69, - 79, - 80, - 90, - 91, - 93, - 94, - 97, - 99, - 100, - 102, - 108, - 118, - 121, - 131, - 132, - 147, - 155, - 158, - 159, - 165, - 167, - 178, - 185, - 186, - 195, - 197, - 198, - 199, - 204, - 210, - 217, - 223, - 225, - 228, - 232, - 241, - 248, - 251, - 258, - 264, - 268, - 269, - 270, - 273, - 277, - 278, - 282, - 287, - 290, - 292, - 306, - 307, - 309, - 310, - 314, - 320, - 330, - 336, - 341, - 342, - 344, - 345, - 346, - 350, - 360, - 365, - 372, - 380, - 385, - 401, - 403, - 404, - 408, - 411, - 421, - 442, - 445, - 446, - 467, - 468, - 481, - 483, - 489, - 493, - 507, - 510, - 521, - 523, - 524, - 528, - 530, - 531, - 537, - 540, - 541, - 549, - 553, - 562, - 564, - 565, - 566, - 572, - 573, - 574, - 575, - 578, - 579, - 582, - 587, - 593, - 594, - 596, - 598, - 603, - 609, - 625, - 627, - 633, - 636, - 638, - 644, - 646, - 661, - 663, - 664, - 670, - 671, - 675, - 679, - 682, - 683, - 686, - 692, - 695, - 703, - 704, - 707, - 708, - 709, - 710, - 717, - 718, - 730, - 735, - 737, - 742, - 743, - 748, - 749, - 753, - 756, - 757, - 768, - 771, - 772, - 774, - 777, - 784, - 789, - 803, - 805, - 806, - 809, - 815, - 819, - 834, - 835, - 836, - 839, - 840, - 845, - 847, - 848, - 850, - 852, - 855, - 868, - 874, - 878, - 884, - 886, - 887, - 888, - 895, - 897, - 899 - ], - "survival": 0.975, - "id": "t27" - }, - { - "slice": "lga_16_31", - "ask": "What is the median total_amount of trips to or from LaGuardia Airport (zone 138) picked up from January 16 to January 31 inclusive that drew a complaint mainly about driver conduct? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 69.89, - "needs": [ - 6, - 9, - 22, - 23, - 24, - 33, - 39, - 40, - 42, - 45, - 47, - 49, - 55, - 60, - 62, - 82, - 93, - 94, - 98, - 99, - 116, - 119, - 121, - 126, - 130, - 131, - 138, - 140, - 147, - 150, - 151, - 153, - 156, - 158, - 161, - 168, - 170, - 178, - 186, - 204, - 210, - 212, - 215, - 228, - 232, - 233, - 234, - 241, - 244, - 248, - 254, - 255, - 256, - 257, - 264, - 269, - 272, - 273, - 274, - 276, - 280, - 290, - 300, - 301, - 309, - 314, - 319, - 320, - 330, - 332, - 340, - 341, - 342, - 346, - 349, - 355, - 365, - 373, - 384, - 393, - 394, - 402, - 403, - 404, - 405, - 411, - 419, - 422, - 424, - 427, - 433, - 435, - 438, - 439, - 441, - 450, - 457, - 460, - 465, - 466, - 469, - 476, - 481, - 484, - 487, - 488, - 489, - 490, - 496, - 502, - 507, - 510, - 521, - 528, - 531, - 538, - 541, - 542, - 544, - 561, - 562, - 564, - 566, - 568, - 569, - 574, - 577, - 581, - 586, - 589, - 598, - 603, - 604, - 608, - 609, - 615, - 618, - 622, - 626, - 627, - 629, - 630, - 641, - 642, - 644, - 645, - 646, - 653, - 657, - 660, - 661, - 664, - 667, - 668, - 670, - 675, - 680, - 683, - 684, - 695, - 708, - 712, - 714, - 715, - 717, - 719, - 720, - 722, - 733, - 737, - 738, - 739, - 742, - 743, - 747, - 748, - 751, - 760, - 762, - 766, - 770, - 773, - 774, - 781, - 782, - 785, - 790, - 803, - 805, - 809, - 810, - 815, - 826, - 832, - 834, - 835, - 839, - 847, - 850, - 855, - 871, - 872, - 874, - 879, - 888, - 899 - ], - "survival": 1.0, - "id": "t28" - }, - { - "slice": "jfk_do", - "ask": "What is the median total_amount of trips dropped off at JFK Airport (zone 132) that drew a complaint mainly about fare disputes? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 92.94, - "needs": [ - 2, - 5, - 7, - 13, - 35, - 41, - 46, - 73, - 79, - 80, - 87, - 105, - 107, - 109, - 128, - 137, - 152, - 154, - 162, - 163, - 179, - 186, - 189, - 192, - 195, - 199, - 208, - 221, - 235, - 246, - 281, - 282, - 287, - 293, - 312, - 314, - 331, - 337, - 339, - 344, - 353, - 358, - 364, - 376, - 381, - 383, - 390, - 396, - 403, - 432, - 434, - 458, - 478, - 483, - 500, - 515, - 518, - 526, - 578, - 582, - 583, - 593, - 606, - 611, - 671, - 688, - 690, - 700, - 729, - 753, - 756, - 757, - 759, - 776, - 777, - 780, - 788, - 790, - 795, - 824, - 837, - 844, - 848, - 857, - 874, - 878, - 882, - 884, - 885, - 894, - 898 - ], - "survival": 1.0, - "id": "t29" - }, - { - "slice": "night_midtown", - "ask": "How many complaints about trips picked up in Midtown (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) between 22:00 and 04:59 are mainly about safety?", - "kind": "count", - "tol": 0.1, - "answer": 50, - "needs": [ - 94, - 163, - 394, - 463, - 478, - 515, - 805, - 902, - 904, - 912, - 923, - 928, - 933, - 934, - 939, - 947, - 955, - 956, - 972, - 1004, - 1013, - 1015, - 1018, - 1023, - 1024, - 1029, - 1031, - 1061, - 1075, - 1080, - 1084, - 1090, - 1092, - 1099, - 1100, - 1115, - 1134, - 1140, - 1146, - 1159, - 1165, - 1166, - 1167, - 1170, - 1185, - 1186, - 1194, - 1198, - 1207, - 1208, - 1213, - 1217, - 1218, - 1223, - 1225, - 1234, - 1244, - 1248, - 1249, - 1263, - 1267, - 1303, - 1311, - 1321, - 1327, - 1339, - 1343, - 1346, - 1352, - 1354, - 1359, - 1372, - 1375, - 1377, - 1381, - 1392, - 1405, - 1421, - 1424, - 1425, - 1436, - 1437, - 1439, - 1443, - 1463, - 1464, - 1472, - 1477, - 1508, - 1511, - 1513, - 1519, - 1521, - 1527, - 1528, - 1536, - 1554, - 1558, - 1561, - 1577, - 1581, - 1587, - 1594, - 1597, - 1836, - 1838, - 1840, - 1848, - 1850, - 1857, - 1860, - 1862, - 1864, - 1867, - 1874, - 1877, - 1878, - 1882, - 1884, - 1896, - 1898, - 1907, - 1913, - 1918, - 1919, - 1920, - 1921, - 1927, - 1930, - 1932, - 1936, - 1944, - 1945, - 1949, - 1950, - 1955, - 1956, - 1959, - 1960, - 1963, - 1966, - 1968, - 1970, - 1972, - 1977, - 1979, - 1988, - 1989, - 1992, - 1997, - 1998, - 1999 - ], - "survival": 0.98, - "id": "t30" - }, - { - "slice": "night_wk2", - "ask": "How many complaints about trips picked up from January 8 to January 14 inclusive, picked up between 22:00 and 04:59 are mainly about driver conduct?", - "kind": "count", - "tol": 0.1, - "answer": 35, - "needs": [ - 10, - 16, - 27, - 68, - 110, - 163, - 166, - 176, - 181, - 245, - 348, - 352, - 363, - 366, - 398, - 418, - 463, - 472, - 473, - 495, - 497, - 509, - 515, - 536, - 628, - 726, - 752, - 758, - 783, - 830, - 934, - 1024, - 1031, - 1080, - 1134, - 1140, - 1198, - 1213, - 1244, - 1248, - 1327, - 1346, - 1359, - 1377, - 1381, - 1425, - 1439, - 1477, - 1511, - 1528, - 1581, - 1601, - 1602, - 1603, - 1604, - 1610, - 1611, - 1620, - 1622, - 1625, - 1626, - 1629, - 1633, - 1634, - 1635, - 1638, - 1645, - 1654, - 1655, - 1661, - 1664, - 1666, - 1678, - 1679, - 1680, - 1683, - 1688, - 1693, - 1696, - 1698, - 1699, - 1704, - 1710, - 1718, - 1719, - 1725, - 1726, - 1734, - 1736, - 1750, - 1757, - 1760, - 1761, - 1773, - 1777, - 1783, - 1784, - 1788, - 1801, - 1802, - 1803, - 1823, - 1825, - 1835, - 1838, - 1842, - 1843, - 1844, - 1847, - 1852, - 1856, - 1858, - 1866, - 1867, - 1869, - 1870, - 1874, - 1876, - 1879, - 1881, - 1886, - 1889, - 1901, - 1903, - 1907, - 1917, - 1925, - 1926, - 1933, - 1934, - 1939, - 1940, - 1941, - 1943, - 1950, - 1951, - 1953, - 1959, - 1962, - 1966, - 1971, - 1985, - 1986, - 1987, - 1990, - 1992, - 1993, - 1995, - 1996, - 2000 - ], - "survival": 0.96, - "id": "t31" - }, - { - "slice": "air_wk3", - "ask": "How many complaints about airport trips (to or from zones 1, 132, 138) picked up from January 15 to January 21 inclusive are mainly about fare disputes?", - "kind": "count", - "tol": 0.1, - "answer": 52, - "needs": [ - 2, - 6, - 12, - 22, - 23, - 28, - 29, - 40, - 49, - 69, - 79, - 80, - 90, - 91, - 93, - 94, - 97, - 99, - 100, - 102, - 108, - 118, - 121, - 131, - 132, - 147, - 155, - 158, - 159, - 165, - 167, - 178, - 185, - 186, - 195, - 197, - 198, - 199, - 204, - 210, - 217, - 223, - 225, - 228, - 232, - 241, - 248, - 251, - 258, - 264, - 268, - 269, - 270, - 273, - 277, - 278, - 282, - 287, - 290, - 292, - 306, - 307, - 309, - 310, - 314, - 320, - 330, - 336, - 341, - 342, - 344, - 345, - 346, - 350, - 360, - 365, - 372, - 380, - 385, - 401, - 403, - 404, - 408, - 411, - 421, - 442, - 445, - 446, - 467, - 468, - 481, - 483, - 489, - 493, - 507, - 510, - 521, - 523, - 524, - 528, - 530, - 531, - 537, - 540, - 541, - 549, - 553, - 562, - 564, - 565, - 566, - 572, - 573, - 574, - 575, - 578, - 579, - 582, - 587, - 593, - 594, - 596, - 598, - 603, - 609, - 625, - 627, - 633, - 636, - 638, - 644, - 646, - 661, - 663, - 664, - 670, - 671, - 675, - 679, - 682, - 683, - 686, - 692, - 695, - 703, - 704, - 707, - 708, - 709, - 710, - 717, - 718, - 730, - 735, - 737, - 742, - 743, - 748, - 749, - 753, - 756, - 757, - 768, - 771, - 772, - 774, - 777, - 784, - 789, - 803, - 805, - 806, - 809, - 815, - 819, - 834, - 835, - 836, - 839, - 840, - 845, - 847, - 848, - 850, - 852, - 855, - 868, - 874, - 878, - 884, - 886, - 887, - 888, - 895, - 897, - 899 - ], - "survival": 0.995, - "id": "t32" - }, - { - "slice": "midtown_wk2", - "ask": "What is the median total_amount of Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 8 to January 14 inclusive that drew a complaint mainly about safety? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 20.0, - "needs": [ - 1, - 46, - 122, - 124, - 163, - 169, - 247, - 315, - 412, - 414, - 429, - 431, - 437, - 454, - 463, - 515, - 518, - 576, - 632, - 799, - 866, - 873, - 901, - 905, - 906, - 907, - 919, - 921, - 929, - 934, - 946, - 952, - 953, - 960, - 965, - 975, - 976, - 977, - 979, - 982, - 987, - 989, - 990, - 993, - 997, - 1002, - 1003, - 1007, - 1008, - 1014, - 1024, - 1031, - 1040, - 1046, - 1064, - 1065, - 1066, - 1073, - 1076, - 1079, - 1080, - 1081, - 1082, - 1083, - 1089, - 1096, - 1097, - 1101, - 1107, - 1108, - 1113, - 1114, - 1130, - 1134, - 1140, - 1158, - 1169, - 1172, - 1175, - 1176, - 1179, - 1189, - 1195, - 1196, - 1198, - 1199, - 1201, - 1212, - 1213, - 1214, - 1216, - 1226, - 1227, - 1231, - 1237, - 1243, - 1244, - 1248, - 1250, - 1252, - 1272, - 1274, - 1290, - 1298, - 1302, - 1305, - 1314, - 1319, - 1320, - 1322, - 1325, - 1327, - 1328, - 1331, - 1332, - 1340, - 1341, - 1342, - 1344, - 1345, - 1346, - 1350, - 1359, - 1363, - 1377, - 1381, - 1383, - 1401, - 1402, - 1406, - 1408, - 1412, - 1413, - 1416, - 1418, - 1419, - 1425, - 1431, - 1435, - 1439, - 1440, - 1442, - 1444, - 1447, - 1471, - 1473, - 1477, - 1479, - 1482, - 1497, - 1498, - 1511, - 1514, - 1523, - 1524, - 1525, - 1528, - 1529, - 1533, - 1537, - 1538, - 1556, - 1560, - 1571, - 1573, - 1574, - 1575, - 1578, - 1579, - 1580, - 1581, - 1582, - 1589, - 1838, - 1867, - 1874, - 1907, - 1950, - 1959, - 1966, - 1992 - ], - "survival": 0.965, - "id": "t33" - }, - { - "slice": "jfk_pu_wk3", - "ask": "What is the median total_amount of trips picked up at JFK Airport (zone 132) picked up from January 15 to January 21 inclusive that drew a complaint mainly about lost items? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 85.19, - "needs": [ - 12, - 28, - 29, - 69, - 90, - 97, - 102, - 108, - 118, - 132, - 155, - 159, - 165, - 167, - 185, - 197, - 198, - 217, - 225, - 251, - 258, - 268, - 269, - 270, - 277, - 278, - 282, - 292, - 306, - 307, - 310, - 336, - 345, - 372, - 380, - 385, - 401, - 408, - 421, - 442, - 445, - 446, - 467, - 468, - 493, - 524, - 530, - 537, - 540, - 553, - 565, - 572, - 573, - 575, - 579, - 594, - 596, - 625, - 636, - 638, - 663, - 679, - 682, - 686, - 692, - 703, - 704, - 707, - 709, - 710, - 730, - 735, - 749, - 768, - 771, - 772, - 784, - 789, - 806, - 819, - 836, - 840, - 845, - 852, - 868, - 886, - 887, - 895, - 897 - ], - "survival": 0.975, - "id": "t34" - }, - { - "slice": "jfk_pu_wk3", - "ask": "Among complaints about trips picked up at JFK Airport (zone 132) picked up from January 15 to January 21 inclusive, what percentage are mainly about detours or bad routes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 7.9, - "needs": [ - 12, - 28, - 29, - 69, - 90, - 97, - 102, - 108, - 118, - 132, - 155, - 159, - 165, - 167, - 185, - 197, - 198, - 217, - 225, - 251, - 258, - 268, - 269, - 270, - 277, - 278, - 282, - 292, - 306, - 307, - 310, - 336, - 345, - 372, - 380, - 385, - 401, - 408, - 421, - 442, - 445, - 446, - 467, - 468, - 493, - 524, - 530, - 537, - 540, - 553, - 565, - 572, - 573, - 575, - 579, - 594, - 596, - 625, - 636, - 638, - 663, - 679, - 682, - 686, - 692, - 703, - 704, - 707, - 709, - 710, - 730, - 735, - 749, - 768, - 771, - 772, - 784, - 789, - 806, - 819, - 836, - 840, - 845, - 852, - 868, - 886, - 887, - 895, - 897 - ], - "survival": 0.955, - "id": "t35" - }, - { - "slice": "midtown_wk1", - "ask": "How many complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 1 to January 7 inclusive are mainly about vehicle condition?", - "kind": "count", - "tol": 0.1, - "answer": 32, - "needs": [ - 5, - 162, - 189, - 192, - 205, - 293, - 351, - 400, - 434, - 449, - 526, - 555, - 601, - 658, - 674, - 763, - 776, - 837, - 844, - 903, - 912, - 932, - 938, - 942, - 947, - 954, - 959, - 963, - 967, - 970, - 978, - 980, - 981, - 986, - 988, - 995, - 998, - 1004, - 1009, - 1015, - 1032, - 1036, - 1041, - 1044, - 1052, - 1056, - 1067, - 1077, - 1085, - 1087, - 1088, - 1092, - 1094, - 1099, - 1100, - 1104, - 1106, - 1112, - 1115, - 1117, - 1120, - 1124, - 1125, - 1127, - 1132, - 1135, - 1138, - 1145, - 1147, - 1155, - 1159, - 1162, - 1163, - 1177, - 1178, - 1186, - 1190, - 1194, - 1197, - 1200, - 1203, - 1206, - 1207, - 1210, - 1211, - 1220, - 1228, - 1230, - 1232, - 1247, - 1256, - 1257, - 1271, - 1287, - 1292, - 1304, - 1316, - 1317, - 1336, - 1347, - 1352, - 1353, - 1356, - 1358, - 1361, - 1362, - 1387, - 1400, - 1404, - 1405, - 1410, - 1415, - 1421, - 1426, - 1428, - 1430, - 1445, - 1449, - 1450, - 1462, - 1465, - 1466, - 1475, - 1483, - 1485, - 1488, - 1490, - 1491, - 1492, - 1495, - 1501, - 1503, - 1504, - 1507, - 1512, - 1516, - 1517, - 1519, - 1526, - 1527, - 1532, - 1542, - 1543, - 1545, - 1549, - 1555, - 1559, - 1564, - 1565, - 1567, - 1568, - 1569, - 1570, - 1577, - 1588, - 1593, - 1598, - 1599, - 1848, - 1850, - 1857, - 1862, - 1864, - 1877, - 1919, - 1921, - 1932, - 1945, - 1955, - 1977, - 1979, - 1988, - 1999 - ], - "survival": 0.965, - "id": "t36" - }, - { - "slice": "jfk_pu_wk1", - "ask": "What is the median total_amount of trips picked up at JFK Airport (zone 132) picked up from January 1 to January 7 inclusive that drew a complaint mainly about vehicle condition? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 89.2, - "needs": [ - 4, - 11, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 44, - 50, - 51, - 58, - 65, - 67, - 75, - 76, - 81, - 84, - 85, - 92, - 103, - 112, - 127, - 129, - 134, - 136, - 141, - 148, - 152, - 177, - 184, - 187, - 191, - 194, - 196, - 200, - 201, - 202, - 207, - 211, - 213, - 216, - 222, - 227, - 229, - 230, - 238, - 239, - 249, - 259, - 260, - 267, - 271, - 275, - 294, - 303, - 311, - 321, - 334, - 337, - 347, - 357, - 359, - 367, - 374, - 375, - 382, - 417, - 430, - 440, - 459, - 470, - 485, - 494, - 508, - 513, - 522, - 527, - 529, - 535, - 545, - 548, - 552, - 563, - 588, - 591, - 612, - 634, - 649, - 650, - 651, - 655, - 662, - 665, - 672, - 681, - 694, - 702, - 713, - 727, - 728, - 732, - 740, - 746, - 755, - 761, - 765, - 786, - 811, - 818, - 823, - 838, - 851, - 853, - 860, - 870, - 875, - 883, - 889, - 890, - 891 - ], - "survival": 0.995, - "id": "t37" - }, - { - "slice": "air_wk3", - "ask": "Among complaints about airport trips (to or from zones 1, 132, 138) picked up from January 15 to January 21 inclusive, what percentage are mainly about detours or bad routes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 9.2, - "needs": [ - 2, - 6, - 12, - 22, - 23, - 28, - 29, - 40, - 49, - 69, - 79, - 80, - 90, - 91, - 93, - 94, - 97, - 99, - 100, - 102, - 108, - 118, - 121, - 131, - 132, - 147, - 155, - 158, - 159, - 165, - 167, - 178, - 185, - 186, - 195, - 197, - 198, - 199, - 204, - 210, - 217, - 223, - 225, - 228, - 232, - 241, - 248, - 251, - 258, - 264, - 268, - 269, - 270, - 273, - 277, - 278, - 282, - 287, - 290, - 292, - 306, - 307, - 309, - 310, - 314, - 320, - 330, - 336, - 341, - 342, - 344, - 345, - 346, - 350, - 360, - 365, - 372, - 380, - 385, - 401, - 403, - 404, - 408, - 411, - 421, - 442, - 445, - 446, - 467, - 468, - 481, - 483, - 489, - 493, - 507, - 510, - 521, - 523, - 524, - 528, - 530, - 531, - 537, - 540, - 541, - 549, - 553, - 562, - 564, - 565, - 566, - 572, - 573, - 574, - 575, - 578, - 579, - 582, - 587, - 593, - 594, - 596, - 598, - 603, - 609, - 625, - 627, - 633, - 636, - 638, - 644, - 646, - 661, - 663, - 664, - 670, - 671, - 675, - 679, - 682, - 683, - 686, - 692, - 695, - 703, - 704, - 707, - 708, - 709, - 710, - 717, - 718, - 730, - 735, - 737, - 742, - 743, - 748, - 749, - 753, - 756, - 757, - 768, - 771, - 772, - 774, - 777, - 784, - 789, - 803, - 805, - 806, - 809, - 815, - 819, - 834, - 835, - 836, - 839, - 840, - 845, - 847, - 848, - 850, - 852, - 855, - 868, - 874, - 878, - 884, - 886, - 887, - 888, - 895, - 897, - 899 - ], - "survival": 0.995, - "id": "t38" - }, - { - "slice": "air_wk1", - "ask": "Among complaints about airport trips (to or from zones 1, 132, 138) picked up from January 1 to January 7 inclusive, what percentage are mainly about driver conduct? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 12.8, - "needs": [ - 4, - 5, - 8, - 11, - 13, - 14, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 43, - 44, - 50, - 51, - 54, - 58, - 64, - 65, - 67, - 72, - 74, - 75, - 76, - 81, - 84, - 85, - 86, - 88, - 92, - 103, - 104, - 106, - 112, - 114, - 115, - 127, - 129, - 134, - 136, - 139, - 141, - 144, - 145, - 148, - 152, - 157, - 162, - 164, - 177, - 184, - 187, - 189, - 191, - 192, - 194, - 196, - 200, - 201, - 202, - 205, - 207, - 211, - 213, - 214, - 216, - 222, - 224, - 226, - 227, - 229, - 230, - 235, - 237, - 238, - 239, - 249, - 259, - 260, - 262, - 263, - 266, - 267, - 271, - 275, - 279, - 283, - 285, - 288, - 289, - 293, - 294, - 296, - 297, - 303, - 311, - 312, - 313, - 321, - 329, - 334, - 337, - 338, - 339, - 347, - 351, - 357, - 358, - 359, - 364, - 367, - 374, - 375, - 381, - 382, - 388, - 389, - 400, - 410, - 415, - 417, - 423, - 430, - 432, - 434, - 440, - 447, - 449, - 451, - 452, - 458, - 459, - 470, - 475, - 480, - 485, - 494, - 499, - 505, - 508, - 513, - 516, - 517, - 519, - 522, - 525, - 526, - 527, - 529, - 535, - 545, - 548, - 552, - 555, - 563, - 583, - 588, - 591, - 597, - 601, - 605, - 610, - 611, - 612, - 617, - 634, - 635, - 639, - 648, - 649, - 650, - 651, - 654, - 655, - 658, - 662, - 665, - 672, - 674, - 681, - 687, - 694, - 697, - 702, - 711, - 713, - 727, - 728, - 732, - 736, - 740, - 746, - 755, - 761, - 763, - 765, - 776, - 778, - 786, - 802, - 807, - 811, - 817, - 818, - 823, - 827, - 837, - 838, - 844, - 851, - 853, - 857, - 859, - 860, - 865, - 867, - 870, - 875, - 883, - 889, - 890, - 891, - 892, - 893 - ], - "survival": 1.0, - "id": "t39" - }, - { - "slice": "night_wk1", - "ask": "How many complaints about trips picked up from January 1 to January 7 inclusive, picked up between 22:00 and 04:59 are mainly about safety?", - "kind": "count", - "tol": 0.1, - "answer": 47, - "needs": [ - 8, - 14, - 17, - 36, - 44, - 50, - 58, - 76, - 85, - 92, - 114, - 141, - 157, - 177, - 222, - 224, - 249, - 263, - 275, - 303, - 359, - 367, - 374, - 417, - 459, - 508, - 516, - 535, - 545, - 591, - 635, - 651, - 654, - 662, - 697, - 711, - 713, - 827, - 851, - 867, - 889, - 912, - 947, - 1004, - 1015, - 1092, - 1099, - 1100, - 1115, - 1159, - 1186, - 1194, - 1207, - 1352, - 1405, - 1421, - 1519, - 1527, - 1577, - 1605, - 1612, - 1613, - 1614, - 1618, - 1623, - 1624, - 1630, - 1631, - 1632, - 1636, - 1639, - 1646, - 1648, - 1650, - 1658, - 1659, - 1665, - 1674, - 1682, - 1689, - 1690, - 1692, - 1694, - 1701, - 1702, - 1711, - 1712, - 1716, - 1724, - 1738, - 1747, - 1749, - 1751, - 1756, - 1762, - 1763, - 1764, - 1767, - 1768, - 1769, - 1774, - 1778, - 1780, - 1781, - 1782, - 1785, - 1787, - 1792, - 1796, - 1799, - 1806, - 1807, - 1808, - 1816, - 1819, - 1829, - 1831, - 1837, - 1845, - 1848, - 1850, - 1853, - 1854, - 1857, - 1859, - 1862, - 1864, - 1871, - 1872, - 1877, - 1887, - 1888, - 1897, - 1909, - 1912, - 1919, - 1921, - 1922, - 1923, - 1924, - 1929, - 1931, - 1932, - 1945, - 1948, - 1955, - 1961, - 1974, - 1975, - 1977, - 1978, - 1979, - 1980, - 1988, - 1991, - 1999 - ], - "survival": 0.955, - "id": "t40" - }, - { - "slice": "midtown_wk1", - "ask": "Among complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 1 to January 7 inclusive, what percentage are mainly about safety? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 19.7, - "needs": [ - 5, - 162, - 189, - 192, - 205, - 293, - 351, - 400, - 434, - 449, - 526, - 555, - 601, - 658, - 674, - 763, - 776, - 837, - 844, - 903, - 912, - 932, - 938, - 942, - 947, - 954, - 959, - 963, - 967, - 970, - 978, - 980, - 981, - 986, - 988, - 995, - 998, - 1004, - 1009, - 1015, - 1032, - 1036, - 1041, - 1044, - 1052, - 1056, - 1067, - 1077, - 1085, - 1087, - 1088, - 1092, - 1094, - 1099, - 1100, - 1104, - 1106, - 1112, - 1115, - 1117, - 1120, - 1124, - 1125, - 1127, - 1132, - 1135, - 1138, - 1145, - 1147, - 1155, - 1159, - 1162, - 1163, - 1177, - 1178, - 1186, - 1190, - 1194, - 1197, - 1200, - 1203, - 1206, - 1207, - 1210, - 1211, - 1220, - 1228, - 1230, - 1232, - 1247, - 1256, - 1257, - 1271, - 1287, - 1292, - 1304, - 1316, - 1317, - 1336, - 1347, - 1352, - 1353, - 1356, - 1358, - 1361, - 1362, - 1387, - 1400, - 1404, - 1405, - 1410, - 1415, - 1421, - 1426, - 1428, - 1430, - 1445, - 1449, - 1450, - 1462, - 1465, - 1466, - 1475, - 1483, - 1485, - 1488, - 1490, - 1491, - 1492, - 1495, - 1501, - 1503, - 1504, - 1507, - 1512, - 1516, - 1517, - 1519, - 1526, - 1527, - 1532, - 1542, - 1543, - 1545, - 1549, - 1555, - 1559, - 1564, - 1565, - 1567, - 1568, - 1569, - 1570, - 1577, - 1588, - 1593, - 1598, - 1599, - 1848, - 1850, - 1857, - 1862, - 1864, - 1877, - 1919, - 1921, - 1932, - 1945, - 1955, - 1977, - 1979, - 1988, - 1999 - ], - "survival": 0.995, - "id": "t41" - }, - { - "slice": "night_wk4", - "ask": "How many complaints about trips picked up from January 22 to January 31 inclusive, picked up between 22:00 and 04:59 are mainly about driver conduct?", - "kind": "count", - "tol": 0.1, - "answer": 33, - "needs": [ - 24, - 60, - 66, - 98, - 116, - 120, - 236, - 284, - 298, - 304, - 326, - 394, - 407, - 409, - 462, - 478, - 520, - 551, - 581, - 585, - 602, - 621, - 660, - 691, - 693, - 724, - 747, - 750, - 764, - 782, - 849, - 871, - 902, - 933, - 939, - 955, - 956, - 1018, - 1023, - 1029, - 1061, - 1084, - 1090, - 1166, - 1167, - 1185, - 1217, - 1218, - 1223, - 1234, - 1249, - 1263, - 1303, - 1311, - 1321, - 1354, - 1372, - 1375, - 1392, - 1424, - 1436, - 1443, - 1464, - 1508, - 1513, - 1554, - 1558, - 1587, - 1594, - 1597, - 1606, - 1607, - 1608, - 1609, - 1615, - 1617, - 1619, - 1621, - 1627, - 1628, - 1640, - 1657, - 1660, - 1667, - 1668, - 1673, - 1675, - 1676, - 1677, - 1681, - 1684, - 1685, - 1686, - 1687, - 1695, - 1697, - 1700, - 1705, - 1708, - 1713, - 1714, - 1717, - 1721, - 1722, - 1723, - 1727, - 1730, - 1731, - 1732, - 1735, - 1739, - 1740, - 1742, - 1744, - 1745, - 1746, - 1748, - 1754, - 1765, - 1770, - 1771, - 1772, - 1775, - 1786, - 1789, - 1790, - 1791, - 1793, - 1794, - 1795, - 1797, - 1798, - 1805, - 1810, - 1812, - 1813, - 1814, - 1817, - 1820, - 1822, - 1826, - 1828, - 1830, - 1832, - 1834, - 1839, - 1840, - 1841, - 1849, - 1855, - 1863, - 1865, - 1873, - 1875, - 1878, - 1882, - 1885, - 1892, - 1895, - 1898, - 1900, - 1902, - 1905, - 1906, - 1910, - 1911, - 1913, - 1914, - 1915, - 1916, - 1920, - 1928, - 1930, - 1938, - 1949, - 1954, - 1956, - 1957, - 1958, - 1960, - 1963, - 1964, - 1965, - 1967, - 1968, - 1970, - 1972, - 1973, - 1976, - 1981, - 1982, - 1983, - 1989, - 1994 - ], - "survival": 0.96, - "id": "t42" - }, - { - "slice": "midtown_wk3", - "ask": "What is the median total_amount of Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 15 to January 21 inclusive that drew a complaint mainly about vehicle condition? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 19.9, - "needs": [ - 2, - 40, - 80, - 94, - 99, - 228, - 346, - 350, - 360, - 411, - 523, - 566, - 582, - 593, - 671, - 695, - 756, - 805, - 809, - 878, - 884, - 904, - 909, - 911, - 914, - 916, - 923, - 924, - 926, - 928, - 936, - 937, - 950, - 951, - 957, - 961, - 969, - 972, - 974, - 991, - 996, - 1000, - 1013, - 1016, - 1022, - 1026, - 1027, - 1039, - 1042, - 1043, - 1045, - 1055, - 1063, - 1069, - 1070, - 1071, - 1074, - 1075, - 1078, - 1093, - 1095, - 1103, - 1111, - 1116, - 1119, - 1121, - 1122, - 1126, - 1136, - 1139, - 1144, - 1146, - 1150, - 1151, - 1154, - 1165, - 1168, - 1170, - 1171, - 1181, - 1183, - 1188, - 1192, - 1193, - 1208, - 1209, - 1221, - 1224, - 1225, - 1229, - 1233, - 1241, - 1251, - 1255, - 1258, - 1259, - 1260, - 1261, - 1267, - 1269, - 1270, - 1279, - 1282, - 1283, - 1285, - 1293, - 1294, - 1309, - 1310, - 1313, - 1324, - 1330, - 1334, - 1338, - 1339, - 1343, - 1349, - 1357, - 1360, - 1364, - 1369, - 1370, - 1373, - 1374, - 1379, - 1380, - 1385, - 1386, - 1388, - 1390, - 1391, - 1395, - 1398, - 1409, - 1417, - 1427, - 1432, - 1433, - 1434, - 1437, - 1441, - 1452, - 1454, - 1457, - 1463, - 1468, - 1472, - 1474, - 1481, - 1484, - 1486, - 1489, - 1493, - 1496, - 1502, - 1505, - 1509, - 1520, - 1521, - 1522, - 1530, - 1534, - 1535, - 1536, - 1540, - 1541, - 1546, - 1547, - 1548, - 1550, - 1553, - 1561, - 1572, - 1585, - 1592, - 1596, - 1600, - 1836, - 1860, - 1884, - 1896, - 1918, - 1927, - 1936, - 1944, - 1997, - 1998 - ], - "survival": 0.99, - "id": "t43" - }, - { - "slice": "night_wk3", - "ask": "Among complaints about trips picked up from January 15 to January 21 inclusive, picked up between 22:00 and 04:59, what percentage are mainly about detours or bad routes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 7.4, - "needs": [ - 12, - 23, - 94, - 102, - 118, - 131, - 167, - 178, - 197, - 217, - 241, - 251, - 264, - 270, - 277, - 342, - 365, - 380, - 404, - 442, - 445, - 467, - 510, - 664, - 675, - 703, - 707, - 709, - 749, - 768, - 805, - 834, - 845, - 874, - 886, - 887, - 904, - 923, - 928, - 972, - 1013, - 1075, - 1146, - 1165, - 1170, - 1208, - 1225, - 1267, - 1339, - 1343, - 1437, - 1463, - 1472, - 1521, - 1536, - 1561, - 1616, - 1637, - 1641, - 1642, - 1643, - 1644, - 1647, - 1649, - 1651, - 1652, - 1653, - 1656, - 1662, - 1663, - 1669, - 1670, - 1671, - 1672, - 1691, - 1703, - 1706, - 1707, - 1709, - 1715, - 1720, - 1728, - 1729, - 1733, - 1737, - 1741, - 1743, - 1752, - 1753, - 1755, - 1758, - 1759, - 1766, - 1776, - 1779, - 1800, - 1804, - 1809, - 1811, - 1815, - 1818, - 1821, - 1824, - 1827, - 1833, - 1836, - 1846, - 1851, - 1860, - 1861, - 1868, - 1880, - 1883, - 1884, - 1890, - 1891, - 1893, - 1894, - 1896, - 1899, - 1904, - 1908, - 1918, - 1927, - 1935, - 1936, - 1937, - 1942, - 1944, - 1946, - 1947, - 1952, - 1969, - 1984, - 1997, - 1998 - ], - "survival": 0.995, - "id": "t44" - }, - { - "slice": "air_wk2", - "ask": "What is the median total_amount of airport trips (to or from zones 1, 132, 138) picked up from January 8 to January 14 inclusive that drew a complaint mainly about lost items? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 85.95, - "needs": [ - 1, - 3, - 7, - 10, - 16, - 18, - 19, - 21, - 27, - 30, - 31, - 34, - 37, - 46, - 48, - 52, - 53, - 57, - 68, - 71, - 77, - 78, - 83, - 95, - 96, - 101, - 110, - 111, - 113, - 122, - 123, - 124, - 133, - 135, - 137, - 142, - 146, - 160, - 163, - 166, - 169, - 172, - 174, - 176, - 179, - 180, - 181, - 182, - 183, - 193, - 206, - 208, - 209, - 240, - 242, - 245, - 247, - 250, - 265, - 286, - 291, - 299, - 305, - 308, - 315, - 317, - 318, - 325, - 331, - 335, - 343, - 348, - 352, - 353, - 354, - 356, - 362, - 363, - 366, - 369, - 370, - 371, - 377, - 383, - 391, - 398, - 406, - 412, - 414, - 416, - 418, - 420, - 425, - 429, - 431, - 436, - 437, - 443, - 448, - 453, - 454, - 456, - 463, - 472, - 473, - 474, - 479, - 495, - 497, - 498, - 500, - 501, - 503, - 506, - 509, - 511, - 514, - 515, - 518, - 534, - 536, - 546, - 550, - 554, - 559, - 560, - 576, - 580, - 590, - 595, - 599, - 600, - 619, - 620, - 628, - 632, - 640, - 652, - 666, - 669, - 677, - 685, - 690, - 701, - 705, - 726, - 734, - 744, - 752, - 754, - 758, - 767, - 769, - 779, - 780, - 783, - 787, - 792, - 793, - 796, - 798, - 799, - 800, - 801, - 812, - 814, - 816, - 820, - 822, - 825, - 828, - 829, - 830, - 831, - 833, - 842, - 846, - 854, - 856, - 861, - 862, - 864, - 866, - 869, - 873, - 876, - 880, - 881, - 896, - 900 - ], - "survival": 0.975, - "id": "t45" - }, - { - "slice": "midtown_wk3", - "ask": "Which category of complaint is most common among complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 15 to January 21 inclusive? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "vehicle_condition", - "needs": [ - 2, - 40, - 80, - 94, - 99, - 228, - 346, - 350, - 360, - 411, - 523, - 566, - 582, - 593, - 671, - 695, - 756, - 805, - 809, - 878, - 884, - 904, - 909, - 911, - 914, - 916, - 923, - 924, - 926, - 928, - 936, - 937, - 950, - 951, - 957, - 961, - 969, - 972, - 974, - 991, - 996, - 1000, - 1013, - 1016, - 1022, - 1026, - 1027, - 1039, - 1042, - 1043, - 1045, - 1055, - 1063, - 1069, - 1070, - 1071, - 1074, - 1075, - 1078, - 1093, - 1095, - 1103, - 1111, - 1116, - 1119, - 1121, - 1122, - 1126, - 1136, - 1139, - 1144, - 1146, - 1150, - 1151, - 1154, - 1165, - 1168, - 1170, - 1171, - 1181, - 1183, - 1188, - 1192, - 1193, - 1208, - 1209, - 1221, - 1224, - 1225, - 1229, - 1233, - 1241, - 1251, - 1255, - 1258, - 1259, - 1260, - 1261, - 1267, - 1269, - 1270, - 1279, - 1282, - 1283, - 1285, - 1293, - 1294, - 1309, - 1310, - 1313, - 1324, - 1330, - 1334, - 1338, - 1339, - 1343, - 1349, - 1357, - 1360, - 1364, - 1369, - 1370, - 1373, - 1374, - 1379, - 1380, - 1385, - 1386, - 1388, - 1390, - 1391, - 1395, - 1398, - 1409, - 1417, - 1427, - 1432, - 1433, - 1434, - 1437, - 1441, - 1452, - 1454, - 1457, - 1463, - 1468, - 1472, - 1474, - 1481, - 1484, - 1486, - 1489, - 1493, - 1496, - 1502, - 1505, - 1509, - 1520, - 1521, - 1522, - 1530, - 1534, - 1535, - 1536, - 1540, - 1541, - 1546, - 1547, - 1548, - 1550, - 1553, - 1561, - 1572, - 1585, - 1592, - 1596, - 1600, - 1836, - 1860, - 1884, - 1896, - 1918, - 1927, - 1936, - 1944, - 1997, - 1998 - ], - "survival": 1.0, - "id": "t46" - }, - { - "slice": "jfk_pu_wk3", - "ask": "How many complaints about trips picked up at JFK Airport (zone 132) picked up from January 15 to January 21 inclusive are mainly about detours or bad routes?", - "kind": "count", - "tol": 0.1, - "answer": 7, - "needs": [ - 12, - 28, - 29, - 69, - 90, - 97, - 102, - 108, - 118, - 132, - 155, - 159, - 165, - 167, - 185, - 197, - 198, - 217, - 225, - 251, - 258, - 268, - 269, - 270, - 277, - 278, - 282, - 292, - 306, - 307, - 310, - 336, - 345, - 372, - 380, - 385, - 401, - 408, - 421, - 442, - 445, - 446, - 467, - 468, - 493, - 524, - 530, - 537, - 540, - 553, - 565, - 572, - 573, - 575, - 579, - 594, - 596, - 625, - 636, - 638, - 663, - 679, - 682, - 686, - 692, - 703, - 704, - 707, - 709, - 710, - 730, - 735, - 749, - 768, - 771, - 772, - 784, - 789, - 806, - 819, - 836, - 840, - 845, - 852, - 868, - 886, - 887, - 895, - 897 - ], - "survival": 0.98, - "id": "t47" - }, - { - "slice": "midtown_wk2", - "ask": "How many complaints about Midtown pickups (zones 100, 161, 162, 163, 164, 170, 186, 230, 234, 48) picked up from January 8 to January 14 inclusive are mainly about fare disputes?", - "kind": "count", - "tol": 0.1, - "answer": 34, - "needs": [ - 1, - 46, - 122, - 124, - 163, - 169, - 247, - 315, - 412, - 414, - 429, - 431, - 437, - 454, - 463, - 515, - 518, - 576, - 632, - 799, - 866, - 873, - 901, - 905, - 906, - 907, - 919, - 921, - 929, - 934, - 946, - 952, - 953, - 960, - 965, - 975, - 976, - 977, - 979, - 982, - 987, - 989, - 990, - 993, - 997, - 1002, - 1003, - 1007, - 1008, - 1014, - 1024, - 1031, - 1040, - 1046, - 1064, - 1065, - 1066, - 1073, - 1076, - 1079, - 1080, - 1081, - 1082, - 1083, - 1089, - 1096, - 1097, - 1101, - 1107, - 1108, - 1113, - 1114, - 1130, - 1134, - 1140, - 1158, - 1169, - 1172, - 1175, - 1176, - 1179, - 1189, - 1195, - 1196, - 1198, - 1199, - 1201, - 1212, - 1213, - 1214, - 1216, - 1226, - 1227, - 1231, - 1237, - 1243, - 1244, - 1248, - 1250, - 1252, - 1272, - 1274, - 1290, - 1298, - 1302, - 1305, - 1314, - 1319, - 1320, - 1322, - 1325, - 1327, - 1328, - 1331, - 1332, - 1340, - 1341, - 1342, - 1344, - 1345, - 1346, - 1350, - 1359, - 1363, - 1377, - 1381, - 1383, - 1401, - 1402, - 1406, - 1408, - 1412, - 1413, - 1416, - 1418, - 1419, - 1425, - 1431, - 1435, - 1439, - 1440, - 1442, - 1444, - 1447, - 1471, - 1473, - 1477, - 1479, - 1482, - 1497, - 1498, - 1511, - 1514, - 1523, - 1524, - 1525, - 1528, - 1529, - 1533, - 1537, - 1538, - 1556, - 1560, - 1571, - 1573, - 1574, - 1575, - 1578, - 1579, - 1580, - 1581, - 1582, - 1589, - 1838, - 1867, - 1874, - 1907, - 1950, - 1959, - 1966, - 1992 - ], - "survival": 0.97, - "id": "t48" - }, - { - "slice": "tsq", - "ask": "What is the median total_amount of trips picked up in Times Sq/Theatre District (zone 230) that drew a complaint mainly about vehicle condition? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 21.24, - "needs": [ - 1, - 42, - 82, - 94, - 122, - 126, - 162, - 189, - 219, - 228, - 246, - 360, - 412, - 429, - 434, - 449, - 478, - 518, - 526, - 589, - 671, - 695, - 759, - 776, - 809, - 866, - 882, - 898, - 901, - 907, - 917, - 920, - 925, - 932, - 949, - 959, - 985, - 997, - 1006, - 1007, - 1015, - 1035, - 1047, - 1055, - 1057, - 1067, - 1104, - 1112, - 1119, - 1126, - 1135, - 1169, - 1181, - 1210, - 1216, - 1235, - 1245, - 1248, - 1258, - 1266, - 1270, - 1278, - 1283, - 1307, - 1322, - 1332, - 1344, - 1346, - 1358, - 1362, - 1363, - 1365, - 1372, - 1376, - 1377, - 1400, - 1414, - 1415, - 1425, - 1427, - 1431, - 1435, - 1465, - 1470, - 1479, - 1496, - 1502, - 1519, - 1521, - 1553, - 1561, - 1563, - 1573, - 1581, - 1864, - 1882, - 1898, - 1930, - 1949, - 1956, - 1977, - 1979 - ], - "survival": 0.97, - "id": "t49" - }, - { - "slice": "night_wk1", - "ask": "What is the median total_amount of trips picked up from January 1 to January 7 inclusive, picked up between 22:00 and 04:59 that drew a complaint mainly about safety? Two decimals.", - "kind": "number_rel", - "tol": 0.05, - "answer": 19.68, - "needs": [ - 8, - 14, - 17, - 36, - 44, - 50, - 58, - 76, - 85, - 92, - 114, - 141, - 157, - 177, - 222, - 224, - 249, - 263, - 275, - 303, - 359, - 367, - 374, - 417, - 459, - 508, - 516, - 535, - 545, - 591, - 635, - 651, - 654, - 662, - 697, - 711, - 713, - 827, - 851, - 867, - 889, - 912, - 947, - 1004, - 1015, - 1092, - 1099, - 1100, - 1115, - 1159, - 1186, - 1194, - 1207, - 1352, - 1405, - 1421, - 1519, - 1527, - 1577, - 1605, - 1612, - 1613, - 1614, - 1618, - 1623, - 1624, - 1630, - 1631, - 1632, - 1636, - 1639, - 1646, - 1648, - 1650, - 1658, - 1659, - 1665, - 1674, - 1682, - 1689, - 1690, - 1692, - 1694, - 1701, - 1702, - 1711, - 1712, - 1716, - 1724, - 1738, - 1747, - 1749, - 1751, - 1756, - 1762, - 1763, - 1764, - 1767, - 1768, - 1769, - 1774, - 1778, - 1780, - 1781, - 1782, - 1785, - 1787, - 1792, - 1796, - 1799, - 1806, - 1807, - 1808, - 1816, - 1819, - 1829, - 1831, - 1837, - 1845, - 1848, - 1850, - 1853, - 1854, - 1857, - 1859, - 1862, - 1864, - 1871, - 1872, - 1877, - 1887, - 1888, - 1897, - 1909, - 1912, - 1919, - 1921, - 1922, - 1923, - 1924, - 1929, - 1931, - 1932, - 1945, - 1948, - 1955, - 1961, - 1974, - 1975, - 1977, - 1978, - 1979, - 1980, - 1988, - 1991, - 1999 - ], - "survival": 1.0, - "id": "t50" - }, - { - "slice": "lga_16_31", - "ask": "Which category of complaint is most common among complaints about trips to or from LaGuardia Airport (zone 138) picked up from January 16 to January 31 inclusive? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "fare_dispute", - "needs": [ - 6, - 9, - 22, - 23, - 24, - 33, - 39, - 40, - 42, - 45, - 47, - 49, - 55, - 60, - 62, - 82, - 93, - 94, - 98, - 99, - 116, - 119, - 121, - 126, - 130, - 131, - 138, - 140, - 147, - 150, - 151, - 153, - 156, - 158, - 161, - 168, - 170, - 178, - 186, - 204, - 210, - 212, - 215, - 228, - 232, - 233, - 234, - 241, - 244, - 248, - 254, - 255, - 256, - 257, - 264, - 269, - 272, - 273, - 274, - 276, - 280, - 290, - 300, - 301, - 309, - 314, - 319, - 320, - 330, - 332, - 340, - 341, - 342, - 346, - 349, - 355, - 365, - 373, - 384, - 393, - 394, - 402, - 403, - 404, - 405, - 411, - 419, - 422, - 424, - 427, - 433, - 435, - 438, - 439, - 441, - 450, - 457, - 460, - 465, - 466, - 469, - 476, - 481, - 484, - 487, - 488, - 489, - 490, - 496, - 502, - 507, - 510, - 521, - 528, - 531, - 538, - 541, - 542, - 544, - 561, - 562, - 564, - 566, - 568, - 569, - 574, - 577, - 581, - 586, - 589, - 598, - 603, - 604, - 608, - 609, - 615, - 618, - 622, - 626, - 627, - 629, - 630, - 641, - 642, - 644, - 645, - 646, - 653, - 657, - 660, - 661, - 664, - 667, - 668, - 670, - 675, - 680, - 683, - 684, - 695, - 708, - 712, - 714, - 715, - 717, - 719, - 720, - 722, - 733, - 737, - 738, - 739, - 742, - 743, - 747, - 748, - 751, - 760, - 762, - 766, - 770, - 773, - 774, - 781, - 782, - 785, - 790, - 803, - 805, - 809, - 810, - 815, - 826, - 832, - 834, - 835, - 839, - 847, - 850, - 855, - 871, - 872, - 874, - 879, - 888, - 899 - ], - "survival": 1.0, - "id": "t51" - }, - { - "slice": "lga_16_31", - "ask": "How many complaints about trips to or from LaGuardia Airport (zone 138) picked up from January 16 to January 31 inclusive are mainly about vehicle condition?", - "kind": "count", - "tol": 0.1, - "answer": 33, - "needs": [ - 6, - 9, - 22, - 23, - 24, - 33, - 39, - 40, - 42, - 45, - 47, - 49, - 55, - 60, - 62, - 82, - 93, - 94, - 98, - 99, - 116, - 119, - 121, - 126, - 130, - 131, - 138, - 140, - 147, - 150, - 151, - 153, - 156, - 158, - 161, - 168, - 170, - 178, - 186, - 204, - 210, - 212, - 215, - 228, - 232, - 233, - 234, - 241, - 244, - 248, - 254, - 255, - 256, - 257, - 264, - 269, - 272, - 273, - 274, - 276, - 280, - 290, - 300, - 301, - 309, - 314, - 319, - 320, - 330, - 332, - 340, - 341, - 342, - 346, - 349, - 355, - 365, - 373, - 384, - 393, - 394, - 402, - 403, - 404, - 405, - 411, - 419, - 422, - 424, - 427, - 433, - 435, - 438, - 439, - 441, - 450, - 457, - 460, - 465, - 466, - 469, - 476, - 481, - 484, - 487, - 488, - 489, - 490, - 496, - 502, - 507, - 510, - 521, - 528, - 531, - 538, - 541, - 542, - 544, - 561, - 562, - 564, - 566, - 568, - 569, - 574, - 577, - 581, - 586, - 589, - 598, - 603, - 604, - 608, - 609, - 615, - 618, - 622, - 626, - 627, - 629, - 630, - 641, - 642, - 644, - 645, - 646, - 653, - 657, - 660, - 661, - 664, - 667, - 668, - 670, - 675, - 680, - 683, - 684, - 695, - 708, - 712, - 714, - 715, - 717, - 719, - 720, - 722, - 733, - 737, - 738, - 739, - 742, - 743, - 747, - 748, - 751, - 760, - 762, - 766, - 770, - 773, - 774, - 781, - 782, - 785, - 790, - 803, - 805, - 809, - 810, - 815, - 826, - 832, - 834, - 835, - 839, - 847, - 850, - 855, - 871, - 872, - 874, - 879, - 888, - 899 - ], - "survival": 0.95, - "id": "t52" - }, - { - "slice": "air_wk2", - "ask": "Among complaints about airport trips (to or from zones 1, 132, 138) picked up from January 8 to January 14 inclusive, what percentage are mainly about detours or bad routes? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 7.4, - "needs": [ - 1, - 3, - 7, - 10, - 16, - 18, - 19, - 21, - 27, - 30, - 31, - 34, - 37, - 46, - 48, - 52, - 53, - 57, - 68, - 71, - 77, - 78, - 83, - 95, - 96, - 101, - 110, - 111, - 113, - 122, - 123, - 124, - 133, - 135, - 137, - 142, - 146, - 160, - 163, - 166, - 169, - 172, - 174, - 176, - 179, - 180, - 181, - 182, - 183, - 193, - 206, - 208, - 209, - 240, - 242, - 245, - 247, - 250, - 265, - 286, - 291, - 299, - 305, - 308, - 315, - 317, - 318, - 325, - 331, - 335, - 343, - 348, - 352, - 353, - 354, - 356, - 362, - 363, - 366, - 369, - 370, - 371, - 377, - 383, - 391, - 398, - 406, - 412, - 414, - 416, - 418, - 420, - 425, - 429, - 431, - 436, - 437, - 443, - 448, - 453, - 454, - 456, - 463, - 472, - 473, - 474, - 479, - 495, - 497, - 498, - 500, - 501, - 503, - 506, - 509, - 511, - 514, - 515, - 518, - 534, - 536, - 546, - 550, - 554, - 559, - 560, - 576, - 580, - 590, - 595, - 599, - 600, - 619, - 620, - 628, - 632, - 640, - 652, - 666, - 669, - 677, - 685, - 690, - 701, - 705, - 726, - 734, - 744, - 752, - 754, - 758, - 767, - 769, - 779, - 780, - 783, - 787, - 792, - 793, - 796, - 798, - 799, - 800, - 801, - 812, - 814, - 816, - 820, - 822, - 825, - 828, - 829, - 830, - 831, - 833, - 842, - 846, - 854, - 856, - 861, - 862, - 864, - 866, - 869, - 873, - 876, - 880, - 881, - 896, - 900 - ], - "survival": 0.995, - "id": "t53" - }, - { - "slice": "lga_1_15", - "ask": "Which category of complaint is most common among complaints about trips to or from LaGuardia Airport (zone 138) picked up from January 1 to January 15 inclusive? Answer with one of: fare_dispute, detour, driver_conduct, vehicle_condition, lost_item, safety.", - "kind": "string", - "tol": 0, - "answer": "fare_dispute", - "needs": [ - 1, - 8, - 14, - 16, - 31, - 43, - 53, - 54, - 57, - 64, - 72, - 74, - 86, - 88, - 91, - 100, - 104, - 106, - 113, - 114, - 115, - 122, - 124, - 135, - 137, - 139, - 144, - 145, - 157, - 164, - 169, - 172, - 180, - 181, - 182, - 205, - 206, - 214, - 224, - 226, - 237, - 240, - 247, - 262, - 263, - 266, - 279, - 283, - 285, - 288, - 289, - 291, - 296, - 297, - 305, - 308, - 313, - 315, - 331, - 338, - 351, - 352, - 354, - 360, - 364, - 369, - 370, - 371, - 388, - 389, - 391, - 410, - 412, - 414, - 415, - 416, - 418, - 423, - 425, - 429, - 432, - 437, - 443, - 447, - 448, - 449, - 451, - 452, - 453, - 454, - 463, - 475, - 480, - 497, - 498, - 499, - 501, - 503, - 505, - 511, - 514, - 516, - 517, - 519, - 525, - 546, - 549, - 559, - 576, - 580, - 587, - 597, - 601, - 610, - 617, - 619, - 635, - 639, - 640, - 648, - 654, - 658, - 666, - 669, - 685, - 687, - 697, - 701, - 705, - 711, - 718, - 736, - 752, - 761, - 763, - 769, - 778, - 787, - 792, - 798, - 799, - 800, - 801, - 802, - 807, - 814, - 816, - 817, - 820, - 822, - 825, - 827, - 829, - 830, - 833, - 854, - 859, - 862, - 864, - 865, - 866, - 867, - 869, - 873, - 881, - 892, - 893, - 900 - ], - "survival": 1.0, - "id": "t54" - }, - { - "slice": "air_wk1", - "ask": "How many complaints about airport trips (to or from zones 1, 132, 138) picked up from January 1 to January 7 inclusive are mainly about fare disputes?", - "kind": "count", - "tol": 0.1, - "answer": 62, - "needs": [ - 4, - 5, - 8, - 11, - 13, - 14, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 43, - 44, - 50, - 51, - 54, - 58, - 64, - 65, - 67, - 72, - 74, - 75, - 76, - 81, - 84, - 85, - 86, - 88, - 92, - 103, - 104, - 106, - 112, - 114, - 115, - 127, - 129, - 134, - 136, - 139, - 141, - 144, - 145, - 148, - 152, - 157, - 162, - 164, - 177, - 184, - 187, - 189, - 191, - 192, - 194, - 196, - 200, - 201, - 202, - 205, - 207, - 211, - 213, - 214, - 216, - 222, - 224, - 226, - 227, - 229, - 230, - 235, - 237, - 238, - 239, - 249, - 259, - 260, - 262, - 263, - 266, - 267, - 271, - 275, - 279, - 283, - 285, - 288, - 289, - 293, - 294, - 296, - 297, - 303, - 311, - 312, - 313, - 321, - 329, - 334, - 337, - 338, - 339, - 347, - 351, - 357, - 358, - 359, - 364, - 367, - 374, - 375, - 381, - 382, - 388, - 389, - 400, - 410, - 415, - 417, - 423, - 430, - 432, - 434, - 440, - 447, - 449, - 451, - 452, - 458, - 459, - 470, - 475, - 480, - 485, - 494, - 499, - 505, - 508, - 513, - 516, - 517, - 519, - 522, - 525, - 526, - 527, - 529, - 535, - 545, - 548, - 552, - 555, - 563, - 583, - 588, - 591, - 597, - 601, - 605, - 610, - 611, - 612, - 617, - 634, - 635, - 639, - 648, - 649, - 650, - 651, - 654, - 655, - 658, - 662, - 665, - 672, - 674, - 681, - 687, - 694, - 697, - 702, - 711, - 713, - 727, - 728, - 732, - 736, - 740, - 746, - 755, - 761, - 763, - 765, - 776, - 778, - 786, - 802, - 807, - 811, - 817, - 818, - 823, - 827, - 837, - 838, - 844, - 851, - 853, - 857, - 859, - 860, - 865, - 867, - 870, - 875, - 883, - 889, - 890, - 891, - 892, - 893 - ], - "survival": 1.0, - "id": "t55" - }, - { - "slice": "air_wk2", - "ask": "How many complaints about airport trips (to or from zones 1, 132, 138) picked up from January 8 to January 14 inclusive are mainly about fare disputes?", - "kind": "count", - "tol": 0.1, - "answer": 69, - "needs": [ - 1, - 3, - 7, - 10, - 16, - 18, - 19, - 21, - 27, - 30, - 31, - 34, - 37, - 46, - 48, - 52, - 53, - 57, - 68, - 71, - 77, - 78, - 83, - 95, - 96, - 101, - 110, - 111, - 113, - 122, - 123, - 124, - 133, - 135, - 137, - 142, - 146, - 160, - 163, - 166, - 169, - 172, - 174, - 176, - 179, - 180, - 181, - 182, - 183, - 193, - 206, - 208, - 209, - 240, - 242, - 245, - 247, - 250, - 265, - 286, - 291, - 299, - 305, - 308, - 315, - 317, - 318, - 325, - 331, - 335, - 343, - 348, - 352, - 353, - 354, - 356, - 362, - 363, - 366, - 369, - 370, - 371, - 377, - 383, - 391, - 398, - 406, - 412, - 414, - 416, - 418, - 420, - 425, - 429, - 431, - 436, - 437, - 443, - 448, - 453, - 454, - 456, - 463, - 472, - 473, - 474, - 479, - 495, - 497, - 498, - 500, - 501, - 503, - 506, - 509, - 511, - 514, - 515, - 518, - 534, - 536, - 546, - 550, - 554, - 559, - 560, - 576, - 580, - 590, - 595, - 599, - 600, - 619, - 620, - 628, - 632, - 640, - 652, - 666, - 669, - 677, - 685, - 690, - 701, - 705, - 726, - 734, - 744, - 752, - 754, - 758, - 767, - 769, - 779, - 780, - 783, - 787, - 792, - 793, - 796, - 798, - 799, - 800, - 801, - 812, - 814, - 816, - 820, - 822, - 825, - 828, - 829, - 830, - 831, - 833, - 842, - 846, - 854, - 856, - 861, - 862, - 864, - 866, - 869, - 873, - 876, - 880, - 881, - 896, - 900 - ], - "survival": 0.98, - "id": "t56" - }, - { - "slice": "jfk_pu_wk2", - "ask": "How many complaints about trips picked up at JFK Airport (zone 132) picked up from January 8 to January 14 inclusive are mainly about driver conduct?", - "kind": "count", - "tol": 0.1, - "answer": 9, - "needs": [ - 3, - 10, - 18, - 19, - 21, - 27, - 30, - 31, - 34, - 37, - 48, - 52, - 68, - 71, - 77, - 78, - 83, - 95, - 96, - 101, - 110, - 111, - 123, - 133, - 142, - 146, - 160, - 166, - 174, - 176, - 183, - 193, - 209, - 242, - 245, - 250, - 265, - 286, - 291, - 299, - 317, - 318, - 325, - 335, - 343, - 348, - 356, - 363, - 366, - 377, - 383, - 398, - 406, - 420, - 436, - 456, - 472, - 473, - 474, - 479, - 495, - 506, - 509, - 534, - 536, - 550, - 554, - 560, - 595, - 599, - 600, - 620, - 628, - 652, - 666, - 677, - 726, - 734, - 744, - 754, - 758, - 767, - 779, - 780, - 793, - 796, - 812, - 820, - 828, - 831, - 842, - 846, - 856, - 861, - 876, - 880, - 896 - ], - "survival": 0.95, - "id": "t57" - }, - { - "slice": "lga_1_15", - "ask": "How many complaints about trips to or from LaGuardia Airport (zone 138) picked up from January 1 to January 15 inclusive are mainly about fare disputes?", - "kind": "count", - "tol": 0.1, - "answer": 49, - "needs": [ - 1, - 8, - 14, - 16, - 31, - 43, - 53, - 54, - 57, - 64, - 72, - 74, - 86, - 88, - 91, - 100, - 104, - 106, - 113, - 114, - 115, - 122, - 124, - 135, - 137, - 139, - 144, - 145, - 157, - 164, - 169, - 172, - 180, - 181, - 182, - 205, - 206, - 214, - 224, - 226, - 237, - 240, - 247, - 262, - 263, - 266, - 279, - 283, - 285, - 288, - 289, - 291, - 296, - 297, - 305, - 308, - 313, - 315, - 331, - 338, - 351, - 352, - 354, - 360, - 364, - 369, - 370, - 371, - 388, - 389, - 391, - 410, - 412, - 414, - 415, - 416, - 418, - 423, - 425, - 429, - 432, - 437, - 443, - 447, - 448, - 449, - 451, - 452, - 453, - 454, - 463, - 475, - 480, - 497, - 498, - 499, - 501, - 503, - 505, - 511, - 514, - 516, - 517, - 519, - 525, - 546, - 549, - 559, - 576, - 580, - 587, - 597, - 601, - 610, - 617, - 619, - 635, - 639, - 640, - 648, - 654, - 658, - 666, - 669, - 685, - 687, - 697, - 701, - 705, - 711, - 718, - 736, - 752, - 761, - 763, - 769, - 778, - 787, - 792, - 798, - 799, - 800, - 801, - 802, - 807, - 814, - 816, - 817, - 820, - 822, - 825, - 827, - 829, - 830, - 833, - 854, - 859, - 862, - 864, - 865, - 866, - 867, - 869, - 873, - 881, - 892, - 893, - 900 - ], - "survival": 0.97, - "id": "t58" - }, - { - "slice": "jfk_pu_wk1", - "ask": "Among complaints about trips picked up at JFK Airport (zone 132) picked up from January 1 to January 7 inclusive, what percentage are mainly about safety? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 16.4, - "needs": [ - 4, - 11, - 15, - 17, - 20, - 26, - 32, - 35, - 36, - 44, - 50, - 51, - 58, - 65, - 67, - 75, - 76, - 81, - 84, - 85, - 92, - 103, - 112, - 127, - 129, - 134, - 136, - 141, - 148, - 152, - 177, - 184, - 187, - 191, - 194, - 196, - 200, - 201, - 202, - 207, - 211, - 213, - 216, - 222, - 227, - 229, - 230, - 238, - 239, - 249, - 259, - 260, - 267, - 271, - 275, - 294, - 303, - 311, - 321, - 334, - 337, - 347, - 357, - 359, - 367, - 374, - 375, - 382, - 417, - 430, - 440, - 459, - 470, - 485, - 494, - 508, - 513, - 522, - 527, - 529, - 535, - 545, - 548, - 552, - 563, - 588, - 591, - 612, - 634, - 649, - 650, - 651, - 655, - 662, - 665, - 672, - 681, - 694, - 702, - 713, - 727, - 728, - 732, - 740, - 746, - 755, - 761, - 765, - 786, - 811, - 818, - 823, - 838, - 851, - 853, - 860, - 870, - 875, - 883, - 889, - 890, - 891 - ], - "survival": 0.985, - "id": "t59" - }, - { - "slice": "lga_16_31", - "ask": "Among complaints about trips to or from LaGuardia Airport (zone 138) picked up from January 16 to January 31 inclusive, what percentage are mainly about lost items? One decimal.", - "kind": "number", - "tol": 3.0, - "answer": 20.4, - "needs": [ - 6, - 9, - 22, - 23, - 24, - 33, - 39, - 40, - 42, - 45, - 47, - 49, - 55, - 60, - 62, - 82, - 93, - 94, - 98, - 99, - 116, - 119, - 121, - 126, - 130, - 131, - 138, - 140, - 147, - 150, - 151, - 153, - 156, - 158, - 161, - 168, - 170, - 178, - 186, - 204, - 210, - 212, - 215, - 228, - 232, - 233, - 234, - 241, - 244, - 248, - 254, - 255, - 256, - 257, - 264, - 269, - 272, - 273, - 274, - 276, - 280, - 290, - 300, - 301, - 309, - 314, - 319, - 320, - 330, - 332, - 340, - 341, - 342, - 346, - 349, - 355, - 365, - 373, - 384, - 393, - 394, - 402, - 403, - 404, - 405, - 411, - 419, - 422, - 424, - 427, - 433, - 435, - 438, - 439, - 441, - 450, - 457, - 460, - 465, - 466, - 469, - 476, - 481, - 484, - 487, - 488, - 489, - 490, - 496, - 502, - 507, - 510, - 521, - 528, - 531, - 538, - 541, - 542, - 544, - 561, - 562, - 564, - 566, - 568, - 569, - 574, - 577, - 581, - 586, - 589, - 598, - 603, - 604, - 608, - 609, - 615, - 618, - 622, - 626, - 627, - 629, - 630, - 641, - 642, - 644, - 645, - 646, - 653, - 657, - 660, - 661, - 664, - 667, - 668, - 670, - 675, - 680, - 683, - 684, - 695, - 708, - 712, - 714, - 715, - 717, - 719, - 720, - 722, - 733, - 737, - 738, - 739, - 742, - 743, - 747, - 748, - 751, - 760, - 762, - 766, - 770, - 773, - 774, - 781, - 782, - 785, - 790, - 803, - 805, - 809, - 810, - 815, - 826, - 832, - 834, - 835, - 839, - 847, - 850, - 855, - 871, - 872, - 874, - 879, - 888, - 899 - ], - "survival": 1.0, - "id": "t60" - } -] diff --git a/tests/test_tabular_memory.py b/tests/test_tabular_memory.py index dfff61a..7e4ef63 100644 --- a/tests/test_tabular_memory.py +++ b/tests/test_tabular_memory.py @@ -614,7 +614,7 @@ def test_a_missing_cli_is_reported_to_the_user_with_the_install_command(env): fresh(env) out = hook("session_start", {"cwd": str(env / "my-repo")}, {"PATH": str(Path(sys.executable).parent)}) - assert "hotdata.dev/install" in out["systemMessage"] + assert "brew install hotdata-dev/tap/cli" in out["systemMessage"] assert "cannot run yet" in out["hookSpecificOutput"]["additionalContext"] assert not (env / "data" / "ready").exists() @@ -697,7 +697,7 @@ def test_doctor_stops_at_login_and_names_the_fix(env): def test_doctor_without_the_cli_says_how_to_install_it(env): out = cli(["doctor"], env, {"PATH": str(Path(sys.executable).parent)}) - assert out.returncode == 1 and "hotdata.dev/install" in out.stdout + assert out.returncode == 1 and "brew install hotdata-dev/tap/cli" in out.stdout def test_demo_loads_the_sample_by_url_and_suggests_questions(env):