diff --git a/.github/actions/build-linux/action.yml b/.github/actions/build-linux/action.yml new file mode 100644 index 0000000..89c59f4 --- /dev/null +++ b/.github/actions/build-linux/action.yml @@ -0,0 +1,89 @@ +name: Build MTConnectExplorer (Linux) +description: > + The COMPLETE Linux release build — freeze, third-party notices, tar.gz, and + the assertions that each one actually produced something. Shared verbatim by + the PR gate (ci.yml) and the release workflow (release.yml); only uploading + the archive is release-specific. + +# WHY THIS IS A COMPOSITE ACTION AND NOT TWO COPIES OF THE STEPS +# The PR gate used to stop after `make` and `test -f dist/MTConnectExplorer`. +# Everything after that — generating THIRD_PARTY_NOTICES.txt and rolling the +# tar.gz — ran ONLY during a release, which is the most expensive place to +# discover a defect: by then the merge into release/X.Y has already happened +# and the version is spent, so a fix costs another bump and another merge. +# +# generate_notices.py is the live hazard. It walks installed package +# metadata via importlib.metadata, so a new dependency whose metadata is +# missing or malformed breaks it — a dependency change the gate happily +# passes. The sibling CellSmith repo learned this twice over: its .deb +# packaging died on the first REAL payload, and its version guard had never +# worked, both in the half its gate skipped. +# +# So the gate now runs the whole build, and there is exactly ONE definition +# of what that is. Referenced as `./.github/actions/build-linux`, which +# resolves to this repo at the commit under test — the gate and the release +# can never run different versions of the build. + +outputs: + archive: + description: Filename of the produced tar.gz, relative to the workspace root. + value: ${{ steps.package.outputs.archive }} + +runs: + using: composite + steps: + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Build with make + shell: bash + run: make + + - name: Verify binary was produced + shell: bash + run: test -f dist/MTConnectExplorer + + - name: Generate third-party notices + # Runs on EVERY PR, not just releases — see the header. `make` built the + # venv this reads from. + shell: bash + run: venv/bin/python packaging/generate_notices.py THIRD_PARTY_NOTICES.txt + + - name: Package tar.gz + id: package + # The archive name is derived from the same single-source version reader + # the workflows use, rather than passed in. That way the gate names the + # artifact exactly as the release will, so a name that would break the + # release breaks the PR instead. + shell: bash + run: | + set -euo pipefail + version=$(bash .github/scripts/read-version.sh) + archive="MTConnectExplorer-v${version}-linux-x86_64.tar.gz" + + chmod +x dist/MTConnectExplorer + cp THIRD_PARTY_NOTICES.txt dist/ + tar -czvf "$archive" -C dist MTConnectExplorer THIRD_PARTY_NOTICES.txt + + # Assert the archive is real. `tar` exits 0 on an empty input set, so + # "the command succeeded" is not the same as "an archive was produced". + [ -s "$archive" ] || { echo "::error::$archive was not produced, or is empty"; exit 1; } + + # ⛔ LIST TO A FILE, THEN GREP THE FILE. Never `tar -tzf … | grep -q`. + # `grep -q` exits at the first match and closes the pipe while tar is + # still streaming through tens of MB of compressed payload to reach the + # next entry; tar's next write gets SIGPIPE ("tar: stdout: write + # error"), and under `pipefail` that failure becomes the PIPELINE's + # status — so the check reports "binary missing" for an archive that + # contains it. Note the shape: it fires only when the check SUCCEEDS, + # and only once the payload is big enough to keep tar busy, so a small + # synthetic fixture passes and the real build fails. This is the same + # defect make_deb.sh hit in the sibling CellSmith repo. + tar -tzf "$archive" > .archive-contents.txt + grep -qx MTConnectExplorer .archive-contents.txt \ + || { echo "::error::$archive does not contain the MTConnectExplorer binary"; exit 1; } + rm -f .archive-contents.txt + + ls -la "$archive" + echo "archive=$archive" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/build-windows/action.yml b/.github/actions/build-windows/action.yml new file mode 100644 index 0000000..39b9bd6 --- /dev/null +++ b/.github/actions/build-windows/action.yml @@ -0,0 +1,76 @@ +name: Build MTConnectExplorer (Windows) +description: > + The COMPLETE Windows release build — freeze, third-party notices, zip, and the + assertions that each one actually produced something. Shared verbatim by the + PR gate (ci.yml) and the release workflow (release.yml); only uploading the + archive is release-specific. + +# See .github/actions/build-linux/action.yml for why this is a composite action +# rather than two copies of the steps. Same reasoning, same hazard: the gate used +# to stop after build.bat, so generate_notices.py and Compress-Archive were only +# ever exercised during an actual release. + +outputs: + archive: + description: Filename of the produced zip, relative to the workspace root. + value: ${{ steps.package.outputs.archive }} + +runs: + using: composite + steps: + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Build with build.bat + shell: cmd + run: build.bat ci + + - name: Verify binary was produced + shell: pwsh + run: | + if (-not (Test-Path dist\MTConnectExplorer.exe)) { + throw "dist\MTConnectExplorer.exe not found" + } + + - name: Generate third-party notices + # Runs on EVERY PR, not just releases — see the Linux action's header. + shell: cmd + run: venv\Scripts\python.exe packaging\generate_notices.py THIRD_PARTY_NOTICES.txt + + - name: Resolve the archive name + # Derived from the same single-source version reader the workflows use, so + # the gate names the artifact exactly as the release will. Git Bash is on + # PATH on windows runners, so the one reader serves both platforms. + shell: bash + run: | + set -euo pipefail + version=$(bash .github/scripts/read-version.sh) + echo "ARCHIVE=MTConnectExplorer-v${version}-win64.zip" >> "$GITHUB_ENV" + + - name: Package zip + id: package + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + Copy-Item THIRD_PARTY_NOTICES.txt dist\ + Compress-Archive ` + -Path dist\MTConnectExplorer.exe, dist\THIRD_PARTY_NOTICES.txt ` + -DestinationPath "$env:ARCHIVE" -Force + + # Assert the archive is real, not merely that the cmdlet returned. + if (-not (Test-Path $env:ARCHIVE)) { throw "$env:ARCHIVE was not produced" } + + # Dispose the handle explicitly: OpenRead holds the file open, and a + # leaked handle keeps the .zip locked for the rest of the job — the + # upload step would then fail on a file "in use by another process". + Add-Type -AssemblyName System.IO.Compression.FileSystem + $zip = [IO.Compression.ZipFile]::OpenRead((Resolve-Path $env:ARCHIVE)) + try { $entries = @($zip.Entries.Name) } + finally { $zip.Dispose() } + if ($entries -notcontains 'MTConnectExplorer.exe') { + throw "$env:ARCHIVE does not contain MTConnectExplorer.exe" + } + + Get-Item $env:ARCHIVE | Format-List Name, Length + "archive=$env:ARCHIVE" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 diff --git a/.github/scripts/read-version.sh b/.github/scripts/read-version.sh new file mode 100644 index 0000000..47c497a --- /dev/null +++ b/.github/scripts/read-version.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Print MTConnectExplorer's single source-of-truth version, validated as semver. +# +# version=$(bash .github/scripts/read-version.sh) # -> 0.1.0-alpha.5 +# +# WHY THIS EXISTS +# ci.yml and release.yml each parsed src/__version__.py with +# +# version=$(grep -oP '"\K[^"]+' src/__version__.py) +# +# which is UNANCHORED. It works only as long as src/__version__.py contains +# exactly one quoted string. Add a module docstring — the obvious next edit +# to that file — and the third quote of the triple-quote opener starts a +# match, so the docstring's first line comes out AHEAD of the version and +# the guard fails with a two-line $version. That is not hypothetical: the +# sibling CellSmith repo shipped the identical snippet, grew a docstring, +# and its first release PR died on it — in both inlined copies, wrong in +# the same way. Which is the argument for one shared reader. +# +# Diagnostics go to stderr; ONLY the version goes to stdout, so the caller can +# use $(...) safely. +set -uo pipefail + +ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +FILE="$ROOT/src/__version__.py" + +die() { echo "::error::$1" >&2; exit 1; } + +[ -f "$FILE" ] || die "version file not found: $FILE" + +# Anchored to the start of a line, so docstring quotes cannot match. +# +# POSIX `sed`, deliberately NOT `grep -oP`: PCRE grep is not portable. Git Bash +# on Windows refuses it outright — "grep: -P supports only unibyte and UTF-8 +# locales" — which would make this reader untestable on a dev machine even +# though it works on the runner. BRE costs nothing here. +mapfile -t matches < <( + sed -n 's/^__version__[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$FILE") + +case "${#matches[@]}" in + 0) die "no '__version__ = \"...\"' assignment found in $FILE" ;; + 1) ;; + # Ambiguity is a hard error, not first-match-wins: anything that execs the + # file keeps the LAST assignment and a text parser keeps the FIRST, so a + # duplicate would silently decouple the git tag from the shipped version. + *) die "expected exactly one __version__ assignment in $FILE, found ${#matches[@]}: ${matches[*]}" ;; +esac + +version="${matches[0]}" +if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + die "__version__ '$version' is not valid semver (expected MAJOR.MINOR.PATCH[-PRERELEASE])" +fi + +printf '%s\n' "$version" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e234682..bf9bbba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,71 +19,115 @@ permissions: jobs: version-check: - # Pre-merge guard for release PRs: a PR into release/X.Y must carry a - # matching X.Y version, so the post-merge release workflow can't fail on a - # version/branch mismatch. No-op for PRs into main / dev/*. - # Mark as a required status check on release/* branches. + # Three guards, deliberately at three different scopes: + # ALWAYS — `__version__` parses and is semver. Not + # release-specific: a malformed version breaks the + # build, the archive names, and the tag. + # main, dev/*, — the version is not one that has already been + # release/* released, so a released number can never be the + # current one on a protected branch. See GUARD 3. + # release/* only — the X.Y line matches the target branch. + # Mark as a required status check on main, dev/* and release/* branches. runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Validate __version__ matches the target release branch + - uses: actions/checkout@v7 + - name: Validate __version__ (and, on protected branches, that it can still ship) run: | - base="${{ github.base_ref }}" - if [[ ! "$base" =~ ^release/([0-9]+)\.([0-9]+)$ ]]; then - echo "Base branch '$base' is not a release branch; skipping version check." - exit 0 - fi + # Reads + semver-validates in one place, shared with release.yml. Do + # NOT inline a parse here: the two inlined copies this replaced were + # both unanchored. See the script's header. + version=$(bash .github/scripts/read-version.sh) || exit 1 + echo "OK: __version__ '$version' is well-formed." - # Read __version__ from src/__version__.py - version=$(grep -oP '"\K[^"]+' src/__version__.py) - if [ -z "$version" ]; then - echo "::error::Failed to parse __version__ from src/__version__.py" - exit 1 - fi + base="${{ github.base_ref }}" + tag="v$version" - # Validate semver (mirrors release.yml) - if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then - echo "::error::__version__ '$version' is not valid semver (expected MAJOR.MINOR.PATCH[-PRERELEASE])" - exit 1 + # GUARD 2 (release PRs only) — the version's X.Y line matches the + # target branch, so the post-merge release workflow can't fail on a + # version/branch mismatch. Catches "merged the wrong PR into the wrong + # branch". + if [[ "$base" =~ ^release/([0-9]+)\.([0-9]+)$ ]]; then + expected_minor="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" + actual_minor=$(echo "$version" | cut -d'-' -f1 | cut -d. -f1-2) + if [ "$expected_minor" != "$actual_minor" ]; then + echo "::error::PR targets '$base' which expects version $expected_minor.x, but __version__ is '$version'" + exit 1 + fi + echo "OK: __version__ '$version' matches target branch '$base'." fi - # Re-match the base branch (the semver test above clobbered BASH_REMATCH) - # to recover the expected X.Y line, then compare to the version's line. - [[ "$base" =~ ^release/([0-9]+)\.([0-9]+)$ ]] - expected_minor="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - actual_minor=$(echo "$version" | cut -d'-' -f1 | cut -d. -f1-2) - if [ "$expected_minor" != "$actual_minor" ]; then - echo "::error::PR targets '$base' which expects version $expected_minor.x, but __version__ is '$version'" - exit 1 + # GUARD 3 (main, dev/* AND release/*) — the version must not be released + # already. One check, two distinct failures it prevents: + # + # release/* — merging IS how a release is made, so a PR whose + # version is already tagged can only produce a run that + # skips itself (release.yml sets should_release=false for + # exactly this). A red PR now beats a silent no-op later. + # main — main stages the NEXT release. Letting it sit at a + # version that already shipped means the stale number + # rides the next forward-merge into release/*, where it + # produces nothing. Refusing it at the door is what keeps + # a released version from ever being current on main. + # + # dev/X.Y — dev/X.Y is the `main` of the X.Y patch line, so the + # same argument applies one level down: a fix landing on + # dev/1.0 while it still reads 1.0.0 would forward into + # release/1.0 and release nothing. + # + # Only the FIRST PR after a release has to bump. Once the branch reads + # an unreleased version, later PRs onto it pass without bumping again — + # so several fixes accumulate under one pending number, which is what + # ships when the branch is forwarded. + # + # CONSEQUENCE: syncing a release branch with main WITHOUT releasing is + # no longer a single merge. That is the intended trade. + # + # Assign-then-test, never `git ls-remote ... | grep -q`: `set -e` must + # abort on a failed LOOKUP rather than let it read as "not released". + if [ "$base" = "main" ] || [[ "$base" =~ ^(dev|release)/ ]]; then + existing=$(git ls-remote --tags origin "refs/tags/$tag") + if [ -n "$existing" ]; then + echo "::error::$tag is already released (tagged on the remote), so it must not be the current __version__ on '$base'. Bump __version__ in this PR — see contributing.md → Triggering a release." + exit 1 + fi + echo "OK: $tag is not yet on the remote." + else + echo "Base branch '$base' is not a protected staging or release branch; skipping the release-state check." fi - echo "OK: __version__ '$version' matches target branch '$base'." - - build-linux: + test: + # API/service test suite (tests/). Tests that need the bundled MTConnect + # agent binary self-skip when src/bin/agent is absent, so this job only + # builds the venv (not the agent). Not a required status check (yet). runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.12' - - name: Build with make - run: make - - name: Verify binary was produced - run: test -f dist/MTConnectExplorer + - name: Set up venv + run: | + python -m venv venv + venv/bin/pip install -Ur requirements-src.txt + - name: Run pytest + run: venv/bin/python -m pytest tests -q + + # ⭐ THE GATE RUNS THE RELEASE BUILD, NOT A CHEAPER APPROXIMATION OF IT. + # Both jobs below are the SAME composite actions release.yml uses, so the + # third-party notices and the tar.gz/zip packaging are exercised on every PR. + # They used to be release-only ("the gate just needs to know it compiles"), + # which meant a packaging break was first seen after the merge into + # release/X.Y — when the version is already spent and the fix costs another + # bump. See .github/actions/build-linux/action.yml. Only the artifact upload + # is release-specific. + build-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-linux build-windows: runs-on: windows-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Build with build.bat - shell: cmd - run: build.bat ci - - name: Verify binary was produced - shell: pwsh - run: | - if (-not (Test-Path dist\MTConnectExplorer.exe)) { - throw "dist\MTConnectExplorer.exe not found" - } + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-windows diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd67ee2..7f39bf1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,30 @@ name: Release +# Fires on EVERY push to release/* — see contributing.md → Triggering a release. +# +# THERE IS DELIBERATELY NO `paths:` FILTER. It used to be +# `paths: ['src/__version__.py']`, which made a merge into a release branch a +# SILENT no-op whenever the version had not changed: GitHub evaluates a push's +# paths filter against a TWO-DOT diff of the push's before/after SHAs, so merging +# main into release/0.1 while both tips carry the same version matched nothing, +# queued no run, and reported nothing anywhere. A release that does not happen +# must not look identical to one that was never asked for. +# +# Firing every time is cheap and safe because the DEDUPE ALREADY EXISTS one layer +# down: the `version` job below sets should_release=false when the tag implied by +# __version__ is already on the remote, and both build jobs are gated on it. So a +# push that cannot release costs one ~15 s job and leaves a ::notice:: saying why +# — visible, instead of silent. +# +# The flip side: a push to a release branch whose version is NOT yet tagged WILL +# publish, whatever files it touched. What decides is tag state, not which paths +# changed. `ci.yml`'s version-check refuses a PR into release/* whose version is +# already tagged, so the mistake is caught before the merge rather than after it. +# +# No recursion: the tag push below is a TAG, and this trigger matches branches. on: push: branches: ['release/*'] - paths: ['src/__version__.py'] workflow_dispatch: permissions: @@ -19,7 +40,7 @@ jobs: prerelease: ${{ steps.derive.outputs.prerelease }} should_release: ${{ steps.derive.outputs.should_release }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - id: derive @@ -29,18 +50,10 @@ jobs: branch="${GITHUB_REF_NAME}" - # Read __version__ from src/__version__.py - version=$(grep -oP '"\K[^"]+' src/__version__.py) - if [ -z "$version" ]; then - echo "::error::Failed to parse __version__ from src/__version__.py" - exit 1 - fi - - # Validate semver - if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then - echo "::error::__version__ '$version' is not valid semver (expected MAJOR.MINOR.PATCH[-PRERELEASE])" - exit 1 - fi + # Reads + semver-validates in one place, shared with ci.yml. Do NOT + # inline a parse here: the two inlined copies this replaced were both + # unanchored. See the script's header. + version=$(bash .github/scripts/read-version.sh) tag="v$version" @@ -67,9 +80,19 @@ jobs: # On a `push` trigger, skip if the tag already exists on the remote — this # commit didn't actually bump the version (e.g. a revert or a no-op edit). # On `workflow_dispatch`, always proceed so broken releases can be re-run. + # + # THIS IS THE ONLY THING STOPPING A DUPLICATE RELEASE now that the + # trigger has no paths filter — both build jobs are gated on it. Assign + # first, then test: `git ls-remote ... | grep -q` would report "tag + # absent" for a NETWORK failure too (and `grep -q` exits early, so under + # `pipefail` a SIGPIPE'd producer can fail an otherwise-successful + # pipeline). With the assignment, `set -e` aborts the job if the query + # itself fails, which is the safe direction — a failed lookup must never + # read as "not released yet". should_release=true if [ "${{ github.event_name }}" = "push" ]; then - if git ls-remote --tags origin "refs/tags/$tag" | grep -q "refs/tags/$tag"; then + existing=$(git ls-remote --tags origin "refs/tags/$tag") + if [ -n "$existing" ]; then echo "::notice::Tag $tag already exists on remote — skipping (use workflow_dispatch to re-run)" should_release=false fi @@ -93,51 +116,33 @@ jobs: needs: version if: needs.version.outputs.should_release == 'true' runs-on: ubuntu-latest - env: - ARCHIVE: MTConnectExplorer-v${{ needs.version.outputs.version }}-linux-x86_64.tar.gz steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Build with make - run: make - - name: Package tar.gz - run: | - test -f dist/MTConnectExplorer - chmod +x dist/MTConnectExplorer - tar -czvf "$ARCHIVE" -C dist MTConnectExplorer - - uses: actions/upload-artifact@v4 + - uses: actions/checkout@v7 + # Shared composite action, identical to the one the PR gate runs — see + # .github/actions/build-linux/action.yml for why. It derives the archive + # name from the same version reader and has already asserted the archive + # exists and holds the binary, so the upload is the only step left. + - id: build + uses: ./.github/actions/build-linux + - uses: actions/upload-artifact@v7 with: name: linux - path: ${{ env.ARCHIVE }} + path: ${{ steps.build.outputs.archive }} if-no-files-found: error build-windows: needs: version if: needs.version.outputs.should_release == 'true' runs-on: windows-latest - env: - ARCHIVE: MTConnectExplorer-v${{ needs.version.outputs.version }}-win64.zip steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Build with build.bat - shell: cmd - run: build.bat ci - - name: Package zip - shell: pwsh - run: | - if (-not (Test-Path dist\MTConnectExplorer.exe)) { - throw "dist\MTConnectExplorer.exe not found" - } - Compress-Archive -Path dist\MTConnectExplorer.exe -DestinationPath "$env:ARCHIVE" -Force - - uses: actions/upload-artifact@v4 + - uses: actions/checkout@v7 + # Shared composite action, identical to the one the PR gate runs. + - id: build + uses: ./.github/actions/build-windows + - uses: actions/upload-artifact@v7 with: name: windows - path: ${{ env.ARCHIVE }} + path: ${{ steps.build.outputs.archive }} if-no-files-found: error release: @@ -145,8 +150,8 @@ jobs: if: needs.version.outputs.should_release == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 with: path: artifacts merge-multiple: true @@ -159,14 +164,16 @@ jobs: set -euo pipefail git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - if git ls-remote --tags origin "refs/tags/$TAG" | grep -q "refs/tags/$TAG"; then + # Assign-then-test, not `| grep -q` — see the note in the `version` job. + existing=$(git ls-remote --tags origin "refs/tags/$TAG") + if [ -n "$existing" ]; then echo "Tag $TAG already exists on remote — reusing" else git tag "$TAG" git push origin "$TAG" fi - name: Create release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ needs.version.outputs.tag }} name: ${{ needs.version.outputs.tag }} @@ -185,7 +192,8 @@ jobs: echo "::notice::${{ needs.version.outputs.tag }} is a prerelease — $DEV is created on the first stable release, skipping" exit 0 fi - if git ls-remote --heads origin "refs/heads/$DEV" | grep -q "refs/heads/$DEV"; then + existing=$(git ls-remote --heads origin "refs/heads/$DEV") # see the `version` job + if [ -n "$existing" ]; then echo "::notice::$DEV already exists — leaving it untouched" else git config user.name "github-actions[bot]" diff --git a/.gitignore b/.gitignore index 3ede49d..e7b9f40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ -.claude/ +.pytest_cache/ +.claude/*.local.json build/ datasets/ dist/ diff --git a/.streamlit/config.toml b/.streamlit/config.toml deleted file mode 100644 index d2fcbe1..0000000 --- a/.streamlit/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Streamlit configuration used for `streamlit run src/app.py` (dev). -# The packaged build sets the same options via flag_options in src/main.py. -[server] -# Maximum size (in MB) of a single uploaded dataset file: 5 GB. -maxUploadSize = 5120 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e91cd25 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,246 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +MTConnectExplorer is an early-stage app for exploring, capturing, replaying, and simulating +[MTConnect](https://github.com/mtconnect/cppagent) machine-tool data, implementing the Testbed as a +Service (TaaS) capture/replay method (see [readme.md](readme.md)). It is a **Streamlit frontend + +FastAPI backend in one process, on one port** (UI at `/`, API at `/api/v1`, Swagger at `/api/docs`), +plus a Python client wheel. + +## Commands + +All commands assume the repo root. Windows uses `.bat` scripts; Linux uses the `makefile`. The +`venv/` and the bundled MTConnect agent (`src/bin/agent[.exe]`) are **not** committed — the setup +scripts download the agent from the cppagent GitHub releases and build the venv from +[requirements-src.txt](requirements-src.txt), which also editable-installs `lib/`. + +| Task | Windows | Linux | +|---|---|---| +| Set up venv + agent + editable lib | `dev.bat` | `make build` | +| Run the app (dev) | `run.bat` | `venv/bin/python -m src` | +| Run the test suite | `venv\Scripts\python.exe -m pytest tests` | `venv/bin/python -m pytest tests` | +| Build a standalone binary | `build.bat` → `dist/MTConnectExplorer.exe` | `make` → `dist/MTConnectExplorer` | +| Build just the client wheel | `venv\Scripts\python.exe -m build --wheel lib` | `venv/bin/python -m build --wheel lib` | +| Clean all generated dirs | `clean.bat` | `make clean` | + +- `run.bat` forwards extra args to the app, e.g. `run.bat --port 9000 --host 0.0.0.0`. +- The app must be launched via `python -m src` ([src/main.py](src/main.py)) — it composes the + Streamlit app and the FastAPI API into one ASGI app (`streamlit.starlette.App` + + `Mount("/api", …)`). Bare `streamlit run src/app.py` starts the UI **without the API**, so the + pages (which talk to the API) will not work. +- Release builds (`build.bat` / `make`) build the `lib/` wheel and `pip install --force-reinstall` + it before PyInstaller, so the frozen binary uses (and proves) the real wheel; `dev.bat` / `make` + restore the editable install. `pip freeze --exclude mtconnect-explorer-client > requirements.txt` + keeps the machine-specific editable path out of the lockfile. +- Run a single test: `venv\Scripts\python.exe -m pytest tests/test_datasets.py::test_rename -q`. + Tests needing the bundled agent self-skip when `src/bin/agent[.exe]` is missing (so CI's `test` + job runs without it). + +### Integration tests (pre-API, still authoritative for the data path) + +Three end-to-end flows in [src/test.py](src/test.py), each a CLI flag `sys.exit()`-ing pass/fail: + +```bash +python -m src --loopback-test # PRIMARY correctness check: data → agent → capture → diff round-trip, no network +python -m src --mirror-test # mirrors https://demo.mtconnect.org/ through a local agent +python -m src --play-test # replays a captured dataset through a local agent +``` + +CI ([.github/workflows/ci.yml](.github/workflows/ci.yml)) runs `pytest` (ubuntu, agent tests +auto-skip) and verifies the app builds on Linux and Windows; it does **not** run the flows above. + +### Headless SHDR-log conversion + +```bash +python -m src --convert-shdr-log adapter.log --devices Devices.xml --out datasets/out.jsonl +``` + +## Architecture + +### The core insight: this app plays both sides of MTConnect + +An MTConnect deployment is *SHDR adapter → agent (cppagent) → HTTP clients*. MTConnectExplorer acts +as an **HTTP client** (Explore/Capture) *and* as an **SHDR adapter + agent host** (Play/Simulate). +For Play/Simulate it launches the bundled `agent[.exe]` in a temp dir with a generated `agent.cfg` + +`Devices.xml`, and the app's own `ShdrAdapter` (a TCP server) is the SHDR source that agent connects +to. Downstream tools then consume the local agent over HTTP. + +### The three-layer split (+ the client wheel) + +``` +lib/ mtconnect-explorer-client wheel — pydantic schema home + sync httpx ApiClient +src/services/ Streamlit-free domain managers (sessions, datasets, ports, events, auth) +src/api/ FastAPI routers (thin wrappers over services), mounted at /api +src/pages/ Streamlit frontend — talks to the API via the lib client (Explore excepted) +src/{client,shdr,agent,converter,dataset,simulator,viz}.py domain code (unchanged data path) +``` + +**Boundary rules (enforced by review, not tooling):** + +- Nothing under `src/services/` or `src/api/` may import `streamlit`. +- `lib/` imports nothing from `src/` (the dependency points the other way: services/api/pages all + import schemas from `mtconnect_explorer_client.schemas` — the single source of truth for the wire + contract). +- Pages import only the lib client + [src/pages/core.py](src/pages/core.py) view-state helpers. + [src/pages/api.py](src/pages/api.py) is the one place the frontend touches `src.services` (to + read the runtime token + port). **Explore is the documented carve-out**: it keeps a direct + `MtConnectClient`/`MtConnectExplorer` because an MTConnect agent is itself a REST API. +- [docs/api.md](docs/api.md) is the normative API contract; it wins over any other document + (including this one) on API details. + +### The API (docs/api.md) + +- One process/port: `streamlit.starlette.App(script_path, routes=[Mount("/api", create_api())])` + in [src/main.py](src/main.py). One `APIRouter` per service → one Swagger tag each (System, + Datasets, Capture, Play, Simulate, Events), `summary=` on every route. +- **Auth:** `X-API-Key` ∈ {config.json `api_key`, per-process runtime token + ([src/services/auth.py](src/services/auth.py), handed to the in-proc frontend)}. `/api/v1/health` + and the docs pages are unauthenticated. Default bind `127.0.0.1`; a non-local `--host` without an + `api_key` **refuses to start**. +- **Trusted-LAN only — never expose to the internet.** The `api_key` gates only `/api`; the + **Streamlit UI has no authentication** and drives the same backend via the runtime token, so an + exposed instance is fully operable by anyone who can reach it (and Capture fetches + arbitrary operator-supplied `base_url`s server-side — an SSRF vector). The startup banner states + this; the `--host` gate is a guard, not a safety guarantee. +- **Errors:** services raise typed [src/services/errors.py](src/services/errors.py) exceptions; + the API maps them onto `HTTPException(status, detail={"code","message","context"})`. +- **Simulate:** session creation is **one endpoint per equipment type** — + `POST /simulate/start/{mill_3axis|mill_5axis_trunnion_bc|robot_6dof}` — each with a dedicated + pydantic body so every parameter set is fully validated (a deliberate RPC-style exception to the + resource-REST rule; docs/api.md decision #12). Axis **limits** are meters (linear) / degrees + (rotary), default to native travel, and scale the routine + 3D geometry + published MTConnect + `Constraints`. Catalog entries carry a `create_path`; the generic `POST /simulate/sessions` was removed. +- **Play scrub/seek:** `GET /play/sessions/{id}/progress` returns the playhead origin + (`duration` + `pass_start_epoch` + `server_time`); `POST /play/sessions/{id}/seek` seeks a running + playback (requires re_timestamp — see the scrub subsection below). +- **Events:** one global bus ([src/services/events.py](src/services/events.py)) — monotonic `seq`, + 10k ring buffer, poll (`after` cursor + `dropped`) and NDJSON `/events/stream`. Log records are + ingested from the `src` logger; the `\x1f` thread-name scheme attributes them to sessions + (thread names carry the *session id*; labels are looked up at render time). **Stream clients must + send `Accept-Encoding: identity`** — a gzip layer buffers quiet streams (the lib client does this + automatically). +- **Datasets:** multipart `POST /datasets/upload`, 1 MB-chunked temp file + atomic move, + **no overwrite** (409), path-traversal guard — all in + [src/services/datasets.py](src/services/datasets.py), the sole owner of `datasets/`. + +### Sessions & the worker-host seam + +All three scopes (capture/play/simulate) share one lifecycle +(`starting → running → stopping → finished|error`) implemented in +[src/services/sessions.py](src/services/sessions.py): global ids (`cap-1`, `play-2`, `sim-3`), one +`STATE_LOCK` for atomic port validate-and-reserve across scopes, terminal sessions retained 15 min +(the frontend auto-deletes them to keep the instant-reap UX). Ports are **derived from the live +session table** ([src/services/ports.py](src/services/ports.py)) — no reservation store to leak. + +A worker is a pure function `fn(config: BaseModel, ctx: WorkerContext)` registered in +[src/services/hosts.py](src/services/hosts.py) and started through the `WorkerHost` seam. The +default host is **`SubprocessHost`** — each worker self-execs the app +(`MTConnectExplorer --worker ` frozen, `python -m src --worker ` in dev; dispatched in +main.py before any streamlit import) so each session's tick loop gets its own GIL and a worker +crash can't take the server down. Protocol ([src/services/worker_main.py](src/services/worker_main.py)): +config = one JSON line on stdin; `{"cmd":"stop"}` or stdin EOF (orphan protection) = graceful stop; +the child emits NDJSON `log`/`status`/`exit` records on stdout, which the parent forwards to the +event bus and manager. Flip a scope back to in-process threads by constructing its manager with +`ThreadHost()` (one line per scope module). Workers report transitions via `ctx.set_status(...)` — +e.g. simulate publishes `start_epoch` in the `running` event's detail, which is how the 3D view +gets its phase-lock origin without reaching into the worker. + +The seam is **bidirectional**: `ctx.poll_command()` drains inbound commands delivered by +`SessionManager.send_command(sid, cmd)` — in-process for `ThreadHost`, or an extra stdin +`{"cmd":…}` line (serialized against the stop-writer) for `SubprocessHost`, decoded by +`worker_main.py`'s stdin reader. Play's seek is the first user of this channel. + +### The 3D view stays synced without data + +The simulate routine is deterministic: `GET /simulate/sessions/{id}/routine` returns +`RoutineSpec.to_dict()` + `start_epoch` + `server_time`, and [src/viz.py](src/viz.py) re-runs the +identical piecewise-linear interpolation in JS from `Date.now()` (corrected by the clock offset +computed from `server_time`) — sync is mathematical, not data-communicated. + +### Play scrubbing (seek) — the two-clock rule + +The Play scrub bar reuses the same origin trick: the worker publishes `pass_start_epoch` + `duration` +(+ `paused` + `position`) on start/loop-wrap/seek/pause (not per tick — no chatty bus), and the UI +computes the live playhead as `server_time − pass_start_epoch` (or holds it at `position` while +paused). The player ([src/converter.py](src/converter.py)) models two decoupled clocks: + +- a **playhead target** (`_pos_anchor`/`_wall_anchor`): advances at 1× wall-clock while playing, + frozen while paused, set directly by a seek. A frame at in-pass offset `O` emits once the target + reaches `O` — **unpaced when behind** (a post-seek catch-up), **paced when caught up**. So + **pause just freezes the target** and never blocks the emit loop: a post-seek fast-forward runs to + the frozen target and then idles (`pause()`/`resume()` work in any timestamp mode). +- an **emit-timestamp clock** (`_emit_origin` + monotonic guard), kept separate so a backward + seek/loop-wrap never emits a decreasing timestamp (the agent rejects those). That only holds when + timestamps are rewritten, so **seek requires `re_timestamp=True`** (UI disables the bar otherwise; + backend rejects `409 seek_unavailable`). + +**Seek is snapshot-indexed, not a full rewind.** At `open()` the player pre-scans the recording once, +folding every observation into running device state (`_DeviceStateFold`) and recording a full-state +snapshot (a `/current` equivalent) every `snapshot_interval_s` dataset seconds — each with the byte +offset of the following frame ([DatasetReader.open_at](src/dataset.py) seeks there; offsets are +line-start cookies so there's no partial-line reassembly). A seek jumps to the nearest snapshot ≤ the +target, emits that state, then fast-forwards only the ≤`snapshot_interval_s` of remaining deltas — +**same path forward and backward**, but bounded instead of replaying from the start. +`snapshot_interval_s` (default 15) is a `PlayCreate` field / "Snapshot Interval" launch setting, +fixed once playing. + +### Streamlit view-state model + +`st.session_state` is per-browser-session, so frontend **view** state (form defaults, tree +selections) lives in a process-global `@st.cache_resource` store in +[src/pages/core.py](src/pages/core.py), shared across users and guarded by an `RLock`. Domain state +lives behind the API. Log panes render per-scope `ScopeFeed`s ([src/pages/api.py](src/pages/api.py)) +— background threads consuming `/events/stream` into bounded deques, merged with frontend `[ui]` +lines. Capture/Play/Simulate pages each run a `@st.fragment(run_every=…)` **watcher** that polls a +coarse backend fingerprint `(id, status)` and triggers a full-page rerun only on change, so +API-driven starts/stops appear without touching the page — the fingerprint deliberately excludes +fast-moving values (e.g. the play position) so a quiet backend never reruns the page or dismisses an +open dialog. The play scrub bar advances in its own per-session fragment (see the seek subsection); +a known wart of the native `st.slider` is an occasional mid-drag thumb snap-back at the ~1 s tick. + +### Dataset formats (the header string selects the schema) + +`DatasetReader`/`DatasetWriter` ([src/dataset.py](src/dataset.py)) are constructed with a header +string that must match the file's first line. Two schemas are in use: + +- **`RawMTConnect-v0.1`** — payload is a `MtConnectClientFrame` (a request + its full raw XML + response). Written by Capture; read by Play and both converters. First frame **must** be a `probe`. +- **`Shdr-v0.1`** — payload is a single SHDR observation (`ID` = the observation class name). + Produced by `XmlToShdrConverter.convert_file` and the loopback-test generator; consumed by + `ShdrDataPlayer`. + +### Versioning & release + +`__version__` in [src/__version__.py](src/__version__.py) is the single source of truth for the +app; `lib/src/mtconnect_explorer_client/__version__.py` mirrors it in PEP 440 form (`0.1.0-alpha.5` +→ `0.1.0a5`) — **bump both together**. Bumping on a `release/*` branch *is* cutting a release; the +branch model and rulesets are documented in [contributing.md](contributing.md) — read it before +touching branches, `__version__`, or the workflows. + +## Gotchas + +- **The bundled agent reads a float with a trailing `.0` back as an int**, breaking SHDR loopback + type round-tripping. Values are stripped of a trailing `.0` before emission — see `_fmt()` in + [simulator.py](src/simulator.py) and `add_observation_if_changed` in [test.py](src/test.py). +- **Use `127.0.0.1`, not `localhost`**, for adapter/agent hosts — IPv6 resolution causes lag on + Windows (noted in [agent.py](src/agent.py)). +- **Start the SHDR adapters before the agent** when a session fronts **multiple** adapters. If the + bundled agent starts before any adapter is listening, on its connection-retry sweep it + deterministically skips the **first-declared adapter**, so an N-device session connects to only + N-1 and hangs in `starting` at the connect-wait. [play_worker](src/services/play.py) opens all + adapters first, then starts the agent; single-adapter simulate/loopback are immune (which is why + it went unnoticed). The connect-wait is bounded and raises with the un-connected device names. +- Paced players guard against corrupt/misordered data: `ShdrDataPlayer`/`DatasetPlayer` reject a + frame >120 s from the pass start; `MtConnectDatasetPlayer` rejects a >10 min gap between frames. +- `config.json` (optional, gitignored) is parsed as JSONC — comments are stripped before pydantic + validation ([src/services/config.py](src/services/config.py)); read with `utf-8-sig` so a + Windows editor's UTF-8 BOM doesn't break parsing. Fields: `log_level`, `api_key`. +- **Streamlit `st.slider` + write-before-create is a trap**: re-seeding a slider's value via + `session_state` each tick fires its `on_change`, so an `on_change` seek handler self-triggers a + feedback loop. The play scrub bar instead compares the slider's committed value to the exact value + it last wrote (both rounded to the step) to tell a real drag from the programmatic re-seed. +- `CLAUDE.md`, `.claude/`, `config.json`, `datasets/`, `dist/`, `build/`, `src/bin/`, `test/` + (but **not** `tests/`), and `venv/` are all gitignored. +- Both build entry points use the same [installer.spec](installer.spec) (PyInstaller), which bundles + all of `src/` and reads `__version__` to stamp Windows file-version fields. diff --git a/LICENSE.TXT b/LICENSE.TXT new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/LICENSE.TXT @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/build.bat b/build.bat index 2000699..a649922 100644 --- a/build.bat +++ b/build.bat @@ -9,6 +9,24 @@ call dev.bat no_pause || ( exit /b %ERRORLEVEL% ) +:: Build the client wheel and install it over the editable install so the +:: frozen binary uses (and proves) the real wheel +echo Building client wheel... +IF EXIST "lib\dist" RD /S /Q "lib\dist" +venv\Scripts\python.exe -m build --wheel lib || ( + echo Error: Could not build the client wheel + pause + exit /b %ERRORLEVEL% +) +for %%f in (lib\dist\*.whl) do ( + venv\Scripts\pip.exe install --force-reinstall "%%f" || ( + echo Error: Could not install the client wheel + pause + exit /b 1 + ) +) +echo Built client wheel + echo Building application... venv\Scripts\pyinstaller.exe installer.spec --clean || ( echo Error: Could not update venv requirements diff --git a/contributing.md b/contributing.md index f2ba3f1..648a42c 100644 --- a/contributing.md +++ b/contributing.md @@ -17,7 +17,7 @@ This document describes the branch model, version conventions, release process, | Branch | Lifetime | Purpose | |---|---|---| | `main` | long-lived, protected | Staging area for the next minor release. All new features land here first. | -| `release/X.Y` | long-lived, protected | Published release branch for version line `X.Y`. Pushes that modify `__version__` here trigger the release workflow and ship `vX.Y.Z`. | +| `release/X.Y` | long-lived, protected | Published release branch for version line `X.Y`. Every push here runs the release workflow; it ships `vX.Y.Z` unless that tag already exists. | | `dev/X.Y` | long-lived, protected | Staging area for `X.Y.x` patches (`X.Y.1`, `X.Y.2`, …). Created automatically by the release workflow when the first stable `X.Y.0` ships. | | `feature/` | short-lived | Feature work, branched off `main`, merged back via PR. | | `bugfix/` | short-lived | Bug fixes for an existing release line, branched off `dev/X.Y`, merged back via PR. | @@ -64,8 +64,8 @@ flowchart LR M -->|"cut + PR
(new minor)"| RL["release/X.<new>
(published)"] B["bugfix/<name>
(short-lived)"] -->|PR| D["dev/X.Y
(staging for X.Y patches)"] D -->|"PR
(ships patch)"| RX["release/X.Y
(published)"] - RX -.->|"__version__ change"| W["release.yml
workflow"] - RL -.->|"__version__ change"| W + RX -.->|"any push"| W["release.yml
workflow"] + RL -.->|"any push"| W W -->|"creates tag
+ GitHub Release"| GR["v{__version__}"] W -.->|"creates on first
stable X.Y.0"| D ``` @@ -76,13 +76,28 @@ Every published release flows through a staging branch (`main` for the next vers ### Triggering a release -The workflow fires on **pushes to `release/*` branches that modify `src/__version__.py`**. The path - filter means non-version commits on a release branch (e.g. dependency bumps, doc fixes) do not - trigger a release. +The workflow fires on **every push to a `release/*` branch**. What decides whether a release is + actually published is not which files the push touched, but whether the tag implied by + `__version__` already exists on the remote — if it does, the workflow skips itself in ~15 + seconds with a `::notice::` and builds nothing. + +> **The trigger has no `paths` filter, deliberately.** +> +> It used to fire only on pushes that modified `src/__version__.py`. That made merging `main` into +> `release/0.1` *without* a version bump a **completely silent no-op** — GitHub evaluates a +> push's `paths` filter against a two-dot diff of the push's before/after SHAs, so when both tips +> carried the same version the filter matched nothing, no run was queued, and nothing was +> reported anywhere. A release that does not happen must not look identical to one that was +> never requested. +> +> ⚠️ **Consequence:** a push to a release branch whose version is **not yet tagged** will publish, +> whatever files it touched. Doc fixes and dependency bumps are no longer automatically +> release-free — land them on `main` (or after the release) rather than on a release branch +> sitting at an unreleased version. ```mermaid flowchart LR - A["PR merge to release/0.1
bumping __version__ to 0.1.1"] -->|push| B["release.yml
workflow"] + A["PR merge to release/0.1
(any push)"] -->|push| B["release.yml
workflow"] B --> C{"Tag v0.1.1
already exists?"} C -->|yes| Skip["Skip — already released"] C -->|no| D["build-linux + build-windows"] @@ -99,6 +114,39 @@ Before building, the workflow validates two things: - **The branch name matches the version's `X.Y`** — `release/0.1` must hold a `0.1.x` version, not `0.2.x`. This catches the "merged the wrong PR into the wrong branch" mistake. +The PR gate checks two more things, **before** the merge: + +- **A PR into `main`, `dev/X.Y` or `release/X.Y` whose `__version__` is already tagged on the remote is + refused.** + A released version must never be the *current* version on a staging or release branch. Into + `release/*`, merging it could only produce a run that skips itself. Into `main`, the stale + number would ride the next forward-merge into `release/*` and silently produce nothing there. + Refusing it at the door is what stops a released version from trickling anywhere. +- ⚠️ **Two consequences.** The first PR into `main` after a release must carry the version bump — + until someone bumps it, `main` sits at a released version and the gate stays red. And syncing a + release branch with `main` *without* releasing is no longer a single merge: bump the version, or + push the sync commits some other way. + +Every protected branch is covered: `main`, `dev/X.Y` and `release/X.Y`. `dev/X.Y` is the `main` of + its patch line, so the same reasoning applies one level down — a fix landing on `dev/1.0` while + it still reads `1.0.0` would forward into `release/1.0` and release nothing. + +**Only the first PR after a release has to bump.** Once a branch reads an unreleased version, later + PRs onto it pass without bumping again, so several fixes accumulate under one pending number — + which is what ships when the branch is forwarded. Concretely, after `v1.0.0`: + +| PR | Version on the branch | Gate | +|---|---|---| +| bugfix A into `dev/1.0`, no bump | `1.0.0` (released) | ❌ refused — bump it | +| bugfix A into `dev/1.0`, bumped | `1.0.1` | ✅ | +| bugfix B into `dev/1.0`, no bump needed | `1.0.1` | ✅ | +| bugfix C into `dev/1.0`, no bump needed | `1.0.1` | ✅ | +| `dev/1.0` → `release/1.0` | `1.0.1` | ✅ ships `v1.0.1` with A + B + C | +| bugfix D into `dev/1.0`, no bump | `1.0.1` (now released) | ❌ refused — bump to `1.0.2` | + +Short-lived branches (`feature/*`, `bugfix/*`, `forwardport/*`) are not PR targets, so nothing + changes for them. + If the tag implied by `__version__` already exists on the remote, the workflow skips (preventing re-runs on a non-version commit, or on the same commit being pushed twice). To re-run a release — for example if the artifact upload flaked — invoke the workflow manually via the **Run @@ -274,7 +322,9 @@ If multi-commit forward-ports become routine, consider configuring `dev/*` to us merge** for bugfix PRs — every bugfix then lands on the dev branch as a single commit, and forward-porting is always a single `git cherry-pick`. Trade-off: the granular history of how the fix was built up is collapsed. If you also want to keep the `__version__` bump out of the - squashed commit, split the bugfix into two PRs (fix-only, then bump). + squashed commit, split the bugfix into two PRs — but **bump first, then fix**. The version + gate refuses a PR onto `dev/X.Y` while the branch still reads the version that just + shipped, so a fix-only PR is red until the bump has landed. Reversing the order avoids it. ## Creating a new release @@ -365,8 +415,8 @@ Protection is configured with **Rulesets** (Settings → Rules → Rulesets), no | Setting | Value | |---|---| -| Allow merge commits | true (Default message) | -| Allow squash merging | true (Default message) | +| Allow merge commits | true (Pull request title) | +| Allow squash merging | true (Pull request title) | | Allow rebase merging | true | All three are enabled because a ruleset can only *narrow* the allowed merge methods, never add one. @@ -466,7 +516,7 @@ This overlays `release/*` with the rules that should be **admin-bypassable**. Re | Require status checks to pass | true | | Require branches to be up to date before merging | true | | Do not require status checks on creation | false | -| Status checks that are required | [build-windows, build-linux] | +| Status checks that are required | [build-windows, build-linux, version-check] | | Block force pushes | true | Rationale: @@ -508,12 +558,37 @@ Rationale: ### Pre-merge build check (`ci.yml`) [`.github/workflows/ci.yml`](.github/workflows/ci.yml) runs on **`pull_request` events targeting - `main`, `dev/*`, and `release/*`**. It builds the app on both shipped platforms (`build-linux` - via `make`, `build-windows` via `build.bat ci`) and fails if the binary isn't produced — catching - build, packaging, and import breakage *before* a PR merges. It does **not** package or upload - artifacts, and it doesn't run the integration loopback test in [`src/test.py`](src/test.py). It - also runs `version-check`, which is a no-op unless the PR targets `release/*` (above). Required - status-check names appear in a ruleset's search box after the workflow has run once on a PR. + `main`, `dev/*`, and `release/*`**, with four jobs: + +| Job | What it does | +|---|---| +| `build-linux` | The **complete** Linux release build, via the shared [`build-linux`](.github/actions/build-linux/action.yml) composite action: `make` → binary assertion → third-party notices → `tar.gz` → archive assertions. | +| `build-windows` | The same for Windows, via [`build-windows`](.github/actions/build-windows/action.yml): `build.bat ci` → `.exe` assertion → notices → `.zip` → archive assertions. | +| `test` | `pytest tests` against a venv built from `requirements-src.txt`. Tests needing the bundled MTConnect agent binary self-skip when `src/bin/agent` is absent. Not a required status check. | +| `version-check` | Always validates that `__version__` parses and is semver. For PRs into `main`, `dev/X.Y` *or* `release/X.Y` it also refuses a version that is already released; for `release/*` it additionally checks the `X.Y` line against the branch (above). | + +> **⭐ The gate runs the release build, not a cheaper approximation of it.** +> +> It used to stop after the binary existed — the third-party notices and the `tar.gz`/`.zip` +> packaging were **release-only**. That meant a packaging break was first seen *after* the merge +> into `release/X.Y`, when the version is already spent and the fix costs another bump and +> another merge. [`packaging/generate_notices.py`](packaging/generate_notices.py) is the live +> hazard: it walks installed package metadata, so a new dependency with missing or malformed +> metadata breaks it — a change the old gate passed happily. +> +> The build now lives in **one composite action per OS**, shared verbatim with `release.yml`; only +> the artifact upload is release-specific. Both actions derive the archive name from the same +> version reader, so the gate names artifacts exactly as the release will. Cost: the gate is +> slower by the packaging time. + +`version-check` reads the version through + [`.github/scripts/read-version.sh`](.github/scripts/read-version.sh) rather than inlining a + parse; `release.yml` and both composite actions call the same script. That is deliberate — the + two workflows used to carry their own copy of the parse, and two copies of a parse drift. + Required status-check names appear in a ruleset's search box after the workflow has run once on + a PR. + +The gate still does not run the integration loopback test in [`src/test.py`](src/test.py). The release workflow itself is **not** a required status check: it runs *after* merge (its trigger is the resulting push to `release/*`), so it can't be a pre-merge gate — which is exactly why diff --git a/dev.bat b/dev.bat index 742e8ab..39367d8 100644 --- a/dev.bat +++ b/dev.bat @@ -44,7 +44,8 @@ echo Updating venv... pause exit /b %ERRORLEVEL% ) -.\venv\Scripts\pip.exe freeze > requirements.txt +:: Exclude the editable lib install - its machine-specific path doesn't belong in the lockfile +.\venv\Scripts\pip.exe freeze --exclude mtconnect-explorer-client > requirements.txt echo Updated venv echo Dev completed successfully diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..8b674fe --- /dev/null +++ b/docs/api.md @@ -0,0 +1,560 @@ +# MTConnectExplorer HTTP API — v1 + +> **Status: IMPLEMENTED** (2026-07-24). This document is the normative wire +> contract; the generated OpenAPI at `/api/openapi.json` (docs at `/api/docs`) +> reflects it. The pydantic schemas live in the `mtconnect-explorer-client` +> wheel (`lib/`), which also ships the reference Python client. + +The API exposes everything the backend owns: **capture, play, and simulate +sessions; datasets; logs; ports; running agents**. It is served by the same +process as the Streamlit UI (UI at `/`, API at `/api`, interactive docs at +`/api/docs`). + +**Deliberately not in v1** (documented so they read as decisions, not gaps): + +- **Live exploration.** An MTConnect agent is already a REST API; clients + explore by calling the agent's own `probe`/`current`/`sample` endpoints + directly — exactly what the built-in Explore page does. +- SHDR-log → dataset conversion endpoint (CLI only for now). +- WebSockets/SSE (NDJSON-over-HTTP streaming covers v1 needs). +- Pagination (list sizes are small; revisit if that changes). + +--- + +## 1. Conventions + +| Topic | Convention | +|---|---| +| Base path | `/api/v1` | +| Auth | `X-API-Key: ` header on every request (see §2). | +| Request/response bodies | JSON (pydantic models end-to-end; OpenAPI generated from them). | +| Streaming | `application/x-ndjson` — one JSON object per line (see §8). | +| Dataset content | `application/octet-stream`, streamed. | +| Timestamps | RFC 3339 UTC with milliseconds: `2026-07-23T18:04:11.234Z`. | +| Session IDs | Opaque strings, prefixed by scope: `cap-1`, `play-2`, `sim-3`. | +| Scopes | `capture`, `play`, `simulate`. | +| Errors | Typed object inside FastAPI-native `detail` (see §3). | +| OpenAPI tags | One Swagger tag per service — `System`, `Datasets`, `Capture`, `Play`, `Simulate`, `Events` — implemented as one `APIRouter` per tag, so `/api/docs` groups endpoints cleanly by service. | +| Reference style | Follows the AMT-DTP ISE/ISCP service layout: router-per-service with `prefix` + `tags`, `summary=` on every route, docs at `/api/docs` + `/api/redoc` + `/api/openapi.json`, `title`/`version` from `__version__`, and a global unhandled-exception handler. Known deviations: resource-REST paths with ids in path segments (vs. the reference's RPC-style `/list`/`/open` with ids in bodies), and API-key auth (reference has none). | + +## 2. Authentication + +> **Deployment posture: trusted LAN only — never the internet.** The `api_key` +> below gates the `/api` endpoints, **not the Streamlit UI**, which has no +> authentication at all. The UI drives the same backend (via the runtime +> token), so anyone who can reach an exposed instance can operate it regardless +> of the key. The refuse-to-start guard forces a key when binding beyond +> localhost, but that hardens only the direct-API surface — it does not make an +> internet-facing deployment safe. One key sharp edge: Capture issues HTTP +> requests to an arbitrary operator-supplied `base_url` server-side (an SSRF +> primitive on an exposed host). Run this only on a network where every machine +> that can reach it is trusted. + +- **Static key** — `api_key` field in `config.json` (JSONC). For external + clients and any future remote UI. +- **Runtime token** — generated at startup, handed to the in-process Streamlit + frontend directly (never written to disk). Middleware accepts either key. +- Default bind is `127.0.0.1`. Exposing requires an explicit `--host`. +- **Resolved:** non-local `--host` with no `api_key` configured → the process + **refuses to start**. +- **Resolved:** `GET /api/v1/health` is the *only* unauthenticated API + endpoint (liveness probes can't send keys). +- The interactive reference pages — `/api/docs`, `/api/redoc`, + `/api/openapi.json` — are likewise served without a key (a browser can't + send headers to read documentation); Swagger's **Authorize** button carries + the key for try-it-out calls. They live outside `/api/v1` and expose no data. + +Auth failures return `401` with error code `unauthorized`. + +## 3. Error model + +Every non-2xx response carries a **typed object inside FastAPI's native +`detail` field** — harmonized with the AMT-DTP services (which use `detail` +strings) while keeping machine-readable codes: + +```json +{ + "detail": { + "code": "port_conflict", + "message": "agent port 5000 is in use by play-2", + "context": {"conflicts": [{"port": 5000, "held_by": "play-2", "kind": "agent"}]} + } +} +``` + +Route code raises plain `HTTPException(status_code, detail={...})`. A global +unhandled-exception handler (adopted from the reference services) returns +`500` with `{"detail": {"code": "internal_error", "message": "..."}}`. +Exception: FastAPI's built-in `422` validation errors keep their native format +(`detail` is a *list* of field errors) — clients should handle `detail` being +either the typed object or that list. + +| HTTP | `code` | When | +|---|---|---| +| 401 | `unauthorized` | Missing/invalid API key | +| 404 | `session_not_found` | Unknown session id (incl. already purged) | +| 404 | `dataset_not_found` | Unknown dataset name | +| 409 | `port_conflict` | Requested port(s) held by a live session (`detail.conflicts` lists each) | +| 409 | `dataset_in_use` | Dataset is being played/captured (rename/delete/overwrite/play-while-capturing) | +| 409 | `dataset_exists` | Upload/rename target already exists (overwriting is not allowed) | +| 413 | `too_large` | Upload exceeds configured cap (default 5 GiB) | +| 415 | `unsupported_type` | Upload extension/MIME is not a `.jsonl` dataset | +| 422 | `validation_error` | Pydantic validation — FastAPI-native format (`detail` is a list of field errors) | +| 422 | `invalid_dataset` | File exists but header/probe frame is malformed | +| 422 | `invalid_name` | Filename fails sanitization (§6) | + +## 4. System + +### `GET /api/v1/health` *(unauthenticated)* +```json +{"status": "ok"} +``` + +### `GET /api/v1/version` +```json +{"version": "0.1.0-alpha.5", "api": "v1"} +``` + +### `GET /api/v1/agents` + +Base URLs of local MTConnect agents run by live Play/Simulate sessions +(feeds the Explore/Capture quick-fill buttons). De-duplicated. + +```json +[ + {"base_url": "http://localhost:5000/", "sessions": ["play-2"]}, + {"base_url": "http://localhost:5001/", "sessions": ["sim-1"]} +] +``` + +### `GET /api/v1/ports?shdr_count=3` + +Used ports across all live sessions plus free suggestions for pre-filling +launch forms. + +```json +{ + "used": [5000, 7878, 7879], + "suggested": {"agent": 5001, "shdr": [7880, 7881, 7882]} +} +``` + +**Port lifecycle:** the used-set is *derived from the live session table* — +there is no separate reservation store to leak. A session holds its ports +through every non-terminal state **including `stopping`**; the moment it +reaches `finished`/`error` or is deleted, its ports are free. Reservation is +atomic inside session creation (single manager lock), so two concurrent `POST`s +cannot both claim the same port — the loser gets `409 port_conflict`. + +## 5. Sessions (common model) + +All three scopes share one lifecycle and envelope. + +```json +{ + "id": "play-2", + "scope": "play", + "label": "demo.jsonl | Mazak | localhost:5000", + "status": "running", + "created_at": "2026-07-23T18:03:59.101Z", + "started_at": "2026-07-23T18:04:11.234Z", + "finished_at": null, + "error": null, + "ports": {"agent": 5000, "shdr": [7878]}, + "config": { "…scope-specific config echo…": "…" } +} +``` + +The `config` echo is **typed per scope** (`CaptureSessionInfo` / +`PlaySessionInfo` / `SimulateSessionInfo` subclass the common `SessionInfo`), +so OpenAPI and the client library expose the exact create-body shape back. + +**Status machine:** `starting → running → (stopping →) finished | error` + +- `starting` — worker launched; agent booting / connecting to SHDR adapter(s). +- `running` — data flowing (Play's `started_event`; Simulate's `start_epoch` set; + Capture's `is_collecting`). "Looping" is not a status — clients derive it + from `config.loop == true && status == running`. +- `error` — worker died; `error` field carries the message (today this only + reaches logs). + +**Lifecycle endpoints (per scope; `{s}` = `capture` | `play` | `simulate`):** + +| Endpoint | Behavior | +|---|---| +| `POST /api/v1/{s}/sessions` | Validate (ports, dataset, …) atomically → `201` + session envelope, or `409`/`404`/`422`. | +| `GET /api/v1/{s}/sessions` | List (includes terminal sessions not yet purged). | +| `GET /api/v1/{s}/sessions/{id}` | Single session. `404` after purge/delete. | +| `DELETE /api/v1/{s}/sessions/{id}` | Stop (signal, join with timeout) and remove → `204`. Idempotent-ish: `404` if unknown. | +| `DELETE /api/v1/{s}/sessions` | Stop-all for the scope → `204`. | + +**Resolved — terminal retention:** the *primary* completion signal is the +`session` status event on the event bus (§8) — streaming and cursor-polling +clients are notified without relying on retention at all. Terminal sessions +additionally **remain listed** until `DELETE` or auto-purge after **15 +minutes** as a REST-ergonomics safety net, so plain request/response clients +(curl, scripts) can still query the outcome. The frontend auto-`DELETE`s +terminal sessions to preserve today's instant-reap look. + +## 6. Datasets + +Names must end in `.jsonl` and pass sanitization: no path separators, no +`< > : " / \ | ? *`, no leading dots; the resolved path must remain inside the +datasets directory (traversal guard). Violations → `422 invalid_name`. + +### `GET /api/v1/datasets` +```json +[ + { + "name": "demo.jsonl", + "size_bytes": 73400320, + "modified": "2026-07-23T14:02:11.000Z", + "in_use": ["play"] + } +] +``` +`in_use` ⊆ `["play", "capture"]` — empty when idle. (This replaces today's +cross-page `_dataset_status()` reach-around with one authoritative answer.) + +### `POST /api/v1/datasets/upload` + +Multipart upload (`multipart/form-data`, single `file` field; the dataset name +comes from the part's filename) — matching the AMT-DTP ISE/ISCP service +convention. The server streams the part to a temp file in 1 MB chunks, +validates the name (rules above; extension must be `.jsonl`), then +**atomically moves** it into `datasets/` — a partially-uploaded file is never +visible to Play/Capture. `201` with the new dataset's metadata (the same shape +as a list entry) on success. + +**Overwriting is not allowed** (deliberate deviation from the reference, which +replaces existing targets silently — datasets are irreplaceable recordings): +uploading onto an existing name is `409 dataset_exists`; to replace a dataset, +delete it first (itself guarded by `dataset_in_use`). + +Errors: `415 unsupported_type` (extension/MIME is not a `.jsonl` upload), +`409 dataset_exists`, `409 dataset_in_use`, `413 too_large`, +`422 invalid_name`. (If a request model is ever needed alongside the file, use +the reference's `Json[Model]` form field + `openapi_extra` multipart-schema +helper.) + +### `GET /api/v1/datasets/{name}` → metadata (same shape as list entry) +### `GET /api/v1/datasets/{name}/content` → streamed download +### `PATCH /api/v1/datasets/{name}` +```json +{"new_name": "renamed.jsonl"} +``` +`204`; errors `409 dataset_in_use`, `409 dataset_exists`, `422 invalid_name`. + +### `DELETE /api/v1/datasets/{name}` → `204`; `409 dataset_in_use` + +### `GET /api/v1/datasets/{name}/devices` + +Devices from the dataset's probe frame (for the Play launch form). + +```json +{"devices": [{"id": "d1", "name": "Mazak", "uuid": "m80-uuid"}, {"id": "agent-id", "name": "Agent", "uuid": "…"}]} +``` + +Returns **all** devices; filtering out `Agent` stays a frontend concern. +`422 invalid_dataset` when the header or first frame isn't a valid probe. + +## 7. Scope-specific configs + +### Capture — `POST /api/v1/capture/sessions` + +```json +{ + "base_url": "https://demo.mtconnect.org/", + "mode": "stream", + "interval_ms": 1000, + "timeout_s": 30.0, + "chunk_size": 4096, + "filename": null +} +``` + +- `mode`: `stream` | `sample` | `current`. +- `filename: null` → **server generates it** (same scheme as today: + sanitized base_url + UTC timestamp, collision-checked against files *and* + in-flight captures). Echoed in the session envelope's `config.filename`. + +### Play — `POST /api/v1/play/sessions` + +```json +{ + "dataset": "demo.jsonl", + "devices": [ + {"device_name": "Mazak", "shdr_host": "127.0.0.1", "shdr_port": 7878} + ], + "agent": { + "port": 5000, + "mtconnect_version": "2.7", + "asset_buffer_size": 1024, + "buffer_size": 131072 + }, + "re_timestamp": true, + "loop": true +} +``` + +Validation (all-or-nothing, under the manager lock): dataset exists +(`404`), dataset not being captured (`409 dataset_in_use`), ≥1 device +(`422`), all ports free and mutually distinct (`409 port_conflict` with the +full conflict list, mirroring today's inline messages). + +#### Playback control (running sessions) + +| Endpoint | Effect | +|---|---| +| `GET /api/v1/play/sessions/{id}/progress` | Scrub-bar origin: `duration`, `pass_start_epoch`, `looping`, `paused`, `position`, `server_time`. The UI computes the live playhead as `server_time − pass_start_epoch`; while `paused` is true it holds the playhead at `position` instead (the origin stops advancing). | +| `POST /api/v1/play/sessions/{id}/seek` | Body `{"position": }`. Seek a running playback. Requires re-timestamp mode (`409 seek_unavailable` otherwise); `404`/`409 not_running` as applicable. `204`. | +| `POST /api/v1/play/sessions/{id}/pause` | Freeze the playhead target (stops it advancing). Works in **any** timestamp mode — unlike seek it does not rewind or re-order data. `409 not_running` if not running. `204`. | +| `POST /api/v1/play/sessions/{id}/resume` | Resume a paused playback. `409 not_running` if not running. `204`. | + +Pause/resume ride the same worker command channel as seek. The player models a +**playhead target** (advances at 1× wall-clock while playing, frozen while paused, +set directly by a seek) separate from the **emit cursor** (how far data has been +released): frames emit while their time ≤ target — unpaced when behind (a post-seek +catch-up), paced when caught up. So **pause never blocks the emit loop** — a +post-seek fast-forward runs to the (frozen) target and then idles. The emit-timestamp +clock is kept separate and monotonic so a backward seek/loop-wrap never emits a +decreasing timestamp. + +**Seek uses a keyframe index.** `snapshot_interval_s` on `PlayCreate` (default `15`, +fixed once playing) sets how often — in dataset seconds — a full-state snapshot +(a `/current` equivalent, folded from the recording) is indexed at load, together +with the byte offset of the frame that follows it. A seek jumps to the nearest +snapshot ≤ the target, emits that state, and replays only the ≤ `snapshot_interval_s` +of deltas up to the target — instead of replaying from the start of the recording. +Smaller intervals mean faster seeks but more memory and a longer one-time pre-scan. + +### Simulate + +Session **creation is one endpoint per equipment type** — each with a +dedicated pydantic body model so every equipment parameter set is fully +validated by its own schema (amends decision #10; see decision log #12): + +| Endpoint | Body model | +|---|---| +| `POST /api/v1/simulate/start/mill_3axis` | `Mill3AxisCreate` | +| `POST /api/v1/simulate/start/mill_5axis_trunnion_bc` | `Mill5AxisTrunnionBcCreate` | +| `POST /api/v1/simulate/start/robot_6dof` | `Robot6DofCreate` | + +The lifecycle endpoints (`GET/DELETE /simulate/sessions[...]`, `/routine`) +are shared across types; the session envelope's `config` echo is the +discriminated union of the create models (`equipment_type` is the +discriminator — a defaulted constant, so requests never need to send it). + +**Axis limits.** Every equipment parameter is optional. Linear axes are +**meters**, rotary axes/joints are **degrees**, each a `{"min", "max"}` object +validated `max > min`. Defaults are the equipment's **native travel** (0–1 m +linear; per-axis rotary ranges), so `{}` reproduces the canned motion +identically. Configured limits **scale the routine** (affine, native → +configured, per axis), the 3D scene's geometry/workpiece, and are published as +MTConnect `Constraints` (``/``) on the axis DataItems in the +generated device model. The trunnion's C axis is continuous and has no limits. +Equipment keys are lowercase with axis orientations encoded where they matter +(`_bc`), anticipating future variants. + +#### `GET /api/v1/simulate/equipment` — catalog for the launch form + +```json +[ + { + "type": "mill_3axis", + "label": "3-Axis Mill", + "default_device_name": "Mill", + "routines": ["Default Routine"], + "create_path": "/api/v1/simulate/start/mill_3axis" + }, + {"type": "mill_5axis_trunnion_bc", "label": "5-Axis Trunnion Mill (B/C)", "default_device_name": "TrunnionMill", "routines": ["Default Routine"], "create_path": "/api/v1/simulate/start/mill_5axis_trunnion_bc"}, + {"type": "robot_6dof", "label": "6-DOF Robot", "default_device_name": "Robot", "routines": ["Default Routine"], "create_path": "/api/v1/simulate/start/robot_6dof"} +] +``` + +Each `create_path`'s request schema is that operation's body in +`/api/openapi.json` — generic clients can build launch forms without +hardcoding equipment knowledge. + +#### `POST /api/v1/simulate/start/mill_5axis_trunnion_bc` (example) + +```json +{ + "device_name": "DMU50", + "routine": "Default Routine", + "interval_ms": 10, + "agent": {"port": 5000, "mtconnect_version": "2.7", "asset_buffer_size": 1024, "buffer_size": 131072}, + "shdr": {"host": "127.0.0.1", "port": 7878}, + "x": {"min": 0.0, "max": 0.5}, + "y": {"min": 0.0, "max": 0.5}, + "z": {"min": 0.0, "max": 0.5}, + "b": {"min": -110.0, "max": 30.0} +} +``` + +Every field is optional (`{}` is a valid body). The robot's model takes +`j1`…`j6` limit objects (degrees) instead of `x`/`y`/`z`/`b`; its link lengths +are fixed physical structure, not parameters. + +#### `GET /api/v1/simulate/sessions/{id}/routine` — feeds the 3D view + +```json +{ + "spec": { "…RoutineSpec.to_dict()…": "equipment, coord_names, start, cycle_time, geometry, moves" }, + "start_epoch": 1753293851.204, + "server_time": "2026-07-23T18:04:15.732Z" +} +``` + +- `start_epoch` is `null` until the session reaches `running` (no data + timeline to phase-lock to yet — the UI keeps its current "reopen in a + moment" behavior). +- `server_time` lets the client compute a clock offset once + (`offset = Date.now() − server_time`), making the phase-lock correct even if + a future remote-UI mode ships. Costs nothing now. + +## 8. Events & logs (NDJSON) + +One **global event bus**: every event gets a **globally monotonic `seq`** and a +**`ts`** (both mandatory on *every* line, all event types). Events are held in +a bounded ring buffer; `scope` / `session` / `level` are *filters*, not +separate counters — so any filter combination works with a single cursor. + +**Default (accepted):** ring buffer capacity **10,000 events**, memory-only +(no disk persistence). This also bounds what today is an unbounded +`log_lines` list. + +### Event records + +```jsonl +{"type":"log","seq":10423,"ts":"2026-07-23T18:04:11.234Z","scope":"play","session":"play-3","level":"INFO","logger":"src.converter","msg":"playback started"} +{"type":"session","seq":10424,"ts":"2026-07-23T18:04:11.240Z","scope":"play","session":"play-3","event":"status","status":"running"} +{"type":"session","seq":10425,"ts":"2026-07-23T18:05:02.001Z","scope":"simulate","session":"sim-2","event":"status","status":"running","detail":{"start_epoch":1753293851.204}} +{"type":"ping","seq":10425,"ts":"2026-07-23T18:05:17.000Z"} +``` + +- `log` — a backend log record attributed to a session (the `\x1f` + thread-name mechanism becomes an internal detail of the ingest path). +- `session` — lifecycle transitions (`status` events on every state change), + so a streaming client never needs to poll for status. +- `ping` — heartbeat every **15 s** of stream idle time, so proxies/clients + can distinguish "quiet" from "dead". +- **Resolved — ping `seq` semantics:** pings satisfy the "seq on every + line" rule by carrying the **last delivered event's seq** (a cursor + checkpoint: "you are current through N"). They are *not* stored in the + buffer and do *not* increment the counter — otherwise backfills would replay + stale pings, and cursor gaps would be indistinguishable from missed events. +- **Integrity contract:** within a connection, TCP already guarantees ordering + and delivery — `seq` exists for *cross-reconnect* continuity (the `after` + cursor) and buffer-overrun detection (`dropped`). Because the counter is + global, a *filtered* view legitimately skips values, so "seq increments by + exactly 1" is **not** a valid gap check on filtered streams. +- Clients must ignore unknown `type`s and unknown fields (forward + compatibility). + +- **Filter semantics:** `scope` and `session` are exact matches. `level` is a + **minimum severity** (`level=INFO` includes `WARNING`/`ERROR`) and only + constrains `log` events — `session` status events always pass a level filter. +- Because the counter is global, a *filtered* poll's `next` cursor may skip + seq values that belonged to filtered-out events; this is by design (they can + never match the same filter later). + +### `GET /api/v1/events?scope=play&session=play-3&level=INFO&after=10400&limit=500` + +Poll mode. All filters optional; `after` is exclusive; `limit` default 500. + +```json +{ + "events": [ { "…": "…" } ], + "next": 10424, + "dropped": false +} +``` + +`dropped: true` signals the cursor fell off the ring buffer's tail (client +should render a gap marker rather than assume continuity). + +### `GET /api/v1/events/stream?scope=play&after=10400` + +Streaming mode: `application/x-ndjson`, chunked. Backfills everything after +the cursor from the buffer, then stays live. Same filters as poll mode. +Reconnect = re-request with your last seen `seq`. + +> **Send `Accept-Encoding: identity` on stream requests.** A gzip layer (the +> app's own middleware stack, or any proxy) buffers compressed output, holding +> lines back indefinitely on a quiet stream. The Python client library does +> this automatically; hand-rolled clients (`curl` defaults to identity) must +> not request compression here. + +`/events/stream?scope=play` **is** the per-service notification stream: +session-status events ride alongside logs on the same connection, so a client +subscribed to a scope learns of completion/errors without polling. + +*(The Streamlit frontend runs one shared background reader per scope on the +stream endpoint — logs and status events feed queues that the rerun/fragment +loops drain, exactly like today's log-queue pattern. Poll mode serves simple +request/response clients and reconnect backfill.)* + +## 9. Schemas & client library + +Shared pydantic models live in the **client library component** (`lib/` at the +repo root) — the reference pattern, where the client lib is also the schema +home. The app installs it into its venv (editable in dev; built wheel for +release builds) and the FastAPI routes, services layer, and the Streamlit +frontend's API client all import the same classes. FastAPI's OpenAPI (at +`/api/openapi.json`, docs at `/api/docs`) is generated from these and is the +contract any future frontend builds against. + +``` +lib/ + pyproject.toml # wheel: mtconnect-explorer-client (version lockstep with the app) + src/mtconnect_explorer_client/ + __init__.py # ApiClient + schema re-exports + client.py # sync httpx ApiClient — flat per-endpoint methods (reference style) + events.py # events_stream() NDJSON iterator + exceptions.py # typed exceptions mapped from error codes + schemas/ + __init__.py + common.py # ErrorDetail, SessionInfo, SessionStatus, PortsInfo + capture.py # CaptureCreate, CaptureConfig + play.py # PlayCreate, PlayDevice, AgentSettings + simulate.py # SimulateCreate, ShdrSettings, EquipmentInfo, RoutineInfo + datasets.py # DatasetInfo, RenameRequest, DeviceInfo + events.py # LogEvent, SessionEvent, PingEvent, EventPage + system.py # VersionInfo, AgentInfo +``` + +## 10. Non-normative implementation notes + +*(Internal architecture — recorded here so the API's behavior guarantees have +a stated basis; not part of the wire contract.)* + +- Routes are thin wrappers over `src/services/` managers; all session mutation + happens under the manager lock (atomic validate-and-reserve). +- Workers run as **threads in v1** behind a host seam (`ThreadHost`), written + subprocess-ready: a worker is a pure function of `(config, event_sink)`. A + later `SubprocessHost` runs the same worker via self-exec + (`MTConnectExplorer --worker `, config on stdin, events as NDJSON on + stdout) — flipping a scope between hosts changes no API behavior. +- The hot data path (SHDR emission → agent → HTTP consumers; collector → + dataset file) lives entirely inside a worker and never crosses the API. +- The Streamlit frontend authenticates with the runtime token; its API base + URL and key are injected config, so pointing it at a remote backend later is + a flag, not a refactor. + +## 11. Decision log + +| # | Item | Status | +|---|---|---| +| 1 | Refuse to start when exposed (`--host` non-local) without `api_key` | ✅ Confirmed (§2) | +| 2 | `/health` unauthenticated | ✅ Confirmed (§2) | +| 3 | Terminal-session retention | ✅ Confirmed — event bus is the primary signal; retention is a REST safety net (§5) | +| 4 | Upload style | ✅ Resolved — multipart `POST /datasets/upload` per the AMT-DTP ISE/ISCP service convention; overwriting existing datasets is not allowed (§6) | +| 5 | Event ring buffer capacity / persistence | ✅ Accepted by default — 10,000, memory-only (§8) | +| 6 | Ping `seq` = checkpoint of last event, not stored | ✅ Confirmed (delegated) (§8) | +| 7 | Heartbeat interval 15 s | ✅ Accepted by default (§8) | +| 8 | One Swagger tag per service | ✅ Confirmed (§1) | +| 9 | Error shape | ✅ Resolved — typed object inside FastAPI-native `detail` (§3) | +| 10 | URL grammar | ✅ Resolved — resource-REST paths kept; deliberate deviation from the reference's RPC style (§1) | +| 11 | Client library | ✅ Resolved — `lib/` component at repo root is the schema home + sync httpx client; wheel `mtconnect-explorer-client`, version lockstep with the app (§9) | +| 12 | Per-equipment simulate creation | ✅ Resolved (2026-07-24) — `POST /simulate/start/{equipment_type}`, one dedicated body model per type (a deliberate RPC-style amendment to #10, chosen so each equipment's parameters are validated by its own schema as simulator types multiply). Axis limits: meters/degrees, `max > min`, defaults = native travel, limits **scale** the routine + 3D scene + published MTConnect Constraints. The generic `POST /simulate/sessions` was **removed** (not deprecated — pre-release). Equipment keys lowercase w/ axis orientation suffixes (`mill_5axis_trunnion_bc`); `tick_ms` renamed `interval_ms`; all routines named "Default Routine" (§7) | diff --git a/installer.spec b/installer.spec index aa33915..18400a4 100644 --- a/installer.spec +++ b/installer.spec @@ -77,7 +77,7 @@ a = Analysis( binaries=binaries, datas=datas, hiddenimports=hiddenimports, - hookspath=['hooks'], + hookspath=['packaging/hooks'], hooksconfig={}, runtime_hooks=[], excludes=[], diff --git a/lib/.gitignore b/lib/.gitignore new file mode 100644 index 0000000..25aacff --- /dev/null +++ b/lib/.gitignore @@ -0,0 +1,3 @@ +build/ +dist/ +*.egg-info/ diff --git a/lib/pyproject.toml b/lib/pyproject.toml new file mode 100644 index 0000000..6c76112 --- /dev/null +++ b/lib/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mtconnect-explorer-client" +description = "Python client and API schemas for the MTConnectExplorer REST API" +readme = "readme.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +dependencies = [ + "pydantic>=2", + "httpx", +] +dynamic = ["version"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.dynamic] +# Read statically from src/mtconnect_explorer_client/__version__.py — the dotted +# submodule path keeps the build from importing the package (and its deps). +version = { attr = "mtconnect_explorer_client.__version__.__version__" } diff --git a/lib/readme.md b/lib/readme.md new file mode 100644 index 0000000..446ac80 --- /dev/null +++ b/lib/readme.md @@ -0,0 +1,32 @@ +# mtconnect-explorer-client + +Python client and pydantic schemas for the [MTConnectExplorer](../readme.md) REST API. +This package is the single source of truth for the API's request/response models — the +server imports its schemas from here. + +## Install + +```bash +pip install /mtconnect_explorer_client-*.whl +``` + +(Development installs use `pip install -e ./lib` from the repo root — wired automatically +by `dev.bat` / `make` via `requirements-src.txt`.) + +## Usage + +```python +from mtconnect_explorer_client import ApiClient, LinearAxisLimits, Mill3AxisCreate + +client = ApiClient("http://127.0.0.1:8501/api/v1", api_key="") + +print(client.version()) +print(client.datasets_list()) + +sid = client.simulate_start(Mill3AxisCreate(x=LinearAxisLimits(min=0.0, max=0.5))).id +for event in client.events_stream(scope="simulate"): + print(event.seq, event.type) +``` + +The full endpoint reference lives at `/api/docs` on a running instance, and in +[docs/api.md](../docs/api.md). diff --git a/lib/src/mtconnect_explorer_client/__init__.py b/lib/src/mtconnect_explorer_client/__init__.py new file mode 100644 index 0000000..c1a59e5 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/__init__.py @@ -0,0 +1,7 @@ +from .__version__ import __version__ +from .client import ApiClient +from .exceptions import ApiError +from .schemas import * # noqa: F401,F403 — the schema package defines __all__ +from . import schemas + +__all__ = ["__version__", "ApiClient", "ApiError", "schemas", *schemas.__all__] diff --git a/lib/src/mtconnect_explorer_client/__version__.py b/lib/src/mtconnect_explorer_client/__version__.py new file mode 100644 index 0000000..6a2b567 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/__version__.py @@ -0,0 +1,3 @@ +# PEP 440 mirror of the app version in src/__version__.py ("0.1.0-alpha.5"). +# Keep the two in lockstep manually when bumping. +__version__ = "0.1.0a5" diff --git a/lib/src/mtconnect_explorer_client/client.py b/lib/src/mtconnect_explorer_client/client.py new file mode 100644 index 0000000..9ff001c --- /dev/null +++ b/lib/src/mtconnect_explorer_client/client.py @@ -0,0 +1,316 @@ +"""Sync httpx client for the MTConnectExplorer API (docs/api.md). + +Flat, reference-style methods — one per endpoint, named ``_``. +Create methods take the endpoint's pydantic model (capture/play also accept +its fields as keyword arguments):: + + from mtconnect_explorer_client import ApiClient, Mill3AxisCreate, LinearAxisLimits + + client = ApiClient("http://127.0.0.1:8501/api/v1", api_key="...") + sid = client.simulate_start(Mill3AxisCreate(x=LinearAxisLimits(max=0.5))).id + for event in client.events_stream(scope="simulate"): + print(event) +""" + +from typing import Any, BinaryIO, Dict, Iterator, List, Optional, Union + +import httpx + +from .events import AnyEvent, iter_events +from .exceptions import ApiError +from .schemas import ( + AgentInfo, + CaptureCreate, + CaptureSessionInfo, + DatasetDevices, + DatasetInfo, + EquipmentInfo, + EventPage, + HealthInfo, + PlayCreate, + PlayProgress, + PlaySeek, + PlaySessionInfo, + PortsInfo, + RenameRequest, + RoutineInfo, + SimulateCreate, + SimulateSessionInfo, + VersionInfo, +) + +DEFAULT_BASE_URL = "http://127.0.0.1:8501/api/v1" + + +class ApiClient: + """Synchronous client for a running MTConnectExplorer instance.""" + + def __init__( + self, + base_url: str = DEFAULT_BASE_URL, + api_key: Optional[str] = None, + timeout: float = 30.0, + transport: Optional[httpx.BaseTransport] = None, + ) -> None: + headers = {"X-API-Key": api_key} if api_key else {} + self._client = httpx.Client( + base_url=base_url.rstrip("/"), + headers=headers, + timeout=timeout, + transport=transport, + ) + + # -- plumbing -------------------------------------------------------------- + def close(self) -> None: + self._client.close() + + def __enter__(self) -> "ApiClient": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + response = self._client.request(method, path, **kwargs) + if response.status_code >= 400: + raise ApiError.from_response(response) + return response + + # -- System ---------------------------------------------------------------- + def health(self) -> HealthInfo: + """GET /health (unauthenticated liveness probe).""" + return HealthInfo.model_validate(self._request("GET", "/health").json()) + + def version(self) -> VersionInfo: + """GET /version.""" + return VersionInfo.model_validate(self._request("GET", "/version").json()) + + def agents_list(self) -> List[AgentInfo]: + """GET /agents — local agents run by live Play/Simulate sessions.""" + return [AgentInfo.model_validate(a) for a in self._request("GET", "/agents").json()] + + def ports(self, shdr_count: int = 1) -> PortsInfo: + """GET /ports — used ports plus free suggestions for launch forms.""" + return PortsInfo.model_validate( + self._request("GET", "/ports", params={"shdr_count": shdr_count}).json() + ) + + # -- Datasets ---------------------------------------------------------------- + def datasets_list(self) -> List[DatasetInfo]: + """GET /datasets.""" + return [DatasetInfo.model_validate(d) for d in self._request("GET", "/datasets").json()] + + def datasets_get(self, name: str) -> DatasetInfo: + """GET /datasets/{name}.""" + return DatasetInfo.model_validate(self._request("GET", f"/datasets/{name}").json()) + + def datasets_upload(self, name: str, file: Union[bytes, BinaryIO]) -> DatasetInfo: + """POST /datasets/upload (multipart; overwriting is not allowed).""" + response = self._request( + "POST", + "/datasets/upload", + files={"file": (name, file, "application/octet-stream")}, + timeout=None, # large uploads + ) + return DatasetInfo.model_validate(response.json()) + + def datasets_download(self, name: str) -> Iterator[bytes]: + """GET /datasets/{name}/content — yields the file in chunks.""" + with self._client.stream( + "GET", f"/datasets/{name}/content", timeout=None + ) as response: + if response.status_code >= 400: + response.read() + raise ApiError.from_response(response) + yield from response.iter_bytes(chunk_size=1024 * 1024) + + def datasets_rename(self, name: str, new_name: str) -> None: + """PATCH /datasets/{name}.""" + self._request( + "PATCH", f"/datasets/{name}", + json=RenameRequest(new_name=new_name).model_dump(), + ) + + def datasets_delete(self, name: str) -> None: + """DELETE /datasets/{name}.""" + self._request("DELETE", f"/datasets/{name}") + + def datasets_devices(self, name: str) -> DatasetDevices: + """GET /datasets/{name}/devices — devices from the probe frame.""" + return DatasetDevices.model_validate( + self._request("GET", f"/datasets/{name}/devices").json() + ) + + # -- Capture ----------------------------------------------------------------- + def capture_start( + self, create: Optional[CaptureCreate] = None, **fields: Any + ) -> CaptureSessionInfo: + """POST /capture/sessions.""" + body = create if create is not None else CaptureCreate(**fields) + return CaptureSessionInfo.model_validate( + self._request("POST", "/capture/sessions", json=body.model_dump()).json() + ) + + def capture_list(self) -> List[CaptureSessionInfo]: + """GET /capture/sessions.""" + return [ + CaptureSessionInfo.model_validate(s) + for s in self._request("GET", "/capture/sessions").json() + ] + + def capture_get(self, session_id: str) -> CaptureSessionInfo: + """GET /capture/sessions/{id}.""" + return CaptureSessionInfo.model_validate( + self._request("GET", f"/capture/sessions/{session_id}").json() + ) + + def capture_stop(self, session_id: str) -> None: + """DELETE /capture/sessions/{id} — stop and remove.""" + self._request("DELETE", f"/capture/sessions/{session_id}") + + def capture_stop_all(self) -> None: + """DELETE /capture/sessions — stop-all.""" + self._request("DELETE", "/capture/sessions") + + # -- Play ---------------------------------------------------------------------- + def play_start( + self, create: Optional[PlayCreate] = None, **fields: Any + ) -> PlaySessionInfo: + """POST /play/sessions.""" + body = create if create is not None else PlayCreate(**fields) + return PlaySessionInfo.model_validate( + self._request("POST", "/play/sessions", json=body.model_dump()).json() + ) + + def play_list(self) -> List[PlaySessionInfo]: + """GET /play/sessions.""" + return [ + PlaySessionInfo.model_validate(s) + for s in self._request("GET", "/play/sessions").json() + ] + + def play_get(self, session_id: str) -> PlaySessionInfo: + """GET /play/sessions/{id}.""" + return PlaySessionInfo.model_validate( + self._request("GET", f"/play/sessions/{session_id}").json() + ) + + def play_progress(self, session_id: str) -> PlayProgress: + """GET /play/sessions/{id}/progress — the scrub bar's playhead origin.""" + return PlayProgress.model_validate( + self._request("GET", f"/play/sessions/{session_id}/progress").json() + ) + + def play_seek(self, session_id: str, position: float) -> None: + """POST /play/sessions/{id}/seek — seek a running playback (re-timestamp mode).""" + self._request( + "POST", f"/play/sessions/{session_id}/seek", + json=PlaySeek(position=position).model_dump(), + ) + + def play_pause(self, session_id: str) -> None: + """POST /play/sessions/{id}/pause — pause a running playback (any timestamp mode).""" + self._request("POST", f"/play/sessions/{session_id}/pause") + + def play_resume(self, session_id: str) -> None: + """POST /play/sessions/{id}/resume — resume a paused playback.""" + self._request("POST", f"/play/sessions/{session_id}/resume") + + def play_stop(self, session_id: str) -> None: + """DELETE /play/sessions/{id} — stop and remove.""" + self._request("DELETE", f"/play/sessions/{session_id}") + + def play_stop_all(self) -> None: + """DELETE /play/sessions — stop-all.""" + self._request("DELETE", "/play/sessions") + + # -- Simulate --------------------------------------------------------------------- + def simulate_equipment(self) -> List[EquipmentInfo]: + """GET /simulate/equipment — catalog for the launch form.""" + return [ + EquipmentInfo.model_validate(e) + for e in self._request("GET", "/simulate/equipment").json() + ] + + def simulate_start(self, create: SimulateCreate) -> SimulateSessionInfo: + """POST /simulate/start/{equipment_type} — dispatched from the create + model's type (Mill3AxisCreate, Mill5AxisTrunnionBcCreate, Robot6DofCreate).""" + return SimulateSessionInfo.model_validate( + self._request( + "POST", + f"/simulate/start/{create.equipment_type}", + json=create.model_dump(), + ).json() + ) + + def simulate_list(self) -> List[SimulateSessionInfo]: + """GET /simulate/sessions.""" + return [ + SimulateSessionInfo.model_validate(s) + for s in self._request("GET", "/simulate/sessions").json() + ] + + def simulate_get(self, session_id: str) -> SimulateSessionInfo: + """GET /simulate/sessions/{id}.""" + return SimulateSessionInfo.model_validate( + self._request("GET", f"/simulate/sessions/{session_id}").json() + ) + + def simulate_routine(self, session_id: str) -> RoutineInfo: + """GET /simulate/sessions/{id}/routine — feeds the phase-locked 3D view.""" + return RoutineInfo.model_validate( + self._request("GET", f"/simulate/sessions/{session_id}/routine").json() + ) + + def simulate_stop(self, session_id: str) -> None: + """DELETE /simulate/sessions/{id} — stop and remove.""" + self._request("DELETE", f"/simulate/sessions/{session_id}") + + def simulate_stop_all(self) -> None: + """DELETE /simulate/sessions — stop-all.""" + self._request("DELETE", "/simulate/sessions") + + # -- Events ----------------------------------------------------------------------- + def events( + self, + after: Optional[int] = None, + scope: Optional[str] = None, + session: Optional[str] = None, + level: Optional[str] = None, + limit: int = 500, + ) -> EventPage: + """GET /events — poll mode with an exclusive ``after`` cursor.""" + params: Dict[str, Any] = {"limit": limit} + if after is not None: + params["after"] = after + if scope is not None: + params["scope"] = scope + if session is not None: + params["session"] = session + if level is not None: + params["level"] = level + return EventPage.model_validate( + self._request("GET", "/events", params=params).json() + ) + + def events_stream( + self, + after: Optional[int] = None, + scope: Optional[str] = None, + session: Optional[str] = None, + level: Optional[str] = None, + reconnect: bool = True, + ) -> Iterator[AnyEvent]: + """GET /events/stream — NDJSON iterator with seq-based reconnect. + + Yields every event including pings (their ``seq`` is a cursor + checkpoint). Iterate forever, or pass ``reconnect=False`` to stop when + the connection drops.""" + return iter_events( + self._client, + after=after, + scope=scope, + session=session, + level=level, + reconnect=reconnect, + ) diff --git a/lib/src/mtconnect_explorer_client/events.py b/lib/src/mtconnect_explorer_client/events.py new file mode 100644 index 0000000..d6d2319 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/events.py @@ -0,0 +1,89 @@ +"""NDJSON event-stream iterator (docs/api.md §8). + +``iter_events`` consumes ``GET /events/stream``: it backfills after the cursor, +stays live, and transparently reconnects from the last seen ``seq`` (pings +count — their seq is a cursor checkpoint). Unknown event types and malformed +lines are skipped (forward compatibility). +""" + +import json +from typing import Any, Dict, Iterator, Optional, Union + +import httpx +from pydantic import TypeAdapter + +from .schemas import Event, LogEvent, PingEvent, SessionEvent + +AnyEvent = Union[LogEvent, SessionEvent, PingEvent] + +_EVENT_ADAPTER: TypeAdapter = TypeAdapter(Event) + +# The server pings every 15 s of idle time; the read timeout must sit above +# that so a quiet-but-alive stream is not treated as dead. +STREAM_READ_TIMEOUT_S = 45.0 + + +def parse_event(line: str) -> Optional[AnyEvent]: + """Parse one NDJSON line into an event model, or None if unknown/invalid.""" + line = line.strip() + if not line: + return None + try: + return _EVENT_ADAPTER.validate_json(line) + except Exception: + return None + + +def iter_events( + client: httpx.Client, + *, + after: Optional[int] = None, + scope: Optional[str] = None, + session: Optional[str] = None, + level: Optional[str] = None, + reconnect: bool = True, +) -> Iterator[AnyEvent]: + """Yield events (including pings) from /events/stream, forever. + + On a dropped connection, reconnects with ``after=`` so no + buffered event is missed. Stop by breaking out of the loop / closing the + client.""" + cursor = after + while True: + params: Dict[str, Any] = {} + if cursor is not None: + params["after"] = cursor + if scope is not None: + params["scope"] = scope + if session is not None: + params["session"] = session + if level is not None: + params["level"] = level + try: + with client.stream( + "GET", + "/events/stream", + params=params, + # identity: a gzip middleware/proxy would buffer the compressed + # stream, holding lines back indefinitely on a quiet feed. + headers={"Accept-Encoding": "identity"}, + timeout=httpx.Timeout(10.0, read=STREAM_READ_TIMEOUT_S), + ) as response: + if response.status_code >= 400: + response.read() + from .exceptions import ApiError + + raise ApiError.from_response(response) + for line in response.iter_lines(): + event = parse_event(line) + if event is None: + continue + cursor = event.seq + yield event + except (httpx.TransportError, httpx.TimeoutException): + if not reconnect: + return + continue + # Server closed the stream cleanly; reconnect from the cursor. + if not reconnect: + return diff --git a/lib/src/mtconnect_explorer_client/exceptions.py b/lib/src/mtconnect_explorer_client/exceptions.py new file mode 100644 index 0000000..02412e0 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/exceptions.py @@ -0,0 +1,52 @@ +"""Typed API errors raised by the MTConnectExplorer client (docs/api.md §3).""" + +from typing import Any, Dict, Optional + +import httpx + + +class ApiError(Exception): + """A non-2xx API response, carrying the typed error detail. + + ``code``/``message``/``context`` mirror the server's + ``{"detail": {"code", "message", "context"}}`` object. FastAPI-native 422 + validation errors (where ``detail`` is a list of field errors) are mapped + to code ``validation_error`` with the list in ``context["errors"]``. + """ + + def __init__( + self, + status_code: int, + code: str, + message: str, + context: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__(f"[{status_code} {code}] {message}") + self.status_code = status_code + self.code = code + self.message = message + self.context = context + + @classmethod + def from_response(cls, response: httpx.Response) -> "ApiError": + status = response.status_code + try: + detail = response.json().get("detail") + except Exception: + detail = None + if isinstance(detail, dict) and "code" in detail: + return cls( + status, + str(detail.get("code")), + str(detail.get("message", "")), + detail.get("context"), + ) + if isinstance(detail, list): # FastAPI-native 422 validation format + first = detail[0] if detail else {} + return cls( + status, + "validation_error", + str(first.get("msg", "request validation failed")), + {"errors": detail}, + ) + return cls(status, "http_error", response.text[:500] or f"HTTP {status}") diff --git a/lib/src/mtconnect_explorer_client/schemas/__init__.py b/lib/src/mtconnect_explorer_client/schemas/__init__.py new file mode 100644 index 0000000..9a14f83 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/__init__.py @@ -0,0 +1,85 @@ +"""Pydantic request/response models for the MTConnectExplorer API (docs/api.md §9). + +This package is the single source of truth for the wire contract: the server's +services layer, the FastAPI routes, and the client all import these classes. +""" + +from .common import ( + TERMINAL_STATUSES, + ErrorDetail, + PortsInfo, + PortSuggestions, + Scope, + SessionInfo, + SessionPorts, + SessionStatus, +) +from .capture import CaptureCreate, CaptureSessionInfo +from .play import ( + AgentSettings, + PlayCreate, + PlayDevice, + PlayProgress, + PlaySeek, + PlaySessionInfo, +) +from .simulate import ( + CREATE_MODELS, + EquipmentInfo, + LinearAxisLimits, + Mill3AxisCreate, + Mill5AxisTrunnionBcCreate, + Robot6DofCreate, + RotaryAxisLimits, + RoutineInfo, + ShdrSettings, + SimulateCreate, + SimulateSessionBase, + SimulateSessionInfo, +) +from .datasets import DatasetDevices, DatasetInfo, DeviceInfo, RenameRequest +from .events import Event, EventPage, LogEvent, PingEvent, SessionEvent +from .system import AgentInfo, HealthInfo, VersionInfo + +__all__ = [ + "TERMINAL_STATUSES", + "ErrorDetail", + "PortsInfo", + "PortSuggestions", + "Scope", + "SessionInfo", + "SessionPorts", + "SessionStatus", + "CaptureCreate", + "CaptureSessionInfo", + "AgentSettings", + "PlayCreate", + "PlayDevice", + "PlayProgress", + "PlaySeek", + "PlaySessionInfo", + "CREATE_MODELS", + "EquipmentInfo", + "LinearAxisLimits", + "Mill3AxisCreate", + "Mill5AxisTrunnionBcCreate", + "Robot6DofCreate", + "RotaryAxisLimits", + "RoutineInfo", + "ShdrSettings", + "SimulateCreate", + "SimulateSessionBase", + "SimulateSessionInfo", + "DatasetDevices", + "DatasetInfo", + "DeviceInfo", + "RenameRequest", + "Event", + "EventPage", + "LogEvent", + "PingEvent", + "SessionEvent", + "AgentInfo", + "HealthInfo", + "VersionInfo", +] diff --git a/lib/src/mtconnect_explorer_client/schemas/capture.py b/lib/src/mtconnect_explorer_client/schemas/capture.py new file mode 100644 index 0000000..42a2f72 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/capture.py @@ -0,0 +1,39 @@ +"""Capture-scope models (docs/api.md §7).""" + +from typing import Literal, Optional + +from pydantic import BaseModel, Field + +from .common import SessionInfo + + +class CaptureCreate(BaseModel): + """Body of POST /capture/sessions. Defaults mirror the app's client classes.""" + + base_url: str = Field( + min_length=1, + description="Base URL of the MTConnect agent to capture from.", + ) + mode: Literal["stream", "sample", "current"] = Field( + "stream", description="Data collection mode." + ) + interval_ms: int = Field( + 1000, ge=1, description="Milliseconds between agent stream buffer checks." + ) + timeout_s: float = Field( + 30.0, gt=0, description="Socket timeout in seconds for non-streaming requests." + ) + chunk_size: int = Field( + 4096, ge=1, description="Read chunk size in bytes for streaming responses." + ) + filename: Optional[str] = Field( + None, + description="Dataset filename to write. null = server generates one " + "(sanitized base_url + UTC timestamp, collision-checked).", + ) + + +class CaptureSessionInfo(SessionInfo): + """Capture session envelope with a typed config echo.""" + + config: CaptureCreate diff --git a/lib/src/mtconnect_explorer_client/schemas/common.py b/lib/src/mtconnect_explorer_client/schemas/common.py new file mode 100644 index 0000000..00e6833 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/common.py @@ -0,0 +1,72 @@ +"""Shared session/error/port models (docs/api.md §3–§5).""" + +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, Field + +# Session lifecycle: starting -> running -> (stopping ->) finished | error. +SessionStatus = Literal["starting", "running", "stopping", "finished", "error"] + +Scope = Literal["capture", "play", "simulate"] + +TERMINAL_STATUSES = ("finished", "error") + + +class ErrorDetail(BaseModel): + """Typed error object carried inside FastAPI's native ``detail`` field.""" + + code: str = Field(description="Machine-readable error code, e.g. 'port_conflict'.") + message: str = Field(description="Human-readable description of the error.") + context: Optional[Dict[str, Any]] = Field( + None, description="Optional structured context, e.g. the full port-conflict list." + ) + + +class SessionPorts(BaseModel): + """Ports held by a session (free once the session reaches a terminal state).""" + + agent: Optional[int] = Field( + None, description="MTConnect agent HTTP port, or null (capture runs no agent)." + ) + shdr: List[int] = Field( + default_factory=list, description="SHDR adapter TCP ports, one per adapter." + ) + + +class SessionInfo(BaseModel): + """Common session envelope shared by all three scopes (docs/api.md §5).""" + + id: str = Field(description="Opaque session id, prefixed by scope: cap-1, play-2, sim-3.") + scope: Scope = Field(description="Owning service: capture, play, or simulate.") + label: str = Field(description="Human-readable label used to tag log lines.") + status: SessionStatus = Field(description="Current lifecycle status.") + created_at: datetime = Field(description="When the session was created (UTC).") + started_at: Optional[datetime] = Field( + None, description="When the session reached 'running', or null." + ) + finished_at: Optional[datetime] = Field( + None, description="When the session reached a terminal status, or null." + ) + error: Optional[str] = Field( + None, description="Error message when status is 'error', else null." + ) + ports: SessionPorts = Field( + default_factory=SessionPorts, description="Ports held by this session." + ) + config: Dict[str, Any] = Field( + default_factory=dict, + description="Echo of the scope-specific create config (typed in scope subclasses).", + ) + + +class PortSuggestions(BaseModel): + agent: int = Field(description="Suggested free agent HTTP port.") + shdr: List[int] = Field(description="Suggested free SHDR ports (one per requested count).") + + +class PortsInfo(BaseModel): + """Used ports across all live sessions plus free suggestions (docs/api.md §4).""" + + used: List[int] = Field(description="Ports held by every non-terminal session.") + suggested: PortSuggestions = Field(description="Free ports for pre-filling launch forms.") diff --git a/lib/src/mtconnect_explorer_client/schemas/datasets.py b/lib/src/mtconnect_explorer_client/schemas/datasets.py new file mode 100644 index 0000000..5557e7c --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/datasets.py @@ -0,0 +1,38 @@ +"""Dataset models (docs/api.md §6).""" + +from datetime import datetime +from typing import List, Literal, Optional + +from pydantic import BaseModel, Field + + +class DatasetInfo(BaseModel): + """One dataset file in the datasets directory.""" + + name: str = Field(description="Dataset filename (always ends in .jsonl).") + size_bytes: int = Field(description="File size in bytes.") + modified: datetime = Field(description="Last-modified time (UTC).") + in_use: List[Literal["play", "capture"]] = Field( + default_factory=list, + description="Scopes currently using this dataset; empty when idle.", + ) + + +class RenameRequest(BaseModel): + """Body of PATCH /datasets/{name}.""" + + new_name: str = Field(min_length=1, description="New dataset filename (.jsonl).") + + +class DeviceInfo(BaseModel): + """One device from a dataset's probe frame.""" + + id: Optional[str] = Field(None, description="Device element id, if present.") + name: str = Field(description="Device name.") + uuid: Optional[str] = Field(None, description="Device uuid, if present.") + + +class DatasetDevices(BaseModel): + """GET /datasets/{name}/devices — all devices, including the Agent device.""" + + devices: List[DeviceInfo] = Field(description="Devices from the probe frame.") diff --git a/lib/src/mtconnect_explorer_client/schemas/events.py b/lib/src/mtconnect_explorer_client/schemas/events.py new file mode 100644 index 0000000..459eb55 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/events.py @@ -0,0 +1,63 @@ +"""Event-bus models (docs/api.md §8). + +Every line/event carries a globally monotonic ``seq`` and an RFC 3339 ``ts``. +Pings are cursor checkpoints: they carry the last delivered event's seq and are +never stored in the ring buffer. Clients must ignore unknown event types and +unknown fields (forward compatibility). +""" + +from datetime import datetime +from typing import Annotated, Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field + +from .common import Scope, SessionStatus + + +class LogEvent(BaseModel): + """A backend log record, attributed to a session when one emitted it.""" + + type: Literal["log"] = "log" + seq: int = Field(description="Globally monotonic sequence number.") + ts: datetime = Field(description="Event timestamp (UTC).") + scope: Optional[Scope] = Field(None, description="Owning scope, if session-attributed.") + session: Optional[str] = Field(None, description="Session id, if session-attributed.") + level: str = Field(description="Log level name, e.g. INFO.") + logger: str = Field(description="Logger name, e.g. src.converter.") + msg: str = Field(description="Log message.") + + +class SessionEvent(BaseModel): + """A session lifecycle transition (posted on every status change).""" + + type: Literal["session"] = "session" + seq: int = Field(description="Globally monotonic sequence number.") + ts: datetime = Field(description="Event timestamp (UTC).") + scope: Scope = Field(description="Owning scope.") + session: str = Field(description="Session id.") + event: str = Field("status", description="Event kind (currently always 'status').") + status: SessionStatus = Field(description="New session status.") + detail: Optional[Dict[str, Any]] = Field( + None, description="Extra transition data, e.g. {'start_epoch': ...} on simulate." + ) + + +class PingEvent(BaseModel): + """Stream heartbeat; ``seq`` is the last delivered event's seq (a checkpoint).""" + + type: Literal["ping"] = "ping" + seq: int = Field(description="Last delivered event's seq — 'current through N'.") + ts: datetime = Field(description="Ping timestamp (UTC).") + + +Event = Annotated[Union[LogEvent, SessionEvent, PingEvent], Field(discriminator="type")] + + +class EventPage(BaseModel): + """GET /events poll response.""" + + events: List[Event] = Field(description="Events after the cursor, oldest first.") + next: int = Field(description="Cursor for the next poll (pass as ?after=).") + dropped: bool = Field( + description="True if the cursor fell off the ring buffer's tail (gap)." + ) diff --git a/lib/src/mtconnect_explorer_client/schemas/play.py b/lib/src/mtconnect_explorer_client/schemas/play.py new file mode 100644 index 0000000..4b3c831 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/play.py @@ -0,0 +1,105 @@ +"""Play-scope models (docs/api.md §7).""" + +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, Field + +from .common import SessionInfo + + +class AgentSettings(BaseModel): + """Bundled MTConnect agent settings. Defaults mirror the app's AgentConfig.""" + + port: int = Field(5000, ge=1, le=65535, description="Agent HTTP port.") + mtconnect_version: str = Field("2.7", description="MTConnect version string.") + asset_buffer_size: int = Field(1024, ge=1, description="Agent asset buffer size.") + buffer_size: int = Field(131072, ge=1, description="Agent circular buffer size.") + + +class PlayDevice(BaseModel): + """One device selected for playback and the SHDR adapter that will feed it.""" + + device_name: str = Field(min_length=1, description="Device name from the dataset.") + shdr_host: str = Field("127.0.0.1", description="SHDR adapter host address.") + shdr_port: int = Field(7878, ge=1, le=65535, description="SHDR adapter TCP port.") + + +class PlayCreate(BaseModel): + """Body of POST /play/sessions.""" + + dataset: str = Field(min_length=1, description="Dataset filename to replay.") + devices: List[PlayDevice] = Field( + min_length=1, + description="Devices to replay; one SHDR adapter is spun up per entry.", + ) + agent: AgentSettings = Field( + default_factory=AgentSettings, description="Bundled agent settings." + ) + re_timestamp: bool = Field( + True, + description="Re-stamp observations with the current time on replay so the " + "data appears live; when false, original recorded timestamps are kept.", + ) + loop: bool = Field( + True, description="Repeat playback from the start once the dataset is exhausted." + ) + snapshot_interval_s: float = Field( + 15.0, + gt=0, + description="Seek keyframe cadence in dataset seconds. A full-state snapshot " + "is indexed every this-many seconds at load, so scrubbing replays only from " + "the nearest snapshot (≤ this many seconds of deltas) instead of from the " + "start. Smaller = faster seeks but more memory and a longer pre-scan.", + ) + + +class PlaySessionInfo(SessionInfo): + """Play session envelope with a typed config echo.""" + + config: PlayCreate + + +class PlayProgress(BaseModel): + """GET /play/sessions/{id}/progress — the scrub bar's origin. + + The live playhead is computed client-side as ``now - pass_start_epoch`` + (clamped to ``duration``; wrapped by ``duration`` when ``looping``), so only + this origin is transmitted — the worker republishes it on start, loop-wrap, + and seek, not per frame. ``server_time`` lets the client correct for + browser<->backend clock skew (same pattern as the 3D view's RoutineInfo).""" + + duration: float = Field( + description="Total dataset span in seconds (the scrub range)." + ) + pass_start_epoch: Optional[float] = Field( + None, + description="Unix time of the current pass's offset 0; null until running.", + ) + looping: bool = Field( + description="Whether playback loops (the UI wraps the playhead at duration)." + ) + paused: bool = Field( + False, + description="Whether playback is paused. While paused, pass_start_epoch stops " + "advancing and the UI holds the playhead at `position` instead of computing " + "`server_time - pass_start_epoch`.", + ) + position: Optional[float] = Field( + None, + description="Current playhead in seconds; authoritative while paused (the " + "frozen value). Null until running.", + ) + server_time: datetime = Field( + description="Server clock at response time, for client clock-offset correction." + ) + + +class PlaySeek(BaseModel): + """Body of POST /play/sessions/{id}/seek.""" + + position: float = Field( + ge=0.0, + description="Target playhead in seconds from the start of the dataset " + "(clamped to the dataset duration).", + ) diff --git a/lib/src/mtconnect_explorer_client/schemas/simulate.py b/lib/src/mtconnect_explorer_client/schemas/simulate.py new file mode 100644 index 0000000..de80b68 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/simulate.py @@ -0,0 +1,229 @@ +"""Simulate-scope models (docs/api.md §7). + +Each equipment type has its own create model and its own endpoint +(``POST /simulate/start/{equipment_type}``), so every parameter set is fully +validated by its dedicated schema. Axis limits are **meters** for linear axes +and **degrees** for rotary axes; each limit defaults to the equipment's native +travel, so an empty body ``{}`` reproduces the canned routine identically. +Configured limits scale the routine (and the 3D scene, and the published +MTConnect Constraints) into the given range. +""" + +from datetime import datetime +from typing import Annotated, Any, Dict, List, Literal, Optional, Type, Union + +from pydantic import BaseModel, Field, model_validator + +from .common import SessionInfo +from .play import AgentSettings + + +class ShdrSettings(BaseModel): + """SHDR adapter bind settings.""" + + host: str = Field("127.0.0.1", description="SHDR adapter host address.") + port: int = Field(7878, ge=1, le=65535, description="SHDR adapter TCP port.") + + +class LinearAxisLimits(BaseModel): + """Travel limits of a linear axis, in meters.""" + + min: float = Field(0.0, description="Lower travel limit in meters.") + max: float = Field(1.0, description="Upper travel limit in meters.") + + @model_validator(mode="after") + def _ordered(self) -> "LinearAxisLimits": + if not self.max > self.min: + raise ValueError(f"max ({self.max}) must be greater than min ({self.min})") + return self + + +class RotaryAxisLimits(BaseModel): + """Rotation limits of a rotary axis or joint, in degrees.""" + + min: float = Field(-180.0, description="Lower rotation limit in degrees.") + max: float = Field(180.0, description="Upper rotation limit in degrees.") + + @model_validator(mode="after") + def _ordered(self) -> "RotaryAxisLimits": + if not self.max > self.min: + raise ValueError(f"max ({self.max}) must be greater than min ({self.min})") + return self + + +class SimulateSessionBase(BaseModel): + """Fields common to every simulate create body.""" + + interval_ms: int = Field( + 10, ge=1, le=60000, + description="Interval in milliseconds at which generated observations " + "are fed to the SHDR adapter. Lower = smoother motion, higher data rate.", + ) + agent: AgentSettings = Field( + default_factory=AgentSettings, description="Bundled MTConnect agent settings." + ) + shdr: ShdrSettings = Field( + default_factory=ShdrSettings, description="SHDR adapter bind settings." + ) + + +class Mill3AxisCreate(SimulateSessionBase): + """Body of POST /simulate/start/mill_3axis — a 3-axis vertical mill. + + Axis limit defaults are the machine's native travel (0–1 m per axis).""" + + equipment_type: Literal["mill_3axis"] = Field( + "mill_3axis", + description="Equipment type (constant; determined by the endpoint).", + ) + device_name: str = Field( + "Mill", min_length=1, description="MTConnect device name." + ) + routine: Literal["Default Routine"] = Field( + "Default Routine", + description="Routine to run: a canned facing raster + circular boring cycle.", + ) + x: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="X-axis travel limits in meters.", + ) + y: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Y-axis travel limits in meters.", + ) + z: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Z-axis travel limits in meters.", + ) + + +class Mill5AxisTrunnionBcCreate(SimulateSessionBase): + """Body of POST /simulate/start/mill_5axis_trunnion_bc — a 5-axis trunnion + mill: the head carries X/Y/Z; the table tilts about Y (B) and rotates about + Z (C). C is continuous and therefore has no limits. + + Axis limit defaults are the machine's native travel (0–1 m linear, ±120° B).""" + + equipment_type: Literal["mill_5axis_trunnion_bc"] = Field( + "mill_5axis_trunnion_bc", + description="Equipment type (constant; determined by the endpoint).", + ) + device_name: str = Field( + "TrunnionMill", min_length=1, description="MTConnect device name." + ) + routine: Literal["Default Routine"] = Field( + "Default Routine", + description="Routine to run: canned 3+2 indexed facing passes followed " + "by a simultaneous 5-axis helical contour.", + ) + x: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="X-axis travel limits in meters.", + ) + y: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Y-axis travel limits in meters.", + ) + z: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Z-axis travel limits in meters.", + ) + b: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-120.0, max=120.0), + description="Trunnion tilt about Y (B axis) limits in degrees.", + ) + + +class Robot6DofCreate(SimulateSessionBase): + """Body of POST /simulate/start/robot_6dof — a generic 6-DOF articulated + robot running a pick-and-place cycle in joint space. + + Joint limit defaults are the arm's native travel; link lengths are fixed + physical structure.""" + + equipment_type: Literal["robot_6dof"] = Field( + "robot_6dof", + description="Equipment type (constant; determined by the endpoint).", + ) + device_name: str = Field( + "Robot", min_length=1, description="MTConnect device name." + ) + routine: Literal["Default Routine"] = Field( + "Default Routine", + description="Routine to run: a canned pick-and-place cycle.", + ) + j1: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-170.0, max=170.0), + description="J1 (base yaw, about Z) limits in degrees.", + ) + j2: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-100.0, max=135.0), + description="J2 (shoulder pitch, about Y) limits in degrees.", + ) + j3: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-80.0, max=155.0), + description="J3 (elbow pitch, about Y) limits in degrees.", + ) + j4: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-180.0, max=180.0), + description="J4 (forearm roll, about Z) limits in degrees.", + ) + j5: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-120.0, max=120.0), + description="J5 (wrist pitch, about Y) limits in degrees.", + ) + j6: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-180.0, max=180.0), + description="J6 (wrist roll, about Z) limits in degrees.", + ) + + +# The discriminated union of every create body — the session envelope's typed +# config echo, and the type the client's simulate_start() dispatches on. +SimulateCreate = Annotated[ + Union[Mill3AxisCreate, Mill5AxisTrunnionBcCreate, Robot6DofCreate], + Field(discriminator="equipment_type"), +] + +# equipment_type key -> create model (form scaffolding, client dispatch). +CREATE_MODELS: Dict[str, Type[SimulateSessionBase]] = { + "mill_3axis": Mill3AxisCreate, + "mill_5axis_trunnion_bc": Mill5AxisTrunnionBcCreate, + "robot_6dof": Robot6DofCreate, +} + + +class EquipmentInfo(BaseModel): + """One entry of the GET /simulate/equipment catalog.""" + + type: str = Field(description="Equipment type key, e.g. mill_3axis.") + label: str = Field(description="Human-readable equipment label.") + default_device_name: str = Field(description="Default MTConnect device name.") + routines: List[str] = Field(description="Available routine labels.") + create_path: str = Field( + description="Endpoint that starts a session of this type, e.g. " + "/api/v1/simulate/start/mill_3axis; its request schema is that " + "operation's body in the OpenAPI document." + ) + + +class RoutineInfo(BaseModel): + """GET /simulate/sessions/{id}/routine — feeds the phase-locked 3D view.""" + + spec: Dict[str, Any] = Field( + description="RoutineSpec.to_dict(): equipment, coord_names, start, cycle_time, " + "geometry, moves — scaled into the session's configured axis limits." + ) + start_epoch: Optional[float] = Field( + None, + description="Unix time emission began; null until the session reaches 'running'.", + ) + server_time: datetime = Field( + description="Server clock at response time, for client clock-offset correction." + ) + + +class SimulateSessionInfo(SessionInfo): + """Simulate session envelope with a typed config echo.""" + + config: SimulateCreate diff --git a/lib/src/mtconnect_explorer_client/schemas/system.py b/lib/src/mtconnect_explorer_client/schemas/system.py new file mode 100644 index 0000000..bc0b270 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/system.py @@ -0,0 +1,25 @@ +"""System models (docs/api.md §4).""" + +from typing import List + +from pydantic import BaseModel, Field + + +class HealthInfo(BaseModel): + """GET /health — the only unauthenticated endpoint.""" + + status: str = Field("ok", description="Liveness indicator.") + + +class VersionInfo(BaseModel): + """GET /version.""" + + version: str = Field(description="Application version, e.g. 0.1.0-alpha.5.") + api: str = Field("v1", description="API version.") + + +class AgentInfo(BaseModel): + """One locally-running MTConnect agent (run by a live Play/Simulate session).""" + + base_url: str = Field(description="Agent base URL, e.g. http://localhost:5000/.") + sessions: List[str] = Field(description="Session ids running this agent.") diff --git a/makefile b/makefile index 9b7f9db..057880d 100644 --- a/makefile +++ b/makefile @@ -2,6 +2,11 @@ all: bin bin: build +# Build the client wheel and install it over the editable install so the +# frozen binary uses (and proves) the real wheel + rm -rf lib/dist + venv/bin/python -m build --wheel lib + venv/bin/pip install --force-reinstall lib/dist/*.whl venv/bin/pyinstaller installer.spec --clean build: src/bin/agent venv/touchfile @@ -23,8 +28,8 @@ venv/touchfile: requirements-src.txt venv/bin/pip install -Ur requirements-src.txt # Signal that venv has been updated due to pyproject.toml change touch venv/touchfile -# Update requirements.txt - venv/bin/pip freeze > requirements.txt +# Update requirements.txt (exclude the editable lib install - machine-specific path) + venv/bin/pip freeze --exclude mtconnect-explorer-client > requirements.txt src/bin/agent: # Create temporary build directory diff --git a/packaging/generate_notices.py b/packaging/generate_notices.py new file mode 100644 index 0000000..c0b5aa7 --- /dev/null +++ b/packaging/generate_notices.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Generate a THIRD_PARTY_NOTICES.txt for the release bundle. + +Run with the *build* virtualenv's interpreter after dependencies are installed +(the release workflow does this), e.g.:: + + venv/bin/python generate_notices.py THIRD_PARTY_NOTICES.txt # Linux + venv\\Scripts\\python.exe generate_notices.py THIRD_PARTY_NOTICES.txt # Windows + +It reads every installed distribution's metadata + bundled license text via +``importlib.metadata`` (no extra dependency), and appends manual entries for the +non-Python components shipped in the release (the MTConnect C++ agent binary and +the vendored three.js). It is intentionally over-inclusive — it lists every +installed distribution — so nothing that PyInstaller may bundle is missed. + +Attribution obligations covered: +- Apache-2.0 / MIT / BSD: preserve each component's copyright + license text. +- certifi (MPL-2.0): weak, file-level copyleft — MPL §3.2 only requires telling + recipients where the source is, so we include the license text plus a source + URL (no need to ship certifi's source). +""" + +import importlib.metadata as im +import sys +from pathlib import Path + +_LICENSE_FILENAMES = ( + "LICENSE", "LICENSE.txt", "LICENSE.md", "LICENSE.rst", + "LICENCE", "LICENCE.txt", "COPYING", "COPYING.txt", "NOTICE", "NOTICE.txt", +) +_SEP = "=" * 80 + +# Non-Python components bundled into the release artifact. +_BUNDLED = [ + ( + "MTConnect C++ Agent (bundled 'agent' / 'agent.exe' binary)", "", + "Apache-2.0", + "Source: https://github.com/mtconnect/cppagent (Apache License 2.0).", + ), + ( + "three.js (src/static/three.min.js, src/static/OrbitControls.js)", "r137", + "MIT", + "Copyright 2010-2022 three.js Authors. " + "Source: https://github.com/mrdoob/three.js (MIT License).", + ), +] + + +def _license_id(meta) -> str: + expr = meta.get("License-Expression") # PEP 639 SPDX expression (preferred) + if expr: + return expr.strip() + classifiers = [ + c.split("::")[-1].strip() + for c in (meta.get_all("Classifier") or []) + if c.startswith("License ") + ] + if classifiers: + return "; ".join(classifiers) + lic = (meta.get("License") or "").strip() + return lic.splitlines()[0][:60] if lic else "(see text below / upstream)" + + +def _license_text(dist) -> str: + # Wheels usually ship the license under the dist-info (sometimes licenses/). + candidates = list(_LICENSE_FILENAMES) + [f"licenses/{n}" for n in _LICENSE_FILENAMES] + for name in candidates: + try: + text = dist.read_text(name) + except Exception: + text = None + if text and text.strip(): + return text.strip() + # Fall back: scan declared files for a license-named file. + for f in (dist.files or []): + if f.name.lower().startswith(("license", "licence", "copying")): + try: + text = f.read_text(encoding="utf-8") + except Exception: + try: + text = Path(f.locate()).read_text(encoding="utf-8", errors="replace") + except Exception: + text = None + if text and text.strip(): + return text.strip() + # Last resort: some packages inline the whole license in the License field. + lic = (dist.metadata.get("License") or "").strip() + return lic if len(lic) > 80 else "" + + +def _notes_for(name: str, version: str) -> str: + if name.lower() == "certifi": + return ( + "certifi is licensed under MPL-2.0 (weak, file-level copyleft). Its " + "source is available unmodified at " + f"https://github.com/certifi/python-certifi (release {version})." + ) + return "" + + +def main() -> None: + out = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("THIRD_PARTY_NOTICES.txt") + + parts = [ + "THIRD-PARTY NOTICES", + "", + "MTConnectExplorer is distributed under the Apache License 2.0. It bundles", + "the third-party components listed below, each under its own license,", + "reproduced here to satisfy their attribution requirements. Generated from", + "the pinned build environment.", + "", + _SEP, + ] + + dists = sorted(im.distributions(), key=lambda d: (d.metadata["Name"] or "").lower()) + for dist in dists: + meta = dist.metadata + name, version = meta["Name"], meta["Version"] + block = [f"{name} {version}", f"License: {_license_id(meta)}"] + note = _notes_for(name, version) + if note: + block.append(note) + text = _license_text(dist) + if text: + block.append("") + block.append(text) + parts.append("\n".join(block)) + parts.append(_SEP) + + for name, version, license_id, note in _BUNDLED: + header = f"{name} {version}".rstrip() + parts.append("\n".join([header, f"License: {license_id}", note])) + parts.append(_SEP) + + out.write_text("\n".join(parts) + "\n", encoding="utf-8") + print( + f"Wrote {out} - {len(dists)} Python distributions " + f"+ {len(_BUNDLED)} bundled components." + ) + + +if __name__ == "__main__": + main() diff --git a/hooks/hook-streamlit.py b/packaging/hooks/hook-streamlit.py similarity index 100% rename from hooks/hook-streamlit.py rename to packaging/hooks/hook-streamlit.py diff --git a/readme.md b/readme.md index 0271258..d8e8d8d 100644 --- a/readme.md +++ b/readme.md @@ -218,6 +218,51 @@ The `Datasets` page allows you to view, upload, download, rename, and delete dat ![Datasets Page](.github/Datasets1.png) +## HTTP API + +Everything the backend owns — capture, play, and simulate sessions; datasets; +events/logs; ports; running agents — is drivable over a REST API served by the +same process as the UI (UI at `/`, API at `/api/v1`, interactive docs at +`/api/docs`). The full contract lives in [docs/api.md](docs/api.md). + +- **Auth:** send `X-API-Key: ` on every request, where `` is the + `api_key` field in `config.json`. `GET /api/v1/health` needs no key. +- **Bind address:** the server binds `127.0.0.1` by default (this machine + only); pass `--host 0.0.0.0` to reach it from other machines on your LAN. + Binding beyond localhost without an `api_key` configured is refused at + startup (the key gates `/api`; see the warning below about the UI). +- **Events:** `GET /api/v1/events/stream` streams logs and session status + changes as NDJSON with cursor-based reconnect (see docs/api.md §8). + +### Configuration + +`config.json` is optional and lives in the working directory (next to the +binary, or the repo root in development). It is parsed as JSONC, so `//` and +`/* */` comments are allowed: + +```jsonc +{ + // Logging verbosity: debug, info, warning, error, or critical (default: info). + "log_level": "info", + + // API key accepted in the X-API-Key header. Required to bind a non-local + // --host; without it the API is only reachable by the built-in UI. + "api_key": "replace-with-a-long-random-secret" +} +``` + +A Python client ships as the `mtconnect-explorer-client` wheel (built from +[lib/](lib/), which is also the home of the API's pydantic schemas): + +```python +from mtconnect_explorer_client import ApiClient + +client = ApiClient("http://127.0.0.1:8501/api/v1", api_key="") +session = client.simulate_start(equipment_type="MILL_3AXIS", device_name="Mill") +for event in client.events_stream(scope="simulate"): + print(event) +``` + ## Play The `Play` page allows you to play a `.jsonl` dataset. diff --git a/requirements-src.txt b/requirements-src.txt index 1930efe..75ab5e8 100644 --- a/requirements-src.txt +++ b/requirements-src.txt @@ -4,3 +4,9 @@ pydantic psutil streamlit streamlit-tree-select + +fastapi +build +pytest + +-e ./lib diff --git a/requirements.txt b/requirements.txt index b1306e9..3583093 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,25 @@ altair==6.1.0 altgraph==0.17.5 -annotated-types==0.7.0 -anyio==4.13.0 +annotated-doc==0.0.4 +annotated-types==0.8.0 +anyio==4.14.2 attrs==26.1.0 blinker==1.9.0 +build==1.5.0 cachetools==7.1.2 -certifi==2026.4.22 +certifi==2026.7.22 charset-normalizer==3.4.7 click==8.4.0 colorama==0.4.6 +fastapi==0.141.1 gitdb==4.0.12 GitPython==3.1.50 h11==0.16.0 +httpcore==1.0.9 httptools==0.7.1 -idna==3.15 +httpx==0.28.1 +idna==3.18 +iniconfig==2.3.0 itsdangerous==2.2.0 Jinja2==3.1.6 jsonschema==4.26.0 @@ -25,14 +31,18 @@ packaging==26.2 pandas==3.0.3 pefile==2024.8.26 pillow==12.2.0 +pluggy==1.6.0 protobuf==7.34.1 psutil==7.2.2 pyarrow==24.0.0 pydantic==2.13.4 pydantic_core==2.46.4 pydeck==0.9.2 -pyinstaller==6.21.0 +Pygments==2.20.0 +pyinstaller==6.22.1 pyinstaller-hooks-contrib==2026.6 +pyproject_hooks==1.2.0 +pytest==9.1.1 python-dateutil==2.9.0.post0 python-multipart==0.0.29 pywin32-ctypes==0.2.3 @@ -44,12 +54,12 @@ setuptools==82.0.1 six==1.17.0 smmap==5.0.3 starlette==1.0.0 -streamlit==1.60.0 +streamlit==1.61.1 streamlit-tree-select==0.0.5 tenacity==9.1.4 toml==0.10.2 typing-inspection==0.4.2 -typing_extensions==4.15.0 +typing_extensions==4.16.0 tzdata==2026.2 urllib3==2.7.0 uvicorn==0.47.0 diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..1ec45f4 --- /dev/null +++ b/run.bat @@ -0,0 +1,21 @@ +@echo off + +:: Change to the directory where this script lives +cd /d "%~dp0" + +:: Set up the venv and agent.exe (no_pause: dev.bat won't pause on success) +call dev.bat no_pause || ( + echo Error: Could not set up the environment + pause + exit /b %ERRORLEVEL% +) + +:: Launch the app straight from the venv - no dist/ build required. Any extra +:: args are forwarded to the app, e.g. run.bat --port 9000 +echo Starting MTConnectExplorer... +venv\Scripts\python.exe -m src %* + +:: Pause if run without args (e.g. double-clicked) so output stays visible. +IF "%~1"=="" ( + pause +) diff --git a/src/__version__.py b/src/__version__.py index ee3f564..199d333 100644 --- a/src/__version__.py +++ b/src/__version__.py @@ -1 +1 @@ -__version__ = "0.1.0-alpha.4" +__version__ = "0.1.0-alpha.5" diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000..cfe0eed --- /dev/null +++ b/src/api/__init__.py @@ -0,0 +1 @@ +"""FastAPI backend (docs/api.md). Routes are thin wrappers over src/services/.""" diff --git a/src/api/app.py b/src/api/app.py new file mode 100644 index 0000000..4c3db3f --- /dev/null +++ b/src/api/app.py @@ -0,0 +1,50 @@ +"""FastAPI application factory (docs/api.md §1–§3). + +Mounted under ``/api`` in main.py, so routes live at ``/api/v1/...`` and the +interactive docs at ``/api/docs`` / ``/api/redoc`` / ``/api/openapi.json``. +""" + +import logging + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from src.__version__ import __version__ +from src.services.errors import ServiceError + +from .routers import routers + +logger = logging.getLogger(__name__) + + +def create_api() -> FastAPI: + api = FastAPI( + title="MTConnectExplorer", + version=__version__, + description=( + "REST API for MTConnectExplorer: capture, play, and simulate " + "sessions; datasets; events/logs; ports; running agents. " + "Authenticate with the `X-API-Key` header (config.json `api_key`); " + "`GET /v1/health` is the only unauthenticated endpoint." + ), + docs_url="/docs", + redoc_url="/redoc", + openapi_url="/openapi.json", + ) + + @api.exception_handler(ServiceError) + async def service_error_handler(request: Request, exc: ServiceError) -> JSONResponse: + return JSONResponse(status_code=exc.status, content={"detail": exc.detail()}) + + @api.exception_handler(Exception) + async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: + logger.error(f"unhandled API error on {request.url.path}: {exc}") + return JSONResponse( + status_code=500, + content={"detail": {"code": "internal_error", "message": str(exc)}}, + ) + + for router in routers: + api.include_router(router, prefix="/v1") + + return api diff --git a/src/api/errors.py b/src/api/errors.py new file mode 100644 index 0000000..b6a7910 --- /dev/null +++ b/src/api/errors.py @@ -0,0 +1,25 @@ +"""HTTP error helpers (docs/api.md §3): every non-2xx response carries a typed +object inside FastAPI's native ``detail`` field.""" + +from typing import Any, Dict, Optional + +from fastapi import HTTPException + +from src.services.errors import ServiceError + + +def http_error( + status: int, + code: str, + message: str, + context: Optional[Dict[str, Any]] = None, +) -> HTTPException: + detail: Dict[str, Any] = {"code": code, "message": message} + if context is not None: + detail["context"] = context + return HTTPException(status_code=status, detail=detail) + + +def from_service_error(ex: ServiceError) -> HTTPException: + """Map a services-layer error 1:1 onto its HTTP form.""" + return HTTPException(status_code=ex.status, detail=ex.detail()) diff --git a/src/api/routers/__init__.py b/src/api/routers/__init__.py new file mode 100644 index 0000000..9b8f30f --- /dev/null +++ b/src/api/routers/__init__.py @@ -0,0 +1,13 @@ +"""One APIRouter per service — one Swagger tag each (docs/api.md §1).""" + +from . import capture, datasets, events, play, simulate, system + +# Included in this order; also the tag order in /api/docs. +routers = [ + system.router, + datasets.router, + capture.router, + play.router, + simulate.router, + events.router, +] diff --git a/src/api/routers/capture.py b/src/api/routers/capture.py new file mode 100644 index 0000000..9cffbb5 --- /dev/null +++ b/src/api/routers/capture.py @@ -0,0 +1,63 @@ +"""Capture session endpoints (docs/api.md §5, §7).""" + +from typing import List + +from fastapi import APIRouter, Depends + +from mtconnect_explorer_client.schemas import CaptureCreate, CaptureSessionInfo + +from src.services.capture import capture_manager + +from ..security import require_api_key + +router = APIRouter( + prefix="/capture", + tags=["Capture"], + dependencies=[Depends(require_api_key)], +) + + +@router.post( + "/sessions", + summary="Start a capture session (validated atomically)", + response_model=CaptureSessionInfo, + status_code=201, +) +def create_session(body: CaptureCreate) -> CaptureSessionInfo: + return capture_manager.start(body).to_info(CaptureSessionInfo) + + +@router.get( + "/sessions", + summary="List capture sessions (including not-yet-purged terminal ones)", + response_model=List[CaptureSessionInfo], +) +def list_sessions() -> List[CaptureSessionInfo]: + return capture_manager.list_info() + + +@router.get( + "/sessions/{session_id}", + summary="One capture session", + response_model=CaptureSessionInfo, +) +def get_session(session_id: str) -> CaptureSessionInfo: + return capture_manager.get_info(session_id) + + +@router.delete( + "/sessions/{session_id}", + summary="Stop and remove a capture session", + status_code=204, +) +def delete_session(session_id: str) -> None: + capture_manager.delete(session_id) + + +@router.delete( + "/sessions", + summary="Stop and remove every capture session", + status_code=204, +) +def delete_all_sessions() -> None: + capture_manager.delete_all() diff --git a/src/api/routers/datasets.py b/src/api/routers/datasets.py new file mode 100644 index 0000000..acc83da --- /dev/null +++ b/src/api/routers/datasets.py @@ -0,0 +1,100 @@ +"""Dataset endpoints (docs/api.md §6).""" + +from typing import List + +from fastapi import APIRouter, Depends, UploadFile +from fastapi.responses import FileResponse + +from mtconnect_explorer_client.schemas import ( + DatasetDevices, + DatasetInfo, + RenameRequest, +) + +from src.services import datasets + +from ..errors import http_error +from ..security import require_api_key + +router = APIRouter( + prefix="/datasets", + tags=["Datasets"], + dependencies=[Depends(require_api_key)], +) + + +@router.get( + "", + summary="List all datasets (newest first) with in-use status", + response_model=List[DatasetInfo], +) +def list_datasets() -> List[DatasetInfo]: + return datasets.list_datasets() + + +@router.post( + "/upload", + summary="Upload a dataset (multipart; overwriting is not allowed)", + response_model=DatasetInfo, + status_code=201, +) +def upload(file: UploadFile) -> DatasetInfo: + name = file.filename or "" + if not name.lower().endswith(datasets.EXT): + raise http_error( + 415, "unsupported_type", + f"upload must be a {datasets.EXT} dataset (got '{name}')", + ) + # Starlette has already spooled the part to a temp file; stream it in 1 MB + # chunks to a temp file beside the datasets dir and atomically move it in. + return datasets.save_upload(name, file.file) + + +@router.get( + "/{name}", + summary="Dataset metadata", + response_model=DatasetInfo, +) +def get_dataset(name: str) -> DatasetInfo: + return datasets.get_dataset(name) + + +@router.get( + "/{name}/content", + summary="Download the dataset file (streamed)", + response_class=FileResponse, +) +def download(name: str) -> FileResponse: + path = datasets.existing_path(name) + return FileResponse( + path, + media_type="application/octet-stream", + filename=name, + ) + + +@router.get( + "/{name}/devices", + summary="Devices from the dataset's probe frame (including the Agent device)", + response_model=DatasetDevices, +) +def devices(name: str) -> DatasetDevices: + return datasets.get_devices(name) + + +@router.patch( + "/{name}", + summary="Rename a dataset", + status_code=204, +) +def rename(name: str, body: RenameRequest) -> None: + datasets.rename(name, body.new_name) + + +@router.delete( + "/{name}", + summary="Delete a dataset", + status_code=204, +) +def delete(name: str) -> None: + datasets.delete(name) diff --git a/src/api/routers/events.py b/src/api/routers/events.py new file mode 100644 index 0000000..9ca0833 --- /dev/null +++ b/src/api/routers/events.py @@ -0,0 +1,106 @@ +"""Event endpoints (docs/api.md §8): cursor polling and NDJSON streaming.""" + +import json +import queue +import time +from typing import Iterator, Optional + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import StreamingResponse + +from mtconnect_explorer_client.schemas import EventPage + +from src.services.events import BUS, rfc3339_now + +from ..security import require_api_key + +router = APIRouter( + prefix="/events", + tags=["Events"], + dependencies=[Depends(require_api_key)], +) + +PING_INTERVAL_S = 15.0 + +_SCOPE_QUERY = Query(None, description="Filter: capture | play | simulate.") +_SESSION_QUERY = Query(None, description="Filter: session id, e.g. play-3.") +_LEVEL_QUERY = Query( + None, description="Filter log events to this minimum level (e.g. INFO)." +) +_AFTER_QUERY = Query(None, ge=0, description="Exclusive seq cursor to resume from.") + + +@router.get( + "", + summary="Poll events after a seq cursor", + response_model=EventPage, +) +def poll( + after: Optional[int] = _AFTER_QUERY, + scope: Optional[str] = _SCOPE_QUERY, + session: Optional[str] = _SESSION_QUERY, + level: Optional[str] = _LEVEL_QUERY, + limit: int = Query(500, ge=1, le=10_000, description="Maximum events to return."), +) -> EventPage: + events, next_cursor, dropped = BUS.query( + after=after, scope=scope, session=session, level=level, limit=limit + ) + return EventPage(events=events, next=next_cursor, dropped=dropped) + + +def _ndjson(event: dict) -> str: + return json.dumps(event, separators=(",", ":")) + "\n" + + +def _event_stream( + after: Optional[int], + scope: Optional[str], + session: Optional[str], + level: Optional[str], +) -> Iterator[str]: + """Backfill after the cursor, then stay live. A sync generator — Starlette + runs it on the threadpool, so the blocking queue.get needs no asyncio + bridging. Pings fire every 15 s of idle time and carry the checkpoint seq + ("you are current through N"); they are never stored in the buffer.""" + backfill, _dropped, live = BUS.subscribe( + after=after, scope=scope, session=session, level=level + ) + try: + checkpoint = after or 0 + last_emit = time.monotonic() + for event in backfill: + checkpoint = event["seq"] + yield _ndjson(event) + last_emit = time.monotonic() + while True: + try: + event = live.get(timeout=1.0) + except queue.Empty: + if time.monotonic() - last_emit >= PING_INTERVAL_S: + yield _ndjson({"type": "ping", "seq": checkpoint, "ts": rfc3339_now()}) + last_emit = time.monotonic() + continue + # The live queue is unfiltered; skipped events still advance the + # checkpoint — under this filter they would never be delivered. + checkpoint = event["seq"] + if BUS._matches(event, scope, session, level): + yield _ndjson(event) + last_emit = time.monotonic() + finally: + BUS.unsubscribe(live) + + +@router.get( + "/stream", + summary="Stream events as NDJSON (backfill after cursor, then live)", +) +def stream( + after: Optional[int] = _AFTER_QUERY, + scope: Optional[str] = _SCOPE_QUERY, + session: Optional[str] = _SESSION_QUERY, + level: Optional[str] = _LEVEL_QUERY, +) -> StreamingResponse: + return StreamingResponse( + _event_stream(after, scope, session, level), + media_type="application/x-ndjson", + ) diff --git a/src/api/routers/play.py b/src/api/routers/play.py new file mode 100644 index 0000000..170370c --- /dev/null +++ b/src/api/routers/play.py @@ -0,0 +1,115 @@ +"""Play session endpoints (docs/api.md §5, §7).""" + +from typing import List + +from fastapi import APIRouter, Depends + +from datetime import datetime, timezone + +from mtconnect_explorer_client.schemas import ( + PlayCreate, + PlayProgress, + PlaySeek, + PlaySessionInfo, +) + +from src.services.play import play_manager + +from ..security import require_api_key + +router = APIRouter( + prefix="/play", + tags=["Play"], + dependencies=[Depends(require_api_key)], +) + + +@router.post( + "/sessions", + summary="Start a playback session (dataset + ports validated atomically)", + response_model=PlaySessionInfo, + status_code=201, +) +def create_session(body: PlayCreate) -> PlaySessionInfo: + return play_manager.start(body).to_info(PlaySessionInfo) + + +@router.get( + "/sessions", + summary="List play sessions (including not-yet-purged terminal ones)", + response_model=List[PlaySessionInfo], +) +def list_sessions() -> List[PlaySessionInfo]: + return play_manager.list_info() + + +@router.get( + "/sessions/{session_id}", + summary="One play session", + response_model=PlaySessionInfo, +) +def get_session(session_id: str) -> PlaySessionInfo: + return play_manager.get_info(session_id) + + +@router.get( + "/sessions/{session_id}/progress", + summary="Playhead origin for the scrub bar (duration + pass_start_epoch)", + response_model=PlayProgress, +) +def progress(session_id: str) -> PlayProgress: + sess = play_manager.get(session_id) + p = play_manager.progress_of(sess) + return PlayProgress( + duration=p["duration"] or 0.0, + pass_start_epoch=p["pass_start_epoch"], + looping=p["looping"], + paused=p.get("paused", False), + position=p.get("position"), + server_time=datetime.now(timezone.utc), + ) + + +@router.post( + "/sessions/{session_id}/seek", + summary="Seek a running playback to a position (requires re-timestamp mode)", + status_code=204, +) +def seek(session_id: str, body: PlaySeek) -> None: + play_manager.seek(session_id, body.position) + + +@router.post( + "/sessions/{session_id}/pause", + summary="Pause a running playback (holds the playhead; any timestamp mode)", + status_code=204, +) +def pause(session_id: str) -> None: + play_manager.pause(session_id) + + +@router.post( + "/sessions/{session_id}/resume", + summary="Resume a paused playback", + status_code=204, +) +def resume(session_id: str) -> None: + play_manager.resume(session_id) + + +@router.delete( + "/sessions/{session_id}", + summary="Stop and remove a play session", + status_code=204, +) +def delete_session(session_id: str) -> None: + play_manager.delete(session_id) + + +@router.delete( + "/sessions", + summary="Stop and remove every play session", + status_code=204, +) +def delete_all_sessions() -> None: + play_manager.delete_all() diff --git a/src/api/routers/simulate.py b/src/api/routers/simulate.py new file mode 100644 index 0000000..96e2755 --- /dev/null +++ b/src/api/routers/simulate.py @@ -0,0 +1,121 @@ +"""Simulate session endpoints (docs/api.md §5, §7). + +Session creation is one endpoint per equipment type +(``POST /simulate/start/{equipment_type}``), each with a dedicated body model +so every equipment parameter set is fully validated by its own schema — the +catalog's ``create_path`` tells clients where to post. The session lifecycle +endpoints (list/get/delete/routine) are shared across types. +""" + +from datetime import datetime, timezone +from typing import List + +from fastapi import APIRouter, Depends + +from mtconnect_explorer_client.schemas import ( + EquipmentInfo, + Mill3AxisCreate, + Mill5AxisTrunnionBcCreate, + Robot6DofCreate, + RoutineInfo, + SimulateSessionInfo, +) + +from src.services.simulate import equipment_catalog, simulate_manager + +from ..security import require_api_key + +router = APIRouter( + prefix="/simulate", + tags=["Simulate"], + dependencies=[Depends(require_api_key)], +) + + +@router.get( + "/equipment", + summary="Equipment catalog: types, labels, routines, and create endpoints", + response_model=List[EquipmentInfo], +) +def equipment() -> List[EquipmentInfo]: + return equipment_catalog() + + +@router.post( + "/start/mill_3axis", + summary="Start a 3-axis mill simulation", + response_model=SimulateSessionInfo, + status_code=201, +) +def start_mill_3axis(body: Mill3AxisCreate) -> SimulateSessionInfo: + return simulate_manager.start(body).to_info(SimulateSessionInfo) + + +@router.post( + "/start/mill_5axis_trunnion_bc", + summary="Start a 5-axis trunnion mill (B/C) simulation", + response_model=SimulateSessionInfo, + status_code=201, +) +def start_mill_5axis_trunnion_bc(body: Mill5AxisTrunnionBcCreate) -> SimulateSessionInfo: + return simulate_manager.start(body).to_info(SimulateSessionInfo) + + +@router.post( + "/start/robot_6dof", + summary="Start a 6-DOF robot simulation", + response_model=SimulateSessionInfo, + status_code=201, +) +def start_robot_6dof(body: Robot6DofCreate) -> SimulateSessionInfo: + return simulate_manager.start(body).to_info(SimulateSessionInfo) + + +@router.get( + "/sessions", + summary="List simulate sessions (including not-yet-purged terminal ones)", + response_model=List[SimulateSessionInfo], +) +def list_sessions() -> List[SimulateSessionInfo]: + return simulate_manager.list_info() + + +@router.get( + "/sessions/{session_id}", + summary="One simulate session", + response_model=SimulateSessionInfo, +) +def get_session(session_id: str) -> SimulateSessionInfo: + return simulate_manager.get_info(session_id) + + +@router.get( + "/sessions/{session_id}/routine", + summary="Routine spec + start epoch for the phase-locked 3D view", + response_model=RoutineInfo, +) +def routine(session_id: str) -> RoutineInfo: + sess = simulate_manager.get(session_id) + return RoutineInfo( + spec=sess.spec.to_dict(), + start_epoch=simulate_manager.start_epoch_of(sess), + server_time=datetime.now(timezone.utc), + ) + + +@router.delete( + "/sessions/{session_id}", + summary="Stop and remove a simulate session", + status_code=204, +) +def delete_session(session_id: str) -> None: + simulate_manager.delete(session_id) + + +@router.delete( + "/sessions", + summary="Stop and remove every simulate session", + status_code=204, +) +def delete_all_sessions() -> None: + simulate_manager.delete_all() diff --git a/src/api/routers/system.py b/src/api/routers/system.py new file mode 100644 index 0000000..169dc19 --- /dev/null +++ b/src/api/routers/system.py @@ -0,0 +1,60 @@ +"""System endpoints (docs/api.md §4).""" + +from typing import List + +from fastapi import APIRouter, Depends, Query + +from mtconnect_explorer_client.schemas import ( + AgentInfo, + HealthInfo, + PortsInfo, + VersionInfo, +) + +from src.__version__ import __version__ +from src.services import ports + +from ..security import require_api_key + +router = APIRouter(tags=["System"]) + + +@router.get( + "/health", + summary="Liveness probe (the only unauthenticated endpoint)", + response_model=HealthInfo, +) +def health() -> HealthInfo: + return HealthInfo(status="ok") + + +@router.get( + "/version", + summary="Application and API version", + response_model=VersionInfo, + dependencies=[Depends(require_api_key)], +) +def version() -> VersionInfo: + return VersionInfo(version=__version__, api="v1") + + +@router.get( + "/agents", + summary="Local MTConnect agents run by live Play/Simulate sessions", + response_model=List[AgentInfo], + dependencies=[Depends(require_api_key)], +) +def agents() -> List[AgentInfo]: + return ports.running_agents() + + +@router.get( + "/ports", + summary="Used ports plus free suggestions for pre-filling launch forms", + response_model=PortsInfo, + dependencies=[Depends(require_api_key)], +) +def ports_info( + shdr_count: int = Query(1, ge=0, le=64, description="How many SHDR ports to suggest."), +) -> PortsInfo: + return ports.suggest_ports(shdr_count) diff --git a/src/api/security.py b/src/api/security.py new file mode 100644 index 0000000..edf69a6 --- /dev/null +++ b/src/api/security.py @@ -0,0 +1,27 @@ +"""X-API-Key authentication dependency (docs/api.md §2). + +Attached at router level to every router; the sole exception is +``GET /v1/health`` (liveness probes can't send keys). Using a Security +dependency (rather than middleware) also registers the scheme in OpenAPI, so +Swagger's Authorize button can set the key for try-it-out calls. +""" + +from typing import Optional + +from fastapi import Security +from fastapi.security import APIKeyHeader + +from src.services import auth + +from .errors import http_error + +_api_key_header = APIKeyHeader( + name="X-API-Key", + auto_error=False, + description="config.json api_key, or the in-process runtime token.", +) + + +def require_api_key(api_key: Optional[str] = Security(_api_key_header)) -> None: + if not api_key or api_key not in auth.valid_keys(): + raise http_error(401, "unauthorized", "missing or invalid API key") diff --git a/src/client.py b/src/client.py index a790918..2ec9597 100644 --- a/src/client.py +++ b/src/client.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import http.client import logging import threading @@ -6,6 +6,7 @@ import urllib.error import urllib.parse import urllib.request +from collections import deque from collections.abc import Iterator from typing import Any, Dict, List, Literal, Optional, Tuple, Annotated import xml.etree.ElementTree as ET @@ -902,17 +903,22 @@ def __init__( configuration: MtConnectClientConfiguration, interval_ms: int = 1000, mode: Literal["stream", "sample", "current"] = "stream", - history_max: int = 100, + history_max_points: int = 100_000, ) -> None: self._configuration = configuration self._interval_ms = interval_ms self._mode = mode - self._history_max = history_max + # Absolute per-DataItem ceiling. The retained span is otherwise driven by + # set_history_window() (the largest configured plot duration), so a plot + # of that duration can fill even at a high sample rate. + self._history_max_points = history_max_points + self._history_max_age: Optional[timedelta] = None self._client = MtConnectClient(configuration) self._probe_devices: List[MtConnectDevice] = [] self._observations: Dict[str, MtConnectObservation] = {} - self._history: Dict[str, List[MtConnectObservation]] = {} + # Per-DataItem ring buffer (maxlen = history_max_points) — O(1) eviction. + self._history: Dict[str, deque] = {} self._lock = threading.Lock() self._thread: Optional[threading.Thread] = None @@ -936,7 +942,16 @@ def get_state(self) -> Tuple[List[MtConnectDevice], Dict[str, MtConnectObservati def get_history(self, data_item_id: str) -> List[MtConnectObservation]: """Return the buffered observation history for a DataItem.""" with self._lock: - return list(self._history.get(data_item_id, [])) + return list(self._history.get(data_item_id, ())) + + def set_history_window(self, max_age: Optional[timedelta]) -> None: + """Set how far back to retain samples per DataItem: everything within + ``max_age`` of that item's most recent sample is kept (still capped at + ``history_max_points``). The Explore UI sets this from the largest of the + history / pinned-plot durations so a plot of that span can fill up. Pass + ``None`` to disable age-based trimming (only the point cap applies).""" + with self._lock: + self._history_max_age = max_age def start(self) -> "MtConnectExplorer": if self._thread is not None: @@ -958,15 +973,24 @@ def stop(self) -> None: def _update_observations(self, observations: List[MtConnectObservation]) -> None: with self._lock: + max_age = self._history_max_age for obs in observations: did = obs.attributes.get("dataItemId", "") if not did: continue self._observations[did] = obs - hist = self._history.setdefault(did, []) + hist = self._history.get(did) + if hist is None: + # maxlen enforces the absolute point cap with O(1) eviction. + hist = deque(maxlen=self._history_max_points) + self._history[did] = hist hist.append(obs) - if len(hist) > self._history_max: - del hist[: -self._history_max] + # Time-based retention: drop samples older than max_age relative to + # this item's most recent sample (front of a time-ordered deque). + if max_age is not None: + cutoff = hist[-1].timestamp - max_age + while hist and hist[0].timestamp < cutoff: + hist.popleft() def _loop(self) -> None: reader = None diff --git a/src/converter.py b/src/converter.py index 09d7112..b4efdd3 100644 --- a/src/converter.py +++ b/src/converter.py @@ -1,4 +1,5 @@ import os +import json import unicodedata import string import logging @@ -324,10 +325,21 @@ def __init__(self, device_name: str): def is_finished(self) -> bool: return self._reader.is_finished + @property + def next_offset(self) -> Optional[int]: + """Byte offset of the next frame to read (for the seek keyframe index).""" + return self._reader.next_offset + def open(self, path: str) -> "MtConnectDatasetReader": self._reader.open(path) return self + def open_at(self, path: str, offset: int) -> "MtConnectDatasetReader": + """Open positioned at a specific frame byte offset (an observation frame, + not the probe) — used to resume delta replay from an indexed snapshot.""" + self._reader.open_at(path, offset) + return self + def close(self) -> None: if self._reader.is_open: self._reader.close() @@ -356,9 +368,127 @@ def read(self) -> Tuple[datetime, List[ShdrObservation]]: return data_frame.timestamp, shdr_obs +def dataset_duration(file_path: str) -> float: + """Total playback span, in seconds, of a RawMTConnect-v0.1 dataset — from the + first observation frame (the frame after the leading probe) to the last frame. + + This matches the span the player replays in one pass (``pass_frame_start`` in + ``_loop`` is the first post-probe frame). Returns 0.0 for an empty or + probe-only dataset. The reader is forward-only with no index, so duration is + not known during playback; this pre-scans the file, but parses only the two + boundary lines' timestamps (not every frame's payload), so it stays cheap on + large files.""" + def _ts(line: str) -> datetime: + return datetime.fromisoformat(json.loads(line)["TIMESTAMP"]) + + with open(file_path, "r") as f: + f.readline() # header + f.readline() # probe frame (first data line) — skipped + start_line = f.readline() # first observation frame = pass start + if not start_line.strip(): + return 0.0 + last_line = start_line + for line in f: + if line.strip(): + last_line = line + return max((_ts(last_line) - _ts(start_line)).total_seconds(), 0.0) + + +class _DeviceStateFold: + """Accumulates a single device's full observation state so it can be re-emitted + as a ``/current``-equivalent snapshot. Used to build the seek keyframe index: + replaying every observation from the start folds into this, and a snapshot taken + at time T is the exact device state at T (SHDR observations that, emitted to a + fresh/uncertain agent, reconstruct that state). + + Cumulative semantics mirror the agent: values/time-series are last-wins; + conditions keep the active set per native code (a code-less NORMAL clears all); + DATA_SET/TABLE accumulate with reset + per-entry delete; assets add/remove.""" + + def __init__(self) -> None: + self.values: Dict[str, ShdrDataItem] = {} + self.time_series: Dict[str, ShdrTimeSeries] = {} + # name -> {native_code (""=no code) -> condition}. Empty dict == all NORMAL. + self.conditions: Dict[str, Dict[str, ShdrCondition]] = {} + self.data_sets: Dict[str, Dict[str, Any]] = {} + self.tables: Dict[str, Dict[str, Dict[str, Any]]] = {} + self.assets: Dict[str, ShdrAsset] = {} + + def apply(self, obs: ShdrObservation) -> None: + if isinstance(obs, ShdrDataItem): + self.values[obs.name] = obs + elif isinstance(obs, ShdrTimeSeries): + self.time_series[obs.name] = obs + elif isinstance(obs, ShdrCondition): + codes = self.conditions.setdefault(obs.name, {}) + code = obs.native_code or "" + if obs.level == ShdrConditionLevel.NORMAL: + if code: + codes.pop(code, None) # NORMAL for a code clears that code + else: + codes.clear() # code-less NORMAL clears all + elif obs.level == ShdrConditionLevel.UNAVAILABLE: + codes.clear() + codes["\x00unavail"] = obs + else: # WARNING / FAULT + codes.pop("\x00unavail", None) + codes[code] = obs + elif isinstance(obs, ShdrDataSet): + cur = self.data_sets.setdefault(obs.name, {}) + if obs.reset_triggered: + cur.clear() + for k, v in obs.entries.items(): + if v is None or v == "": + cur.pop(k, None) + else: + cur[k] = v + elif isinstance(obs, ShdrTable): + cur = self.tables.setdefault(obs.name, {}) + if obs.reset_triggered: + cur.clear() + for rk, cells in obs.rows.items(): + if not cells: + cur.pop(rk, None) + else: + cur[rk] = dict(cells) + elif isinstance(obs, ShdrAsset): + self.assets[obs.asset_id] = obs + elif isinstance(obs, ShdrRemoveAsset): + self.assets.pop(obs.asset_id, None) + elif isinstance(obs, ShdrRemoveAllAssets): + self.assets = { + aid: a for aid, a in self.assets.items() if a.asset_type != obs.asset_type + } + + def snapshot(self) -> List[ShdrObservation]: + """A fresh list of timestamp-less observations that reconstructs the current + state when emitted (timestamps are assigned by the player at emit time).""" + out: List[ShdrObservation] = [] + for v in self.values.values(): + out.append(v.model_copy(update={"timestamp": None})) + for ts in self.time_series.values(): + out.append(ts.model_copy(update={"timestamp": None})) + for name, codes in self.conditions.items(): + # Clear any stale agent state for this condition first, then set actives. + out.append(ShdrCondition(name=name, level=ShdrConditionLevel.NORMAL)) + for c in codes.values(): + out.append(c.model_copy(update={"timestamp": None})) + for name, entries in self.data_sets.items(): + out.append(ShdrDataSet(name=name, entries=dict(entries), reset_triggered="MANUAL")) + for name, rows in self.tables.items(): + out.append(ShdrTable( + name=name, + rows={rk: dict(cells) for rk, cells in rows.items()}, + reset_triggered="MANUAL", + )) + for a in self.assets.values(): + out.append(a.model_copy(update={"timestamp": None})) + return out + + class MtConnectDatasetPlayer: - def __init__(self, host: str = "127.0.0.1", port: int = 7878, re_timestamp: bool = True, loop: bool = True, thread_name: str = "dataset-player"): + def __init__(self, host: str = "127.0.0.1", port: int = 7878, re_timestamp: bool = True, loop: bool = True, thread_name: str = "dataset-player", snapshot_interval_s: float = 15.0): if not isinstance(host, str): raise TypeError("host must be a str") self._host = host @@ -389,6 +519,13 @@ def __init__(self, host: str = "127.0.0.1", port: int = 7878, re_timestamp: bool # NB: not named ``self._loop`` so it does not shadow the _loop() method. self._loop_enabled = loop + if not isinstance(snapshot_interval_s, (int, float)) or snapshot_interval_s <= 0: + raise ValueError("snapshot_interval_s must be a positive number") + # Keyframe cadence (dataset seconds): a full-state snapshot is indexed every + # this-many seconds so a seek replays only from the nearest snapshot, not + # from the top. Set at construction, fixed for the session. + self._snapshot_interval_s = float(snapshot_interval_s) + self._file_path: Optional[str] = None self._device_name: Optional[str] = None self._reader: Optional[MtConnectDatasetReader] = None @@ -398,10 +535,117 @@ def __init__(self, host: str = "127.0.0.1", port: int = 7878, re_timestamp: bool self._is_alive = False self._is_finished = False + # Total dataset span (seconds), computed at open(); the scrub bar's range. + self._duration: float = 0.0 + + # Seek keyframe index, built once at open(): sorted list of + # (snapshot_time_s, file_byte_offset, snapshot_observations). snapshot_time_s + # is dataset in-pass time; the observations reconstruct full device state as + # of just-before that time; file_byte_offset points at the first frame at/after + # it (where delta replay resumes). _pass_frame_start is the dataset timestamp + # of in-pass offset 0 (so frame_offset = frame_ts - _pass_frame_start). + self._snapshots: List[Tuple[float, int, List[ShdrObservation]]] = [] + self._pass_frame_start: Optional[datetime] = None + + # --- Playhead (target) clock: what pause freezes and the UI reads ---------- + # The playhead advances at 1x wall-clock while playing, is frozen while + # paused, and is set directly by a seek. Modelled as an anchor: + # target(now) = _pos_anchor + (now - _wall_anchor) [playing] + # target = _pos_anchor [paused] + # A frame at in-pass offset O is emitted once target >= O: unpaced when + # behind (post-seek catch-up), paced when caught up. Pause never blocks the + # emit loop — it only stops the target advancing, so an in-flight catch-up + # runs to the (now frozen) target and then idles. + self._pos_anchor: float = 0.0 + self._wall_anchor: Optional[datetime] = None + self._paused: bool = False + + # --- Emit-timestamp clock (re_timestamp): kept separate from the playhead -- + # so a backward seek/loop-wrap never emits a decreasing timestamp (the agent + # rejects those). Emitted ts = _emit_origin + _elapsed_offset + frame_offset, + # with a monotonic guard. _elapsed_offset accumulates completed passes. + self._emit_origin: Optional[datetime] = None + self._elapsed_offset: timedelta = timedelta(0) + self._last_emit_ts: Optional[datetime] = None + + # Pending seek target (in-pass seconds); latest-wins, applied by _loop. + self._seek_target: Optional[float] = None + self._seek_lock = threading.Lock() + @property def is_finished(self) -> bool: return self._is_finished + @property + def duration(self) -> float: + """Total dataset span in seconds (the scrub range).""" + return self._duration + + def _clamp(self, pos: float) -> float: + return min(max(pos, 0.0), self._duration) if self._duration else max(0.0, pos) + + def _target_pos(self, now: Optional[datetime] = None) -> float: + """In-pass playhead target (seconds): advances with wall-clock while + playing, frozen while paused. Not clamped (the loop wraps at duration).""" + if self._wall_anchor is None: + return 0.0 + if self._paused: + return self._pos_anchor + now = now or datetime.now(timezone.utc) + return self._pos_anchor + (now - self._wall_anchor).total_seconds() + + @property + def pass_start_epoch(self) -> Optional[float]: + """Unix time corresponding to offset 0 of the current pass, or None until + playback has started. The UI computes the live playhead as + ``now - pass_start_epoch`` (except while paused — see ``paused``/``position``), + so this is republished on start, loop-wrap, seek, and pause/resume.""" + if self._wall_anchor is None: + return None + return self._wall_anchor.timestamp() - self._pos_anchor + + @property + def position(self) -> float: + """Current playhead within the pass, in seconds (clamped to [0, duration]). + Frozen while paused (the target stops advancing).""" + if self._wall_anchor is None: + return 0.0 + return self._clamp(self._target_pos()) + + @property + def is_paused(self) -> bool: + return self._paused + + def pause(self) -> None: + """Freeze the playhead target (no-op if already paused). Does NOT block the + emit loop — a post-seek fast-forward keeps releasing data up to the frozen + target and then idles. Works in any timestamp mode (pause does not rewind).""" + with self._seek_lock: + if self._paused: + return + self._pos_anchor = self._clamp(self._target_pos()) + self._wall_anchor = datetime.now(timezone.utc) + self._paused = True + + def resume(self) -> None: + """Resume (no-op if not paused): the target advances again from where it was.""" + with self._seek_lock: + if not self._paused: + return + self._wall_anchor = datetime.now(timezone.utc) + self._paused = False + + def seek(self, offset_seconds: float) -> None: + """Request a seek to ``offset_seconds`` into the dataset (latest-wins). + + Applied by the playback thread: it jumps to the nearest indexed snapshot at + or before the target, emits that full-state snapshot, then fast-forwards the + remaining (≤ snapshot interval) deltas to the target and resumes paced + playback. Only valid with re_timestamp on (the caller enforces this).""" + target = self._clamp(float(offset_seconds)) + with self._seek_lock: + self._seek_target = target + def open(self, file_path: str, device_name: str, autoplay: bool = True) -> "MtConnectDatasetPlayer": if not isinstance(file_path, str): raise TypeError("file_path must be a str") @@ -417,6 +661,12 @@ def open(self, file_path: str, device_name: str, autoplay: bool = True) -> "MtCo self._file_path = file_path self._device_name = device_name + self._duration = dataset_duration(file_path) + + # Build the seek keyframe index (one pre-scan pass over this device's frames) + # so a later scrub replays only from the nearest snapshot, not from the top. + self._build_snapshot_index() + self._reader = MtConnectDatasetReader(device_name).open(file_path) self._adapter = ShdrAdapter( ShdrAdapterConfiguration( @@ -430,11 +680,45 @@ def open(self, file_path: str, device_name: str, autoplay: bool = True) -> "MtCo if autoplay: self.play() return self + + def _build_snapshot_index(self) -> None: + """One pre-scan pass: fold every observation into running device state and + record a full-state snapshot every ``_snapshot_interval_s`` of dataset time. + Each entry is (snapshot_time_s, byte_offset_of_first_frame_at/after_it, + snapshot_observations); the snapshot is the state strictly before that frame, + so a seek emits it then replays the ≤interval remaining deltas to the target.""" + reader = MtConnectDatasetReader(self._device_name).open(self._file_path) + try: + reader.read_probe_device_model() # consume the probe/device-model frame + fold = _DeviceStateFold() + snapshots: List[Tuple[float, int, List[ShdrObservation]]] = [] + interval = self._snapshot_interval_s + next_snap = interval + pass_frame_start: Optional[datetime] = None + while not reader.is_finished: + byte_offset = reader.next_offset + frame_timestamp, obs_list = reader.read() + if pass_frame_start is None: + pass_frame_start = frame_timestamp + off = (frame_timestamp - pass_frame_start).total_seconds() + while off >= next_snap: + snapshots.append((next_snap, byte_offset, fold.snapshot())) + next_snap += interval + for o in obs_list: + fold.apply(o) + self._snapshots = snapshots + self._pass_frame_start = pass_frame_start + logger.debug( + "Built %d seek snapshot(s) at %.0fs spacing for '%s'", + len(snapshots), interval, self._device_name, + ) + finally: + reader.close() def play(self) -> None: if self._reader is None or self._adapter is None: raise RuntimeError("Not opened") - + self._is_alive = False self._is_finished = False self._thread = threading.Thread( @@ -454,52 +738,146 @@ def stop(self) -> None: self._adapter.stop() self._adapter = None + def _reopen_reader(self) -> "MtConnectDatasetReader": + """Close the current reader and open a fresh one positioned at the first + observation frame (probe discarded) — used for loop-wrap and for a seek to a + target before the first snapshot, since the reader is forward-only.""" + if self._reader is not None: + try: + self._reader.close() + except Exception: + pass + reader = MtConnectDatasetReader(self._device_name).open(self._file_path) + reader.read_probe_device_model() # discard device-model (probe) line + self._reader = reader + return reader + + def _reopen_reader_at(self, byte_offset: int) -> "MtConnectDatasetReader": + """Reopen the reader positioned at an indexed frame byte offset (a snapshot's + resume point), skipping everything before it.""" + if self._reader is not None: + try: + self._reader.close() + except Exception: + pass + self._reader = MtConnectDatasetReader(self._device_name).open_at( + self._file_path, byte_offset + ) + return self._reader + + def _emit_ts(self, frame_offset: float) -> datetime: + """Next emit timestamp (re_timestamp mode): emit_origin + elapsed + offset, + with a strictly-increasing guard so a backward seek/loop-wrap never emits a + decreasing timestamp (the agent rejects those).""" + ts = self._emit_origin + self._elapsed_offset + timedelta(seconds=frame_offset) + if self._last_emit_ts is not None and ts <= self._last_emit_ts: + ts = self._last_emit_ts + timedelta(milliseconds=1) + self._last_emit_ts = ts + return ts + + def _emit_observations(self, obs_list: List[ShdrObservation], frame_offset: float) -> None: + if not obs_list: + return + if self._re_timestamp: + new_ts = self._emit_ts(frame_offset) + obs_list = [o.model_copy(update={"timestamp": new_ts}) for o in obs_list] + self._adapter.enqueue(obs_list) + + def _snapshot_at(self, target: float) -> Optional[Tuple[float, int, List[ShdrObservation]]]: + """The indexed snapshot with the greatest time ≤ target, or None if target is + before the first snapshot (replay from the top).""" + chosen: Optional[Tuple[float, int, List[ShdrObservation]]] = None + for snap in self._snapshots: + if snap[0] <= target: + chosen = snap + else: + break + return chosen + + def _apply_seek(self, target: float) -> None: + """Seek to in-pass offset ``target`` via the keyframe index: jump to the + nearest snapshot ≤ target, emit its full state, and let the loop fast-forward + the remaining (≤ interval) deltas to the target. + + Playhead: anchor so the target is 'now' (paused state preserved). Emit: rebase + ``_emit_origin`` so the first re-emitted timestamp is just past the last one, + keeping timestamps strictly increasing across a backward jump.""" + snap = self._snapshot_at(target) + self._pos_anchor = self._clamp(target) + self._wall_anchor = datetime.now(timezone.utc) + if snap is not None: + snap_time, byte_offset, snap_obs = snap + self._reopen_reader_at(byte_offset) + emit_base = snap_time + else: + self._reopen_reader() # target precedes the first snapshot → from top + emit_base = 0.0 + snap_obs = None + if self._last_emit_ts is not None: + self._emit_origin = ( + self._last_emit_ts + timedelta(milliseconds=1) + - self._elapsed_offset - timedelta(seconds=emit_base) + ) + if snap_obs: + # Emit the snapshot state first so the agent is at the snapshot before the + # ≤interval of deltas replay on top of it. + self._emit_observations(snap_obs, emit_base) + logger.info("Seek to %.1fs (from snapshot at %.0fs)", target, emit_base) + def _loop(self) -> None: try: self._is_alive = True self._is_finished = False - start = datetime.now(timezone.utc) - # Cumulative dataset duration of all completed passes. Frame offsets - # are scheduled and re-stamped relative to (start + elapsed_offset) - # so looping playback advances monotonically without resetting time. - elapsed_offset = timedelta(0) + # Playhead anchored at offset 0 = now; advances at 1x while playing. + self._pos_anchor = 0.0 + self._wall_anchor = datetime.now(timezone.utc) + self._paused = False + # Emit clock. On the non-seek path _emit_origin == _wall_anchor, so + # emitted timestamps match the pre-refactor behavior (loopback test). + self._elapsed_offset = timedelta(0) + self._emit_origin = self._wall_anchor + self._last_emit_ts = None first_pass = True loop_number = 1 logger.debug("Thread started") while not self._stop_event.is_set(): - # Obtain a reader positioned at the first observation frame. The - # first pass reuses the reader opened in open() (probe already - # consumed); later passes re-open and discard the probe line. + # First pass reuses the reader from open() (probe consumed); later + # passes re-open from the top. if first_pass: reader = self._reader first_pass = False else: - reader = MtConnectDatasetReader(self._device_name).open(self._file_path) - reader.read_probe_device_model() # discard device-model (probe) line - self._reader = reader + reader = self._reopen_reader() if self._loop_enabled: logger.info(f"Playback loop {loop_number}") - pass_frame_start: Optional[datetime] = None - pass_last_offset = timedelta(0) prev_frame_timestamp: Optional[datetime] = None + any_frames = False while not reader.is_finished: if self._stop_event.is_set(): logger.debug("Thread stop received") break + # Apply a pending seek before reading the next frame: jump to the + # nearest snapshot, emit its state, then fast-forward to target. + with self._seek_lock: + seek_target = self._seek_target + self._seek_target = None + if seek_target is not None: + self._apply_seek(seek_target) + reader = self._reader + prev_frame_timestamp = None + continue + frame_timestamp, frame_observations = reader.read() - if pass_frame_start is None: - pass_frame_start = frame_timestamp - frame_offset = frame_timestamp - pass_frame_start - - # Guard against an absurd GAP between consecutive frames (which - # would otherwise stall paced playback for that long and usually - # signals corrupt/misordered data) — NOT the dataset's total - # length, which is unbounded. + any_frames = True + frame_offset = (frame_timestamp - self._pass_frame_start).total_seconds() + + # Guard against an absurd GAP between consecutive frames (corrupt/ + # misordered data) — NOT the dataset's total length. max_gap_minutes = 10 if prev_frame_timestamp is not None: gap = frame_timestamp - prev_frame_timestamp @@ -510,48 +888,61 @@ def _loop(self) -> None: ) prev_frame_timestamp = frame_timestamp - total_delta = elapsed_offset + frame_offset - - # Wait for frame time - delta = datetime.now(timezone.utc) - start - while total_delta > delta: + # Emit once the playhead target reaches this frame. Behind-target + # frames (a post-seek fast-forward) fall through immediately; a + # caught-up frame paces to real time; a paused (frozen) target + # holds here WITHOUT blocking any earlier catch-up. A seek + # interrupts so it applies at once. + interrupted = False + while frame_offset > self._target_pos(): if self._stop_event.is_set(): logger.debug("Thread stop received") return + with self._seek_lock: + if self._seek_target is not None: + interrupted = True + if interrupted: + break time.sleep(0.001) - delta = datetime.now(timezone.utc) - start - - # Re-stamp relative to the play start so each frame's offset - # from the start of playback exactly matches its original - # offset (carried forward across loops by elapsed_offset). - # Using the actual emit time (datetime.now) would drift by the - # scheduling jitter accumulated up to this frame. - if self._re_timestamp: - new_ts = start + total_delta - frame_observations = [ - o.model_copy(update={"timestamp": new_ts}) for o in frame_observations - ] - - self._adapter.enqueue(frame_observations) + if interrupted: + # Abandon this frame; the seek re-opens the reader, so it is + # simply re-read after the jump (no data lost). + continue + + self._emit_observations(frame_observations, frame_offset) logger.debug(f"Enqueued {len(frame_observations)} SHDR observations.") - pass_last_offset = frame_offset - # Pass complete. Stop here unless looping is enabled. + # Pass complete (reader exhausted) or stopped. if self._stop_event.is_set() or not self._loop_enabled: break - - # An empty pass (no observation frames) would otherwise spin the - # loop; treat it as the end of playback. - if pass_frame_start is None: + if not any_frames: logger.debug("No observation frames to loop; stopping") break - # Carry this pass's duration forward so the next pass continues - # seamlessly from where this one ended. - elapsed_offset += pass_last_offset - reader.close() - loop_number += 1 - logger.debug("Looping dataset playback") + # Let the playhead run out the remaining span so the pass lasts the + # full duration (keeps the scrub bar aligned), unless a seek arrives. + tail_seek = False + while self._target_pos() < self._duration: + if self._stop_event.is_set(): + logger.debug("Thread stop received") + return + with self._seek_lock: + if self._seek_target is not None: + tail_seek = True + if tail_seek: + break + time.sleep(0.02) + + if not tail_seek: + # Wrap: carry a full pass forward for emit-ts monotonicity and + # shift the playhead anchor back one duration so the next pass's + # target restarts near 0. + self._elapsed_offset += timedelta(seconds=self._duration) + self._pos_anchor -= self._duration + loop_number += 1 + logger.debug("Looping dataset playback") + # else: a seek is pending — loop back, reopen, and let it apply + # (no wrap, so the seek's anchor/emit rebase is authoritative). self._is_finished = True logger.debug("Player finished") diff --git a/src/dataset.py b/src/dataset.py index 58f5b59..4fd16a7 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -41,11 +41,22 @@ def __init__(self, header: str = "X-v0.2"): self._file: TextIOWrapper = None self._next_line: str = None + # Byte-offset (an opaque text-stream tell() cookie, captured at a line + # boundary) of the frame that read()/peek() will return next. Lets callers + # index frame positions and later reopen positioned exactly at a frame line + # (no partial-line reassembly, since offsets are always line-start). + self._next_offset: Optional[int] = None @property def is_open(self) -> bool: return self._file is not None + @property + def next_offset(self) -> Optional[int]: + """tell() cookie of the next frame line (the one read()/peek() returns), + or None if not open. Pass it to open_at() to resume from this frame.""" + return self._next_offset + @property def is_finished(self) -> bool: if not self.is_open: @@ -72,20 +83,44 @@ def open(self, path: str) -> "DatasetReader": self._file.close() raise Exception(f"dataset header '{header}' does not match expected header: '{self._header}'") + self._next_offset = self._file.tell() + self._next_line = self._file.readline() + return self + + def open_at(self, path: str, offset: int) -> "DatasetReader": + """Open ``path`` and position the reader at the frame line starting at + ``offset`` (a cookie previously returned by ``next_offset``). The header + is still validated. Used to resume playback from an indexed frame without + re-reading everything before it.""" + if not isinstance(path, str): + raise TypeError("path must be a str") + if self._file is not None: + raise Exception("file is already open") + if not os.path.isfile(path): + raise FileExistsError(f"file '{path}' is not a file.") + self._path = path + self._file = open(self._path, 'r') + header = self._file.readline().rstrip("\n") + if header != self._header: + self._file.close() + raise Exception(f"dataset header '{header}' does not match expected header: '{self._header}'") + self._file.seek(offset) + self._next_offset = offset self._next_line = self._file.readline() return self def close(self) -> None: if self._file is None: raise Exception("file is not open") - + self._file.close() def read(self) -> DataFrame: if self._file is None: raise Exception("file is not open") - + frame = DataFrame.model_validate_json(self._next_line) + self._next_offset = self._file.tell() self._next_line = self._file.readline() if self._next_line is None or self._next_line == "": self.close() diff --git a/src/main.py b/src/main.py index 31b5529..a9c9cfe 100644 --- a/src/main.py +++ b/src/main.py @@ -1,13 +1,12 @@ -import os import argparse import logging -import streamlit.web.bootstrap -from typing import Literal -from pydantic import BaseModel, Field +import sys from pathlib import Path from .__version__ import __version__ -from .test import run_mirror_test, run_loopback_test, run_play_test + +# Hosts considered local for the refuse-to-start check (docs/api.md §2). +_LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"} def main(): @@ -30,9 +29,20 @@ def main(): parser.add_argument("--device", metavar="NAME", help="--convert-shdr-log: restrict output to a single device") parser.add_argument("--port", type=int, default=8501, help="Port to run the web server on (default: 8501)") + parser.add_argument("--host", default="127.0.0.1", + help="Address to bind the web server to (default: 127.0.0.1). " + "Binding a non-local address requires an api_key in config.json.") + # Hidden: self-exec worker mode (spawned by services.hosts.SubprocessHost; + # config on stdin, NDJSON events on stdout). Dispatched before any + # streamlit import so worker processes start lean and PyInstaller + # re-entry is safe. + parser.add_argument("--worker", choices=["capture", "play", "simulate"], + help=argparse.SUPPRESS) args = parser.parse_args() - #_signal_register() + if args.worker: + from .services.worker_main import run_worker + sys.exit(run_worker(args.worker)) if args.convert_shdr_log: if not args.devices: parser.error("--convert-shdr-log requires --devices") @@ -43,47 +53,102 @@ def main(): ) return if args.loopback_test: + from .test import run_loopback_test run_loopback_test() elif args.mirror_test: + from .test import run_mirror_test run_mirror_test() elif args.play_test: + from .test import run_play_test run_play_test() else: - app_path = str(Path(__file__).parent / "app.py") - flag_options = { - "global.developmentMode": False, - "server.port": args.port, - "server.headless": True, - "server.showEmailPrompt": False, - "server.maxUploadSize": 5120, # 5 GB dataset uploads - "browser.gatherUsageStats": False, - "client.toolbarMode": "minimal", - # Suppress Streamlit's "You can now view your Streamlit app..." banner - # (and its URL list) so we can print our own startup message below. - "logger.hideWelcomeMessage": True, - } - streamlit.web.bootstrap.load_config_options(flag_options=flag_options) - flag_options["_is_running_with_streamlit"] = True - - # Silence urllib3's connection DEBUG logs (emitted by the external-IP - # lookup below, which fetches the public IP over HTTP). - logging.getLogger("urllib3").setLevel(logging.WARNING) - - # Print our own startup banner (Streamlit's is suppressed above), reusing - # Streamlit's URL helpers so the Network/External URLs match its output. - from streamlit import net_util - from streamlit.web.server import server_util - print(f" Open MTConnectExplorer v{__version__} in your browser:") - print(f" Local URL: {server_util.get_url('localhost')}") + _run_server(host=args.host, port=args.port) + + +def _run_server(host: str, port: int) -> None: + """Serve the Streamlit UI at / and the FastAPI backend at /api — one + process, one port, via streamlit.starlette.App with the API mounted.""" + from starlette.routing import Mount + + import streamlit.web.bootstrap + from streamlit.starlette import App + + from .api.app import create_api + from .services import auth + from .services.config import load_config, setup_logging + + config = load_config() + setup_logging(config) + + exposed = host not in _LOCAL_HOSTS + if exposed and not config.api_key: + print( + f"Error: refusing to bind '{host}' without an API key.\n" + "Set an 'api_key' in config.json before binding beyond localhost. " + "(It gates the /api endpoints; the UI itself has no authentication, " + "so only ever run this on a trusted LAN, never the internet.)", + file=sys.stderr, + ) + sys.exit(1) + + # The in-process frontend authenticates with the runtime token; external + # clients use the config.json api_key. + auth.configure(api_key=config.api_key, server_port=port) + + flag_options = { + "global.developmentMode": False, + "server.port": port, + "server.address": host, + "server.headless": True, + "server.showEmailPrompt": False, + "server.maxUploadSize": 5120, # 5 GB dataset uploads + "browser.gatherUsageStats": False, + "client.toolbarMode": "minimal", + # Suppress Streamlit's "You can now view your Streamlit app..." banner + # (and its URL list) so we can print our own startup message below. + "logger.hideWelcomeMessage": True, + } + # Load options now so the URL helpers below read the right port/address; + # App.run() re-applies the same mapping. + streamlit.web.bootstrap.load_config_options(flag_options=flag_options) + + # Silence urllib3's connection DEBUG logs (emitted by the external-IP + # lookup below, which fetches the public IP over HTTP). + logging.getLogger("urllib3").setLevel(logging.WARNING) + + # Print our own startup banner (Streamlit's is suppressed above), reusing + # Streamlit's URL helpers so the URLs match its output. Only the LAN + # (Network) URL is shown when exposed — the public-IP "External" URL is + # deliberately omitted: this app has no UI authentication and is for + # trusted-LAN use only, so advertising an internet-facing URL would invite + # exactly the deployment the warning below tells you not to do. + from streamlit import net_util + from streamlit.web.server import server_util + print(f" Open MTConnectExplorer v{__version__} in your browser:") + print(f" Local URL: {server_util.get_url('localhost')}") + if exposed: internal_ip = net_util.get_internal_ip() if internal_ip: print(f" Network URL: {server_util.get_url(internal_ip)}") - external_ip = net_util.get_external_ip() - if external_ip: - print(f" External URL: {server_util.get_url(external_ip)}") - print() + print(f" API Docs: {server_util.get_url('localhost')}/api/docs") + print() + if exposed: + print(f" WARNING: bound to {host} - reachable by other machines on the network,") + print(" and the UI has no authentication. Use only on a trusted LAN; never") + print(" expose this app to the internet.") + else: + print(f" Bound to {host} (this machine only). To reach it from other machines on") + print(" a trusted LAN, launch with --host 0.0.0.0 (requires an 'api_key' in") + print(" config.json). This app has no UI authentication; never expose it to the") + print(" internet.") + print() + + app = App( + str(Path(__file__).parent / "app.py"), + routes=[Mount("/api", app=create_api())], + ) + app.run(config=flag_options) - streamlit.web.bootstrap.run(app_path, False, [], flag_options) if __name__ == "__main__": main() diff --git a/src/pages/api.py b/src/pages/api.py new file mode 100644 index 0000000..f28a3ca --- /dev/null +++ b/src/pages/api.py @@ -0,0 +1,98 @@ +"""Frontend → backend wiring. + +The Streamlit pages talk to the backend exclusively through the lib's +``ApiClient`` against this process's own API (loopback), authenticating with +the runtime token — the single place the frontend touches ``src.services``. +Pointing the frontend at a remote backend later is a config change here, not a +refactor. + +Each scope gets one shared background reader on ``/events/stream`` (a +``ScopeFeed``) that renders backend log events into the page's log pane, +merged with frontend-generated ``[ui]`` lines (e.g. API-call failures). +""" + +import threading +from collections import deque +from datetime import datetime +from typing import Dict, List, Optional + +from mtconnect_explorer_client import ApiClient +from mtconnect_explorer_client.schemas import SessionInfo + +from src.services import auth + +_lock = threading.RLock() +_client: Optional[ApiClient] = None + + +def get_client() -> ApiClient: + """The process-wide client for this app's own API (thread-safe).""" + global _client + with _lock: + if _client is None: + _client = ApiClient( + f"http://127.0.0.1:{auth.server_port()}/api/v1", + api_key=auth.runtime_token, + ) + return _client + + +def _fmt_ts(ts: datetime) -> str: + """Local-time log prefix matching the classic '%(asctime)s' look.""" + local = ts.astimezone() + return local.strftime("%Y-%m-%d %H:%M:%S") + f",{local.microsecond // 1000:03d}" + + +class ScopeFeed: + """Background reader of /events/stream for one scope. + + Log events are rendered into ``lines`` tagged with the emitting session's + label (ids are mapped via ``update_labels``, fed by the page's session + list). The deque bounds what was an unbounded log list.""" + + def __init__(self, scope: str) -> None: + self.scope = scope + self.lines: deque = deque(maxlen=2000) + self.labels: Dict[str, str] = {} + self._thread: Optional[threading.Thread] = None + + def ensure_started(self) -> None: + with _lock: + if self._thread is None or not self._thread.is_alive(): + self._thread = threading.Thread( + target=self._run, name=f"event-feed-{self.scope}", daemon=True + ) + self._thread.start() + + def update_labels(self, sessions: List[SessionInfo]) -> None: + for sess in sessions: + self.labels[sess.id] = sess.label + + def post_ui(self, message: str, level: str = "ERROR") -> None: + """Append a frontend-generated line (API-call failures etc.).""" + self.lines.append(f"{_fmt_ts(datetime.now().astimezone())} [{level}] [ui] {message}") + + def _run(self) -> None: + try: + for event in get_client().events_stream(scope=self.scope): + if event.type != "log": + continue + who = self.labels.get(event.session, event.session or "-") + self.lines.append( + f"{_fmt_ts(event.ts)} [{event.level}] [{who}] {event.msg}" + ) + except Exception as ex: + self.post_ui(f"event stream for '{self.scope}' stopped: {ex}") + + +_feeds: Dict[str, ScopeFeed] = {} + + +def get_feed(scope: str) -> ScopeFeed: + """The process-wide event feed for a scope (reader thread lazily started).""" + with _lock: + feed = _feeds.get(scope) + if feed is None: + feed = _feeds[scope] = ScopeFeed(scope) + feed.ensure_started() + return feed diff --git a/src/pages/capture.py b/src/pages/capture.py index b2aba5f..eb764fa 100644 --- a/src/pages/capture.py +++ b/src/pages/capture.py @@ -1,107 +1,79 @@ -import logging -import re -import time -from datetime import datetime, timezone - import streamlit as st -from src.client import ( - MtConnectClientConfiguration, - MtConnectDataCollector, - MtConnectDataCollectorConfiguration, -) +from mtconnect_explorer_client import ApiError +from mtconnect_explorer_client.schemas import CaptureCreate + +from src.pages.api import get_client, get_feed from src.pages.core import ( - DATASETS_DIR, CaptureConfig, - CaptureSession, - CapturePageState, GlobalState, - agent_url, - attach_session_log, - detach_log, - drain, get_capture_state, get_global_state, get_state_lock, log_area, - running_agent_urls, - session_thread_name, ) -_log = logging.getLogger(__name__) - - -def _filename(base_url: str) -> str: - cleaned = re.sub( - r"_+", "_", - re.sub(r"[^a-zA-Z0-9._-]", "_", base_url.replace("://", "_").rstrip("/")) - ).strip("_") - return f"{cleaned}-{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H-%M-%SZ')}.jsonl" - - -def _unique_filename(base_url: str, state: CapturePageState) -> str: - """A dataset filename for this capture that doesn't collide with an existing - file or another in-progress capture.""" - active = {s.filename for s in state.sessions.values()} - name = _filename(base_url) - if not (DATASETS_DIR / name).exists() and name not in active: - return name - stem, _, ext = name.rpartition(".") - i = 2 - while True: - candidate = f"{stem}-{i}.{ext}" - if not (DATASETS_DIR / candidate).exists() and candidate not in active: - return candidate - i += 1 - - -def _capture_label(filename: str, base_url: str) -> str: - """Human-readable session label used to tag log lines and label the grid.""" - return f"{filename} | {base_url}" - - -def _sync_log_handler(state: CapturePageState) -> None: - """Detach the consolidated log handler once no captures remain. Caller holds the lock.""" - if not state.sessions and state.handler is not None: - detach_log(state.handler) - state.handler = None - - -def _remove_session(state: CapturePageState, sid: str) -> None: - """Stop a capture's collector and drop it. Caller holds the lock.""" - sess = state.sessions.get(sid) - if sess is None: +def _backend_fingerprint() -> tuple: + """What this page renders from the backend: capture sessions (id, status) + plus the running-agent quick-fill URLs.""" + client = get_client() + return ( + tuple(sorted((s.id, s.status) for s in client.capture_list())), + tuple(sorted(a.base_url for a in client.agents_list())), + ) + + +@st.fragment(run_every=1.0) +def _watch_backend_fragment() -> None: + """The API can change sessions behind this page's back — poll the backend + and trigger a full-page rerun only when what the page rendered has actually + changed, so idle pages pick up API-driven starts/stops without constant + rerun churn.""" + try: + current = _backend_fingerprint() + except ApiError: return - if sess.collector is not None: - try: - sess.collector.stop() - except Exception: - pass - state.sessions.pop(sid, None) - _sync_log_handler(state) + if current != st.session_state.get("cap_backend_fp"): + st.rerun(scope="app") -def _reap_finished(state: CapturePageState) -> None: - """Drop captures whose collector exited on its own. Caller holds the lock.""" - for sid, sess in list(state.sessions.items()): - if sess.collector is None or not sess.collector.is_alive: - state.sessions.pop(sid, None) - _sync_log_handler(state) +@st.fragment(run_every=0.5) +def _log_output_fragment() -> None: + """Live-refresh the consolidated log via a fragment (partial) rerun.""" + log_area("cap_log_area", get_feed("capture").lines) def capture_page() -> None: st.set_page_config(page_title="MTConnectExplorer", layout="wide") st.title("Capture") + client = get_client() + feed = get_feed("capture") global_state = get_global_state() state = get_capture_state() - drain(state.log_q, state.log_lines) - - # Reap captures whose collector has finished/died so the grid stays accurate. - with get_state_lock(): - _reap_finished(state) + # Auto-delete terminal sessions (finished/error) so the grid reaps + # instantly, exactly like the pre-API behavior. + sessions = [] + try: + for sess in client.capture_list(): + if sess.status in ("finished", "error"): + try: + client.capture_stop(sess.id) + except ApiError: + pass + else: + sessions.append(sess) + except ApiError as ex: + st.error(f"API error: {ex.message}") + feed.update_labels(sessions) + # Snapshot what this render is based on; the watcher fragment reruns the + # page when the backend diverges from it (e.g. API-driven start/stop). + try: + st.session_state["cap_backend_fp"] = _backend_fingerprint() + except ApiError: + pass config: CaptureConfig = state.config @@ -113,15 +85,16 @@ def capture_page() -> None: form_container = st.container() # --- Capturing --- - if state.sessions: + if sessions: with capturing_container: head_l, head_r = st.columns([3, 1]) head_l.subheader("Capturing") with head_r: if st.button("Stop All", width="stretch"): - with get_state_lock(): - for sid in list(state.sessions): - _remove_session(state, sid) + try: + client.capture_stop_all() + except ApiError as ex: + feed.post_ui(f"stop all failed: {ex.message}") st.rerun() h = st.columns([4, 3, 2, 2, 1]) @@ -130,17 +103,18 @@ def capture_page() -> None: h[2].markdown("**Mode**") h[3].markdown("**Status**") h[4].markdown("**Stop**") - for sid, sess in list(state.sessions.items()): + for sess in sessions: c = st.columns([4, 3, 2, 2, 1]) - c[0].write(sess.filename) - c[1].write(sess.base_url) + c[0].write(sess.config.filename) + c[1].write(sess.config.base_url) c[2].write(sess.config.mode) - alive = sess.collector is not None and sess.collector.is_alive - c[3].write("capturing" if alive else "stopping") + c[3].write("stopping" if sess.status == "stopping" else "capturing") with c[4]: - if st.button("Stop", key=f"cap_stop_{sid}", width="stretch"): - with get_state_lock(): - _remove_session(state, sid) + if st.button("Stop", key=f"cap_stop_{sess.id}", width="stretch"): + try: + client.capture_stop(sess.id) + except ApiError as ex: + feed.post_ui(f"stop failed: {ex.message}") st.rerun() st.divider() @@ -152,7 +126,10 @@ def capture_page() -> None: # as entered — tab, enter, or clicking Start all commit it. Currently- # running local agents (Play and Simulator sessions) are offered as # one-click quick-fills below. - playing_urls = running_agent_urls() + try: + playing_urls = [a.base_url for a in client.agents_list()] + except ApiError: + playing_urls = [] default_url = GlobalState.model_fields["base_url"].default if "cap_base_url" not in st.session_state: st.session_state["cap_base_url"] = global_state.base_url or default_url @@ -221,52 +198,30 @@ def _use_base_url(url: str) -> None: if start_clicked and str(base_url).strip(): with get_state_lock(): global_state.base_url = str(base_url) - new_config = CaptureConfig( + state.config = CaptureConfig( interval_ms=int(stream_interval_ms), mode=str(mode), ) - state.config = new_config - - DATASETS_DIR.mkdir(exist_ok=True) - filename = _unique_filename(str(base_url), state) - label = _capture_label(filename, str(base_url)) - - if state.handler is None: - state.handler = attach_session_log(state.log_q, state.sessions) - - sid = str(state.next_id) - state.next_id += 1 - try: - collector = MtConnectDataCollector( - MtConnectDataCollectorConfiguration( - configuration=MtConnectClientConfiguration( - base_url=str(base_url), - timeout=new_config.timeout, - chunk_size=new_config.chunk_size, - ), - stream_interval_ms=new_config.interval_ms, - mode=new_config.mode, - ), - thread_name=session_thread_name(label, "collector"), - ).start(str(DATASETS_DIR / filename)) - except Exception as ex: - _sync_log_handler(state) - st.error(f"Failed to start capture: {ex}") - else: - state.sessions[sid] = CaptureSession( - id=sid, - label=label, - filename=filename, - base_url=str(base_url), - config=new_config, - collector=collector, - ) - st.rerun() + try: + client.capture_start(CaptureCreate( + base_url=str(base_url), + mode=state.config.mode, + interval_ms=state.config.interval_ms, + timeout_s=state.config.timeout, + chunk_size=state.config.chunk_size, + )) + except ApiError as ex: + feed.post_ui(f"start capture failed: {ex.message}") + st.error(f"Failed to start capture: {ex.message}") + except Exception as ex: + st.error(f"Failed to start capture: {ex}") + else: + st.rerun() # --- Log Output (full width) --- st.subheader("Log Output") - log_area("cap_log_area", state.log_lines) + _log_output_fragment() - if state.sessions: - time.sleep(0.5) - st.rerun() + # Keep the page in sync with API-driven changes (change-triggered rerun — + # a quiet backend never disturbs the form). + _watch_backend_fragment() diff --git a/src/pages/core.py b/src/pages/core.py index b6ddd07..4e1e6a4 100644 --- a/src/pages/core.py +++ b/src/pages/core.py @@ -1,84 +1,21 @@ -import inspect -import logging -import os -import queue +"""Frontend view state and shared UI helpers. + +Everything here is *view* state (form defaults, tree selections, log-pane +widgets). Domain state — sessions, datasets, ports, events — lives behind the +HTTP API and is reached through the lib client (see src/pages/api.py). The one +carve-out is Explore, which keeps a direct ``MtConnectExplorer`` (an MTConnect +agent is itself a REST API; see docs/api.md). +""" + import threading -from pathlib import Path -from typing import Dict, List, Literal, Optional +from typing import List, Optional import streamlit as st from pydantic import BaseModel, ConfigDict, Field -from src.agent import AgentConfig, AgentAdapter -from src.client import ( - MtConnectClientConfiguration, - MtConnectDataCollector, - MtConnectDataCollectorConfiguration, - MtConnectExplorer, -) -from src.shdr import ShdrDataPlayer -from src.simulator import EquipmentType, EquipmentSimulator - -DATASETS_DIR = Path("datasets") -_LOG_FMT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" - - -# --------------------------------------------------------------------------- -# Logging setup — mirrors src/main.py Config class and config.json logic -# --------------------------------------------------------------------------- -class _Config(BaseModel): - log_level: Literal["debug", "info", "warning", "error", "critical"] = Field("info") - - def get_log_level_value(self) -> int: - return logging._nameToLevel[self.log_level.upper()] - - -_log_level = ( - _Config.model_validate_json(Path("config.json").read_text()).get_log_level_value() - if os.path.isfile("config.json") - else _Config().get_log_level_value() -) -logging.basicConfig(level=_log_level, format=_LOG_FMT) -logging.getLogger("src").setLevel(_log_level) - - -# --------------------------------------------------------------------------- -# Logging helpers -# --------------------------------------------------------------------------- -class _QueueHandler(logging.Handler): - def __init__(self, q: queue.Queue) -> None: - super().__init__() - self.queue = q - - def emit(self, record: logging.LogRecord) -> None: - try: - self.queue.put_nowait(self.format(record)) - except Exception: - self.handleError(record) - - -def attach_log(q: queue.Queue) -> _QueueHandler: - h = _QueueHandler(q) - h.setFormatter(logging.Formatter(_LOG_FMT)) - logging.getLogger("src").addHandler(h) - return h - - -def detach_log(h: Optional[_QueueHandler]) -> None: - if h is not None: - logging.getLogger("src").removeHandler(h) - - -# A play/capture session tags its worker/collector threads as "