From ea893529ac8554dd72066db120b54082f5b9c8a2 Mon Sep 17 00:00:00 2001 From: m-r-mccormick Date: Thu, 23 Jul 2026 17:58:29 -0400 Subject: [PATCH 1/7] updated --- .github/workflows/ci.yml | 10 +- .github/workflows/release.yml | 31 ++++-- .gitignore | 1 + LICENSE.TXT | 202 ++++++++++++++++++++++++++++++++++ generate_notices.py | 143 ++++++++++++++++++++++++ run.bat | 21 ++++ src/__version__.py | 2 +- src/client.py | 40 +++++-- src/pages/core.py | 17 ++- src/pages/explore.py | 31 +++++- src/test.py | 17 ++- 11 files changed, 483 insertions(+), 32 deletions(-) create mode 100644 LICENSE.TXT create mode 100644 generate_notices.py create mode 100644 run.bat diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e234682..d04c73c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: # Mark as a required status check on release/* branches. runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Validate __version__ matches the target release branch run: | base="${{ github.base_ref }}" @@ -62,8 +62,8 @@ jobs: build-linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.12' - name: Build with make @@ -74,8 +74,8 @@ jobs: build-windows: runs-on: windows-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.12' - name: Build with build.bat diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd67ee2..078c2a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: prerelease: ${{ steps.derive.outputs.prerelease }} should_release: ${{ steps.derive.outputs.should_release }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - id: derive @@ -96,18 +96,21 @@ jobs: env: ARCHIVE: MTConnectExplorer-v${{ needs.version.outputs.version }}-linux-x86_64.tar.gz steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.12' - name: Build with make run: make + - name: Generate third-party notices + run: venv/bin/python generate_notices.py THIRD_PARTY_NOTICES.txt - name: Package tar.gz run: | test -f dist/MTConnectExplorer chmod +x dist/MTConnectExplorer - tar -czvf "$ARCHIVE" -C dist MTConnectExplorer - - uses: actions/upload-artifact@v4 + cp THIRD_PARTY_NOTICES.txt dist/ + tar -czvf "$ARCHIVE" -C dist MTConnectExplorer THIRD_PARTY_NOTICES.txt + - uses: actions/upload-artifact@v7 with: name: linux path: ${{ env.ARCHIVE }} @@ -120,21 +123,25 @@ jobs: env: ARCHIVE: MTConnectExplorer-v${{ needs.version.outputs.version }}-win64.zip steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.12' - name: Build with build.bat shell: cmd run: build.bat ci + - name: Generate third-party notices + shell: cmd + run: venv\Scripts\python.exe generate_notices.py THIRD_PARTY_NOTICES.txt - name: Package zip shell: pwsh run: | if (-not (Test-Path dist\MTConnectExplorer.exe)) { throw "dist\MTConnectExplorer.exe not found" } - Compress-Archive -Path dist\MTConnectExplorer.exe -DestinationPath "$env:ARCHIVE" -Force - - uses: actions/upload-artifact@v4 + Copy-Item THIRD_PARTY_NOTICES.txt dist\ + Compress-Archive -Path dist\MTConnectExplorer.exe, dist\THIRD_PARTY_NOTICES.txt -DestinationPath "$env:ARCHIVE" -Force + - uses: actions/upload-artifact@v7 with: name: windows path: ${{ env.ARCHIVE }} @@ -145,8 +152,8 @@ jobs: if: needs.version.outputs.should_release == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 with: path: artifacts merge-multiple: true @@ -166,7 +173,7 @@ jobs: git push origin "$TAG" fi - name: Create release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ needs.version.outputs.tag }} name: ${{ needs.version.outputs.tag }} diff --git a/.gitignore b/.gitignore index 3ede49d..f4997ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ +CLAUDE.md .claude/ build/ datasets/ diff --git a/LICENSE.TXT b/LICENSE.TXT new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/LICENSE.TXT @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/generate_notices.py b/generate_notices.py new file mode 100644 index 0000000..c0b5aa7 --- /dev/null +++ b/generate_notices.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Generate a THIRD_PARTY_NOTICES.txt for the release bundle. + +Run with the *build* virtualenv's interpreter after dependencies are installed +(the release workflow does this), e.g.:: + + venv/bin/python generate_notices.py THIRD_PARTY_NOTICES.txt # Linux + venv\\Scripts\\python.exe generate_notices.py THIRD_PARTY_NOTICES.txt # Windows + +It reads every installed distribution's metadata + bundled license text via +``importlib.metadata`` (no extra dependency), and appends manual entries for the +non-Python components shipped in the release (the MTConnect C++ agent binary and +the vendored three.js). It is intentionally over-inclusive — it lists every +installed distribution — so nothing that PyInstaller may bundle is missed. + +Attribution obligations covered: +- Apache-2.0 / MIT / BSD: preserve each component's copyright + license text. +- certifi (MPL-2.0): weak, file-level copyleft — MPL §3.2 only requires telling + recipients where the source is, so we include the license text plus a source + URL (no need to ship certifi's source). +""" + +import importlib.metadata as im +import sys +from pathlib import Path + +_LICENSE_FILENAMES = ( + "LICENSE", "LICENSE.txt", "LICENSE.md", "LICENSE.rst", + "LICENCE", "LICENCE.txt", "COPYING", "COPYING.txt", "NOTICE", "NOTICE.txt", +) +_SEP = "=" * 80 + +# Non-Python components bundled into the release artifact. +_BUNDLED = [ + ( + "MTConnect C++ Agent (bundled 'agent' / 'agent.exe' binary)", "", + "Apache-2.0", + "Source: https://github.com/mtconnect/cppagent (Apache License 2.0).", + ), + ( + "three.js (src/static/three.min.js, src/static/OrbitControls.js)", "r137", + "MIT", + "Copyright 2010-2022 three.js Authors. " + "Source: https://github.com/mrdoob/three.js (MIT License).", + ), +] + + +def _license_id(meta) -> str: + expr = meta.get("License-Expression") # PEP 639 SPDX expression (preferred) + if expr: + return expr.strip() + classifiers = [ + c.split("::")[-1].strip() + for c in (meta.get_all("Classifier") or []) + if c.startswith("License ") + ] + if classifiers: + return "; ".join(classifiers) + lic = (meta.get("License") or "").strip() + return lic.splitlines()[0][:60] if lic else "(see text below / upstream)" + + +def _license_text(dist) -> str: + # Wheels usually ship the license under the dist-info (sometimes licenses/). + candidates = list(_LICENSE_FILENAMES) + [f"licenses/{n}" for n in _LICENSE_FILENAMES] + for name in candidates: + try: + text = dist.read_text(name) + except Exception: + text = None + if text and text.strip(): + return text.strip() + # Fall back: scan declared files for a license-named file. + for f in (dist.files or []): + if f.name.lower().startswith(("license", "licence", "copying")): + try: + text = f.read_text(encoding="utf-8") + except Exception: + try: + text = Path(f.locate()).read_text(encoding="utf-8", errors="replace") + except Exception: + text = None + if text and text.strip(): + return text.strip() + # Last resort: some packages inline the whole license in the License field. + lic = (dist.metadata.get("License") or "").strip() + return lic if len(lic) > 80 else "" + + +def _notes_for(name: str, version: str) -> str: + if name.lower() == "certifi": + return ( + "certifi is licensed under MPL-2.0 (weak, file-level copyleft). Its " + "source is available unmodified at " + f"https://github.com/certifi/python-certifi (release {version})." + ) + return "" + + +def main() -> None: + out = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("THIRD_PARTY_NOTICES.txt") + + parts = [ + "THIRD-PARTY NOTICES", + "", + "MTConnectExplorer is distributed under the Apache License 2.0. It bundles", + "the third-party components listed below, each under its own license,", + "reproduced here to satisfy their attribution requirements. Generated from", + "the pinned build environment.", + "", + _SEP, + ] + + dists = sorted(im.distributions(), key=lambda d: (d.metadata["Name"] or "").lower()) + for dist in dists: + meta = dist.metadata + name, version = meta["Name"], meta["Version"] + block = [f"{name} {version}", f"License: {_license_id(meta)}"] + note = _notes_for(name, version) + if note: + block.append(note) + text = _license_text(dist) + if text: + block.append("") + block.append(text) + parts.append("\n".join(block)) + parts.append(_SEP) + + for name, version, license_id, note in _BUNDLED: + header = f"{name} {version}".rstrip() + parts.append("\n".join([header, f"License: {license_id}", note])) + parts.append(_SEP) + + out.write_text("\n".join(parts) + "\n", encoding="utf-8") + print( + f"Wrote {out} - {len(dists)} Python distributions " + f"+ {len(_BUNDLED)} bundled components." + ) + + +if __name__ == "__main__": + main() diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..1ec45f4 --- /dev/null +++ b/run.bat @@ -0,0 +1,21 @@ +@echo off + +:: Change to the directory where this script lives +cd /d "%~dp0" + +:: Set up the venv and agent.exe (no_pause: dev.bat won't pause on success) +call dev.bat no_pause || ( + echo Error: Could not set up the environment + pause + exit /b %ERRORLEVEL% +) + +:: Launch the app straight from the venv - no dist/ build required. Any extra +:: args are forwarded to the app, e.g. run.bat --port 9000 +echo Starting MTConnectExplorer... +venv\Scripts\python.exe -m src %* + +:: Pause if run without args (e.g. double-clicked) so output stays visible. +IF "%~1"=="" ( + pause +) diff --git a/src/__version__.py b/src/__version__.py index ee3f564..199d333 100644 --- a/src/__version__.py +++ b/src/__version__.py @@ -1 +1 @@ -__version__ = "0.1.0-alpha.4" +__version__ = "0.1.0-alpha.5" diff --git a/src/client.py b/src/client.py index a790918..2ec9597 100644 --- a/src/client.py +++ b/src/client.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import http.client import logging import threading @@ -6,6 +6,7 @@ import urllib.error import urllib.parse import urllib.request +from collections import deque from collections.abc import Iterator from typing import Any, Dict, List, Literal, Optional, Tuple, Annotated import xml.etree.ElementTree as ET @@ -902,17 +903,22 @@ def __init__( configuration: MtConnectClientConfiguration, interval_ms: int = 1000, mode: Literal["stream", "sample", "current"] = "stream", - history_max: int = 100, + history_max_points: int = 100_000, ) -> None: self._configuration = configuration self._interval_ms = interval_ms self._mode = mode - self._history_max = history_max + # Absolute per-DataItem ceiling. The retained span is otherwise driven by + # set_history_window() (the largest configured plot duration), so a plot + # of that duration can fill even at a high sample rate. + self._history_max_points = history_max_points + self._history_max_age: Optional[timedelta] = None self._client = MtConnectClient(configuration) self._probe_devices: List[MtConnectDevice] = [] self._observations: Dict[str, MtConnectObservation] = {} - self._history: Dict[str, List[MtConnectObservation]] = {} + # Per-DataItem ring buffer (maxlen = history_max_points) — O(1) eviction. + self._history: Dict[str, deque] = {} self._lock = threading.Lock() self._thread: Optional[threading.Thread] = None @@ -936,7 +942,16 @@ def get_state(self) -> Tuple[List[MtConnectDevice], Dict[str, MtConnectObservati def get_history(self, data_item_id: str) -> List[MtConnectObservation]: """Return the buffered observation history for a DataItem.""" with self._lock: - return list(self._history.get(data_item_id, [])) + return list(self._history.get(data_item_id, ())) + + def set_history_window(self, max_age: Optional[timedelta]) -> None: + """Set how far back to retain samples per DataItem: everything within + ``max_age`` of that item's most recent sample is kept (still capped at + ``history_max_points``). The Explore UI sets this from the largest of the + history / pinned-plot durations so a plot of that span can fill up. Pass + ``None`` to disable age-based trimming (only the point cap applies).""" + with self._lock: + self._history_max_age = max_age def start(self) -> "MtConnectExplorer": if self._thread is not None: @@ -958,15 +973,24 @@ def stop(self) -> None: def _update_observations(self, observations: List[MtConnectObservation]) -> None: with self._lock: + max_age = self._history_max_age for obs in observations: did = obs.attributes.get("dataItemId", "") if not did: continue self._observations[did] = obs - hist = self._history.setdefault(did, []) + hist = self._history.get(did) + if hist is None: + # maxlen enforces the absolute point cap with O(1) eviction. + hist = deque(maxlen=self._history_max_points) + self._history[did] = hist hist.append(obs) - if len(hist) > self._history_max: - del hist[: -self._history_max] + # Time-based retention: drop samples older than max_age relative to + # this item's most recent sample (front of a time-ordered deque). + if max_age is not None: + cutoff = hist[-1].timestamp - max_age + while hist and hist[0].timestamp < cutoff: + hist.popleft() def _loop(self) -> None: reader = None diff --git a/src/pages/core.py b/src/pages/core.py index b6ddd07..0aeb159 100644 --- a/src/pages/core.py +++ b/src/pages/core.py @@ -2,6 +2,7 @@ import logging import os import queue +import re import threading from pathlib import Path from typing import Dict, List, Literal, Optional @@ -33,8 +34,22 @@ def get_log_level_value(self) -> int: return logging._nameToLevel[self.log_level.upper()] +def _strip_json_comments(text: str) -> str: + """Strip // line and /* */ block comments from JSON text so config.json can + be commented like JSONC. Comment sequences inside string values (e.g. a + "http://..." URL) are preserved: string literals are matched first and kept.""" + return re.sub( + r'"(?:\\.|[^"\\])*"|//[^\n]*|/\*.*?\*/', + lambda m: m.group(0) if m.group(0).startswith('"') else "", + text, + flags=re.DOTALL, + ) + + _log_level = ( - _Config.model_validate_json(Path("config.json").read_text()).get_log_level_value() + _Config.model_validate_json( + _strip_json_comments(Path("config.json").read_text()) + ).get_log_level_value() if os.path.isfile("config.json") else _Config().get_log_level_value() ) diff --git a/src/pages/explore.py b/src/pages/explore.py index 57c31b3..762dd0d 100644 --- a/src/pages/explore.py +++ b/src/pages/explore.py @@ -18,11 +18,10 @@ ) from src.pages.core import ( GlobalState, - agent_url, get_explore_state, get_global_state, - get_play_state, get_state_lock, + running_agent_urls, ) _log = logging.getLogger(__name__) @@ -113,6 +112,12 @@ _HIST_UNITS = ["seconds", "minutes", "hours", "days"] +# Cap on points drawn per chart. A high-rate DataItem over a long window can hold +# far more samples than are worth rendering (and Altair errors past ~5000 rows), +# so history is decimated to this many points for display; the X domain still +# spans the full window, so the plot looks filled. +_MAX_PLOT_POINTS = 2000 + def _fmt_local_ts(ts_str: str) -> str: """Parse a UTC ISO timestamp string (from MTConnect) and return local time.""" @@ -245,6 +250,13 @@ def _render_history( if not pairs: st.caption("No numeric data in history.") return + if len(pairs) > _MAX_PLOT_POINTS: + # Uniform decimation, always keeping the most recent sample. + stride = -(-len(pairs) // _MAX_PLOT_POINTS) # ceil division + sampled = pairs[::stride] + if sampled[-1] is not pairs[-1]: + sampled.append(pairs[-1]) + pairs = sampled ts_list, val_list = zip(*pairs) df = pd.DataFrame({"timestamp": list(ts_list), "value": list(val_list)}) # Vega-Lite converts UTC timestamps to browser local time for display, @@ -295,8 +307,9 @@ def explore_page() -> None: # selectbox only commits typed text on enter/click and discarded it on # tab, snapping back to the first option.) Currently-playing local agents # are offered as one-click quick-fills below. - play_state = get_play_state() - playing_urls = [agent_url(s.config.agent_port) for s in play_state.sessions.values()] + # Quick-fill suggestions: all currently-running local agents (Play + + # Simulator sessions), matching the Capture page. + playing_urls = running_agent_urls() default_url = GlobalState.model_fields["base_url"].default if "exp_base_url" not in st.session_state: st.session_state["exp_base_url"] = global_state.base_url or default_url @@ -552,6 +565,16 @@ def _detail_fragment() -> None: if state.running and not explorer.is_alive: st.rerun(scope="app") return + # Retain enough history to fill the largest configured plot duration + # (history or pinned-plot window); the explorer caps it at 100,000 + # samples per DataItem. (hasattr guards an explorer instance from before + # this method existed — a session started across a code reload; it takes + # full effect once Explore is stopped and restarted.) + if hasattr(explorer, "set_history_window"): + explorer.set_history_window(max( + timedelta(**{state.hist_window_unit: state.hist_window_val}), + timedelta(**{state.pin_window_unit: state.pin_window_val}), + )) _, live_obs = explorer.get_state() node = id_to_node.get(state.selected_id) if node is None: diff --git a/src/test.py b/src/test.py index 314c411..890f31d 100644 --- a/src/test.py +++ b/src/test.py @@ -2,6 +2,7 @@ import time import logging import os +import re import signal import sys import time @@ -28,8 +29,22 @@ class Config(BaseModel): def get_log_level_value(self) -> int: return logging._nameToLevel[self.log_level.upper()] + +def _strip_json_comments(text: str) -> str: + """Strip // line and /* */ block comments from JSON text so config.json can + be commented like JSONC. Comment sequences inside string values (e.g. a + "http://..." URL) are preserved: string literals are matched first and kept.""" + return re.sub( + r'"(?:\\.|[^"\\])*"|//[^\n]*|/\*.*?\*/', + lambda m: m.group(0) if m.group(0).startswith('"') else "", + text, + flags=re.DOTALL, + ) + + if os.path.isfile("config.json"): - l_level = Config.model_validate_json(Path("config.json").read_text()).get_log_level_value() + config_content = _strip_json_comments(Path("config.json").read_text()) + l_level = Config.model_validate_json(config_content).get_log_level_value() else: l_level = Config().get_log_level_value() logging.basicConfig(level=l_level, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") From 30af56c24efba44e292ad232152ebd0e80f26f36 Mon Sep 17 00:00:00 2001 From: m-r-mccormick Date: Fri, 24 Jul 2026 11:33:17 -0400 Subject: [PATCH 2/7] added initial api --- .github/workflows/ci.yml | 17 + build.bat | 18 + dev.bat | 3 +- docs/api.md | 523 +++++++++++++++++ lib/.gitignore | 3 + lib/pyproject.toml | 33 ++ lib/readme.md | 32 ++ lib/src/mtconnect_explorer_client/__init__.py | 7 + .../mtconnect_explorer_client/__version__.py | 3 + lib/src/mtconnect_explorer_client/client.py | 293 ++++++++++ lib/src/mtconnect_explorer_client/events.py | 89 +++ .../mtconnect_explorer_client/exceptions.py | 52 ++ .../schemas/__init__.py | 76 +++ .../schemas/capture.py | 39 ++ .../schemas/common.py | 72 +++ .../schemas/datasets.py | 38 ++ .../schemas/events.py | 63 +++ .../mtconnect_explorer_client/schemas/play.py | 51 ++ .../schemas/simulate.py | 229 ++++++++ .../schemas/system.py | 25 + makefile | 9 +- readme.md | 44 ++ requirements-src.txt | 6 + requirements.txt | 10 + src/api/__init__.py | 1 + src/api/app.py | 50 ++ src/api/errors.py | 25 + src/api/routers/__init__.py | 13 + src/api/routers/capture.py | 63 +++ src/api/routers/datasets.py | 100 ++++ src/api/routers/events.py | 106 ++++ src/api/routers/play.py | 63 +++ src/api/routers/simulate.py | 121 ++++ src/api/routers/system.py | 60 ++ src/api/security.py | 27 + src/main.py | 122 ++-- src/pages/api.py | 98 ++++ src/pages/capture.py | 225 +++----- src/pages/core.py | 422 ++------------ src/pages/datasets.py | 121 ++-- src/pages/explore.py | 14 +- src/pages/play.py | 385 +++++-------- src/pages/simulator.py | 534 ++++++++---------- src/services/__init__.py | 7 + src/services/auth.py | 35 ++ src/services/capture.py | 119 ++++ src/services/config.py | 63 +++ src/services/datasets.py | 250 ++++++++ src/services/errors.py | 57 ++ src/services/events.py | 220 ++++++++ src/services/hosts.py | 278 +++++++++ src/services/play.py | 154 +++++ src/services/ports.py | 117 ++++ src/services/sessions.py | 233 ++++++++ src/services/simulate.py | 191 +++++++ src/services/worker_main.py | 103 ++++ src/simulator.py | 234 ++++++-- src/test.py | 29 +- src/viz.py | 99 ++-- tests/__init__.py | 0 tests/conftest.py | 82 +++ tests/test_datasets.py | 95 ++++ tests/test_events.py | 63 +++ tests/test_ports.py | 13 + tests/test_sessions.py | 163 ++++++ tests/test_simulate_scaling.py | 123 ++++ tests/test_system.py | 36 ++ 67 files changed, 5841 insertions(+), 1208 deletions(-) create mode 100644 docs/api.md create mode 100644 lib/.gitignore create mode 100644 lib/pyproject.toml create mode 100644 lib/readme.md create mode 100644 lib/src/mtconnect_explorer_client/__init__.py create mode 100644 lib/src/mtconnect_explorer_client/__version__.py create mode 100644 lib/src/mtconnect_explorer_client/client.py create mode 100644 lib/src/mtconnect_explorer_client/events.py create mode 100644 lib/src/mtconnect_explorer_client/exceptions.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/__init__.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/capture.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/common.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/datasets.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/events.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/play.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/simulate.py create mode 100644 lib/src/mtconnect_explorer_client/schemas/system.py create mode 100644 src/api/__init__.py create mode 100644 src/api/app.py create mode 100644 src/api/errors.py create mode 100644 src/api/routers/__init__.py create mode 100644 src/api/routers/capture.py create mode 100644 src/api/routers/datasets.py create mode 100644 src/api/routers/events.py create mode 100644 src/api/routers/play.py create mode 100644 src/api/routers/simulate.py create mode 100644 src/api/routers/system.py create mode 100644 src/api/security.py create mode 100644 src/pages/api.py create mode 100644 src/services/__init__.py create mode 100644 src/services/auth.py create mode 100644 src/services/capture.py create mode 100644 src/services/config.py create mode 100644 src/services/datasets.py create mode 100644 src/services/errors.py create mode 100644 src/services/events.py create mode 100644 src/services/hosts.py create mode 100644 src/services/play.py create mode 100644 src/services/ports.py create mode 100644 src/services/sessions.py create mode 100644 src/services/simulate.py create mode 100644 src/services/worker_main.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_datasets.py create mode 100644 tests/test_events.py create mode 100644 tests/test_ports.py create mode 100644 tests/test_sessions.py create mode 100644 tests/test_simulate_scaling.py create mode 100644 tests/test_system.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d04c73c..ebbea27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,23 @@ jobs: echo "OK: __version__ '$version' matches target branch '$base'." + test: + # API/service test suite (tests/). Tests that need the bundled MTConnect + # agent binary self-skip when src/bin/agent is absent, so this job only + # builds the venv (not the agent). Not a required status check (yet). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - name: Set up venv + run: | + python -m venv venv + venv/bin/pip install -Ur requirements-src.txt + - name: Run pytest + run: venv/bin/python -m pytest tests -q + build-linux: runs-on: ubuntu-latest steps: diff --git a/build.bat b/build.bat index 2000699..a649922 100644 --- a/build.bat +++ b/build.bat @@ -9,6 +9,24 @@ call dev.bat no_pause || ( exit /b %ERRORLEVEL% ) +:: Build the client wheel and install it over the editable install so the +:: frozen binary uses (and proves) the real wheel +echo Building client wheel... +IF EXIST "lib\dist" RD /S /Q "lib\dist" +venv\Scripts\python.exe -m build --wheel lib || ( + echo Error: Could not build the client wheel + pause + exit /b %ERRORLEVEL% +) +for %%f in (lib\dist\*.whl) do ( + venv\Scripts\pip.exe install --force-reinstall "%%f" || ( + echo Error: Could not install the client wheel + pause + exit /b 1 + ) +) +echo Built client wheel + echo Building application... venv\Scripts\pyinstaller.exe installer.spec --clean || ( echo Error: Could not update venv requirements diff --git a/dev.bat b/dev.bat index 742e8ab..39367d8 100644 --- a/dev.bat +++ b/dev.bat @@ -44,7 +44,8 @@ echo Updating venv... pause exit /b %ERRORLEVEL% ) -.\venv\Scripts\pip.exe freeze > requirements.txt +:: Exclude the editable lib install - its machine-specific path doesn't belong in the lockfile +.\venv\Scripts\pip.exe freeze --exclude mtconnect-explorer-client > requirements.txt echo Updated venv echo Dev completed successfully diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..2747eca --- /dev/null +++ b/docs/api.md @@ -0,0 +1,523 @@ +# MTConnectExplorer HTTP API — v1 + +> **Status: IMPLEMENTED** (2026-07-24). This document is the normative wire +> contract; the generated OpenAPI at `/api/openapi.json` (docs at `/api/docs`) +> reflects it. The pydantic schemas live in the `mtconnect-explorer-client` +> wheel (`lib/`), which also ships the reference Python client. + +The API exposes everything the backend owns: **capture, play, and simulate +sessions; datasets; logs; ports; running agents**. It is served by the same +process as the Streamlit UI (UI at `/`, API at `/api`, interactive docs at +`/api/docs`). + +**Deliberately not in v1** (documented so they read as decisions, not gaps): + +- **Live exploration.** An MTConnect agent is already a REST API; clients + explore by calling the agent's own `probe`/`current`/`sample` endpoints + directly — exactly what the built-in Explore page does. +- SHDR-log → dataset conversion endpoint (CLI only for now). +- WebSockets/SSE (NDJSON-over-HTTP streaming covers v1 needs). +- Pagination (list sizes are small; revisit if that changes). + +--- + +## 1. Conventions + +| Topic | Convention | +|---|---| +| Base path | `/api/v1` | +| Auth | `X-API-Key: ` header on every request (see §2). | +| Request/response bodies | JSON (pydantic models end-to-end; OpenAPI generated from them). | +| Streaming | `application/x-ndjson` — one JSON object per line (see §8). | +| Dataset content | `application/octet-stream`, streamed. | +| Timestamps | RFC 3339 UTC with milliseconds: `2026-07-23T18:04:11.234Z`. | +| Session IDs | Opaque strings, prefixed by scope: `cap-1`, `play-2`, `sim-3`. | +| Scopes | `capture`, `play`, `simulate`. | +| Errors | Typed object inside FastAPI-native `detail` (see §3). | +| OpenAPI tags | One Swagger tag per service — `System`, `Datasets`, `Capture`, `Play`, `Simulate`, `Events` — implemented as one `APIRouter` per tag, so `/api/docs` groups endpoints cleanly by service. | +| Reference style | Follows the AMT-DTP ISE/ISCP service layout: router-per-service with `prefix` + `tags`, `summary=` on every route, docs at `/api/docs` + `/api/redoc` + `/api/openapi.json`, `title`/`version` from `__version__`, and a global unhandled-exception handler. Known deviations: resource-REST paths with ids in path segments (vs. the reference's RPC-style `/list`/`/open` with ids in bodies), and API-key auth (reference has none). | + +## 2. Authentication + +- **Static key** — `api_key` field in `config.json` (JSONC). For external + clients and any future remote UI. +- **Runtime token** — generated at startup, handed to the in-process Streamlit + frontend directly (never written to disk). Middleware accepts either key. +- Default bind is `127.0.0.1`. Exposing requires an explicit `--host`. +- **Resolved:** non-local `--host` with no `api_key` configured → the process + **refuses to start**. +- **Resolved:** `GET /api/v1/health` is the *only* unauthenticated API + endpoint (liveness probes can't send keys). +- The interactive reference pages — `/api/docs`, `/api/redoc`, + `/api/openapi.json` — are likewise served without a key (a browser can't + send headers to read documentation); Swagger's **Authorize** button carries + the key for try-it-out calls. They live outside `/api/v1` and expose no data. + +Auth failures return `401` with error code `unauthorized`. + +## 3. Error model + +Every non-2xx response carries a **typed object inside FastAPI's native +`detail` field** — harmonized with the AMT-DTP services (which use `detail` +strings) while keeping machine-readable codes: + +```json +{ + "detail": { + "code": "port_conflict", + "message": "agent port 5000 is in use by play-2", + "context": {"conflicts": [{"port": 5000, "held_by": "play-2", "kind": "agent"}]} + } +} +``` + +Route code raises plain `HTTPException(status_code, detail={...})`. A global +unhandled-exception handler (adopted from the reference services) returns +`500` with `{"detail": {"code": "internal_error", "message": "..."}}`. +Exception: FastAPI's built-in `422` validation errors keep their native format +(`detail` is a *list* of field errors) — clients should handle `detail` being +either the typed object or that list. + +| HTTP | `code` | When | +|---|---|---| +| 401 | `unauthorized` | Missing/invalid API key | +| 404 | `session_not_found` | Unknown session id (incl. already purged) | +| 404 | `dataset_not_found` | Unknown dataset name | +| 409 | `port_conflict` | Requested port(s) held by a live session (`detail.conflicts` lists each) | +| 409 | `dataset_in_use` | Dataset is being played/captured (rename/delete/overwrite/play-while-capturing) | +| 409 | `dataset_exists` | Upload/rename target already exists (overwriting is not allowed) | +| 413 | `too_large` | Upload exceeds configured cap (default 5 GiB) | +| 415 | `unsupported_type` | Upload extension/MIME is not a `.jsonl` dataset | +| 422 | `validation_error` | Pydantic validation — FastAPI-native format (`detail` is a list of field errors) | +| 422 | `invalid_dataset` | File exists but header/probe frame is malformed | +| 422 | `invalid_name` | Filename fails sanitization (§6) | + +## 4. System + +### `GET /api/v1/health` *(unauthenticated)* +```json +{"status": "ok"} +``` + +### `GET /api/v1/version` +```json +{"version": "0.1.0-alpha.5", "api": "v1"} +``` + +### `GET /api/v1/agents` + +Base URLs of local MTConnect agents run by live Play/Simulate sessions +(feeds the Explore/Capture quick-fill buttons). De-duplicated. + +```json +[ + {"base_url": "http://localhost:5000/", "sessions": ["play-2"]}, + {"base_url": "http://localhost:5001/", "sessions": ["sim-1"]} +] +``` + +### `GET /api/v1/ports?shdr_count=3` + +Used ports across all live sessions plus free suggestions for pre-filling +launch forms. + +```json +{ + "used": [5000, 7878, 7879], + "suggested": {"agent": 5001, "shdr": [7880, 7881, 7882]} +} +``` + +**Port lifecycle:** the used-set is *derived from the live session table* — +there is no separate reservation store to leak. A session holds its ports +through every non-terminal state **including `stopping`**; the moment it +reaches `finished`/`error` or is deleted, its ports are free. Reservation is +atomic inside session creation (single manager lock), so two concurrent `POST`s +cannot both claim the same port — the loser gets `409 port_conflict`. + +## 5. Sessions (common model) + +All three scopes share one lifecycle and envelope. + +```json +{ + "id": "play-2", + "scope": "play", + "label": "demo.jsonl | Mazak | localhost:5000", + "status": "running", + "created_at": "2026-07-23T18:03:59.101Z", + "started_at": "2026-07-23T18:04:11.234Z", + "finished_at": null, + "error": null, + "ports": {"agent": 5000, "shdr": [7878]}, + "config": { "…scope-specific config echo…": "…" } +} +``` + +The `config` echo is **typed per scope** (`CaptureSessionInfo` / +`PlaySessionInfo` / `SimulateSessionInfo` subclass the common `SessionInfo`), +so OpenAPI and the client library expose the exact create-body shape back. + +**Status machine:** `starting → running → (stopping →) finished | error` + +- `starting` — worker launched; agent booting / connecting to SHDR adapter(s). +- `running` — data flowing (Play's `started_event`; Simulate's `start_epoch` set; + Capture's `is_collecting`). "Looping" is not a status — clients derive it + from `config.loop == true && status == running`. +- `error` — worker died; `error` field carries the message (today this only + reaches logs). + +**Lifecycle endpoints (per scope; `{s}` = `capture` | `play` | `simulate`):** + +| Endpoint | Behavior | +|---|---| +| `POST /api/v1/{s}/sessions` | Validate (ports, dataset, …) atomically → `201` + session envelope, or `409`/`404`/`422`. | +| `GET /api/v1/{s}/sessions` | List (includes terminal sessions not yet purged). | +| `GET /api/v1/{s}/sessions/{id}` | Single session. `404` after purge/delete. | +| `DELETE /api/v1/{s}/sessions/{id}` | Stop (signal, join with timeout) and remove → `204`. Idempotent-ish: `404` if unknown. | +| `DELETE /api/v1/{s}/sessions` | Stop-all for the scope → `204`. | + +**Resolved — terminal retention:** the *primary* completion signal is the +`session` status event on the event bus (§8) — streaming and cursor-polling +clients are notified without relying on retention at all. Terminal sessions +additionally **remain listed** until `DELETE` or auto-purge after **15 +minutes** as a REST-ergonomics safety net, so plain request/response clients +(curl, scripts) can still query the outcome. The frontend auto-`DELETE`s +terminal sessions to preserve today's instant-reap look. + +## 6. Datasets + +Names must end in `.jsonl` and pass sanitization: no path separators, no +`< > : " / \ | ? *`, no leading dots; the resolved path must remain inside the +datasets directory (traversal guard). Violations → `422 invalid_name`. + +### `GET /api/v1/datasets` +```json +[ + { + "name": "demo.jsonl", + "size_bytes": 73400320, + "modified": "2026-07-23T14:02:11.000Z", + "in_use": ["play"] + } +] +``` +`in_use` ⊆ `["play", "capture"]` — empty when idle. (This replaces today's +cross-page `_dataset_status()` reach-around with one authoritative answer.) + +### `POST /api/v1/datasets/upload` + +Multipart upload (`multipart/form-data`, single `file` field; the dataset name +comes from the part's filename) — matching the AMT-DTP ISE/ISCP service +convention. The server streams the part to a temp file in 1 MB chunks, +validates the name (rules above; extension must be `.jsonl`), then +**atomically moves** it into `datasets/` — a partially-uploaded file is never +visible to Play/Capture. `201` with the new dataset's metadata (the same shape +as a list entry) on success. + +**Overwriting is not allowed** (deliberate deviation from the reference, which +replaces existing targets silently — datasets are irreplaceable recordings): +uploading onto an existing name is `409 dataset_exists`; to replace a dataset, +delete it first (itself guarded by `dataset_in_use`). + +Errors: `415 unsupported_type` (extension/MIME is not a `.jsonl` upload), +`409 dataset_exists`, `409 dataset_in_use`, `413 too_large`, +`422 invalid_name`. (If a request model is ever needed alongside the file, use +the reference's `Json[Model]` form field + `openapi_extra` multipart-schema +helper.) + +### `GET /api/v1/datasets/{name}` → metadata (same shape as list entry) +### `GET /api/v1/datasets/{name}/content` → streamed download +### `PATCH /api/v1/datasets/{name}` +```json +{"new_name": "renamed.jsonl"} +``` +`204`; errors `409 dataset_in_use`, `409 dataset_exists`, `422 invalid_name`. + +### `DELETE /api/v1/datasets/{name}` → `204`; `409 dataset_in_use` + +### `GET /api/v1/datasets/{name}/devices` + +Devices from the dataset's probe frame (for the Play launch form). + +```json +{"devices": [{"id": "d1", "name": "Mazak", "uuid": "m80-uuid"}, {"id": "agent-id", "name": "Agent", "uuid": "…"}]} +``` + +Returns **all** devices; filtering out `Agent` stays a frontend concern. +`422 invalid_dataset` when the header or first frame isn't a valid probe. + +## 7. Scope-specific configs + +### Capture — `POST /api/v1/capture/sessions` + +```json +{ + "base_url": "https://demo.mtconnect.org/", + "mode": "stream", + "interval_ms": 1000, + "timeout_s": 30.0, + "chunk_size": 4096, + "filename": null +} +``` + +- `mode`: `stream` | `sample` | `current`. +- `filename: null` → **server generates it** (same scheme as today: + sanitized base_url + UTC timestamp, collision-checked against files *and* + in-flight captures). Echoed in the session envelope's `config.filename`. + +### Play — `POST /api/v1/play/sessions` + +```json +{ + "dataset": "demo.jsonl", + "devices": [ + {"device_name": "Mazak", "shdr_host": "127.0.0.1", "shdr_port": 7878} + ], + "agent": { + "port": 5000, + "mtconnect_version": "2.7", + "asset_buffer_size": 1024, + "buffer_size": 131072 + }, + "re_timestamp": true, + "loop": true +} +``` + +Validation (all-or-nothing, under the manager lock): dataset exists +(`404`), dataset not being captured (`409 dataset_in_use`), ≥1 device +(`422`), all ports free and mutually distinct (`409 port_conflict` with the +full conflict list, mirroring today's inline messages). + +### Simulate + +Session **creation is one endpoint per equipment type** — each with a +dedicated pydantic body model so every equipment parameter set is fully +validated by its own schema (amends decision #10; see decision log #12): + +| Endpoint | Body model | +|---|---| +| `POST /api/v1/simulate/start/mill_3axis` | `Mill3AxisCreate` | +| `POST /api/v1/simulate/start/mill_5axis_trunnion_bc` | `Mill5AxisTrunnionBcCreate` | +| `POST /api/v1/simulate/start/robot_6dof` | `Robot6DofCreate` | + +The lifecycle endpoints (`GET/DELETE /simulate/sessions[...]`, `/routine`) +are shared across types; the session envelope's `config` echo is the +discriminated union of the create models (`equipment_type` is the +discriminator — a defaulted constant, so requests never need to send it). + +**Axis limits.** Every equipment parameter is optional. Linear axes are +**meters**, rotary axes/joints are **degrees**, each a `{"min", "max"}` object +validated `max > min`. Defaults are the equipment's **native travel** (0–1 m +linear; per-axis rotary ranges), so `{}` reproduces the canned motion +identically. Configured limits **scale the routine** (affine, native → +configured, per axis), the 3D scene's geometry/workpiece, and are published as +MTConnect `Constraints` (``/``) on the axis DataItems in the +generated device model. The trunnion's C axis is continuous and has no limits. +Equipment keys are lowercase with axis orientations encoded where they matter +(`_bc`), anticipating future variants. + +#### `GET /api/v1/simulate/equipment` — catalog for the launch form + +```json +[ + { + "type": "mill_3axis", + "label": "3-Axis Mill", + "default_device_name": "Mill", + "routines": ["Default Routine"], + "create_path": "/api/v1/simulate/start/mill_3axis" + }, + {"type": "mill_5axis_trunnion_bc", "label": "5-Axis Trunnion Mill (B/C)", "default_device_name": "TrunnionMill", "routines": ["Default Routine"], "create_path": "/api/v1/simulate/start/mill_5axis_trunnion_bc"}, + {"type": "robot_6dof", "label": "6-DOF Robot", "default_device_name": "Robot", "routines": ["Default Routine"], "create_path": "/api/v1/simulate/start/robot_6dof"} +] +``` + +Each `create_path`'s request schema is that operation's body in +`/api/openapi.json` — generic clients can build launch forms without +hardcoding equipment knowledge. + +#### `POST /api/v1/simulate/start/mill_5axis_trunnion_bc` (example) + +```json +{ + "device_name": "DMU50", + "routine": "Default Routine", + "interval_ms": 10, + "agent": {"port": 5000, "mtconnect_version": "2.7", "asset_buffer_size": 1024, "buffer_size": 131072}, + "shdr": {"host": "127.0.0.1", "port": 7878}, + "x": {"min": 0.0, "max": 0.5}, + "y": {"min": 0.0, "max": 0.5}, + "z": {"min": 0.0, "max": 0.5}, + "b": {"min": -110.0, "max": 30.0} +} +``` + +Every field is optional (`{}` is a valid body). The robot's model takes +`j1`…`j6` limit objects (degrees) instead of `x`/`y`/`z`/`b`; its link lengths +are fixed physical structure, not parameters. + +#### `GET /api/v1/simulate/sessions/{id}/routine` — feeds the 3D view + +```json +{ + "spec": { "…RoutineSpec.to_dict()…": "equipment, coord_names, start, cycle_time, geometry, moves" }, + "start_epoch": 1753293851.204, + "server_time": "2026-07-23T18:04:15.732Z" +} +``` + +- `start_epoch` is `null` until the session reaches `running` (no data + timeline to phase-lock to yet — the UI keeps its current "reopen in a + moment" behavior). +- `server_time` lets the client compute a clock offset once + (`offset = Date.now() − server_time`), making the phase-lock correct even if + a future remote-UI mode ships. Costs nothing now. + +## 8. Events & logs (NDJSON) + +One **global event bus**: every event gets a **globally monotonic `seq`** and a +**`ts`** (both mandatory on *every* line, all event types). Events are held in +a bounded ring buffer; `scope` / `session` / `level` are *filters*, not +separate counters — so any filter combination works with a single cursor. + +**Default (accepted):** ring buffer capacity **10,000 events**, memory-only +(no disk persistence). This also bounds what today is an unbounded +`log_lines` list. + +### Event records + +```jsonl +{"type":"log","seq":10423,"ts":"2026-07-23T18:04:11.234Z","scope":"play","session":"play-3","level":"INFO","logger":"src.converter","msg":"playback started"} +{"type":"session","seq":10424,"ts":"2026-07-23T18:04:11.240Z","scope":"play","session":"play-3","event":"status","status":"running"} +{"type":"session","seq":10425,"ts":"2026-07-23T18:05:02.001Z","scope":"simulate","session":"sim-2","event":"status","status":"running","detail":{"start_epoch":1753293851.204}} +{"type":"ping","seq":10425,"ts":"2026-07-23T18:05:17.000Z"} +``` + +- `log` — a backend log record attributed to a session (the `\x1f` + thread-name mechanism becomes an internal detail of the ingest path). +- `session` — lifecycle transitions (`status` events on every state change), + so a streaming client never needs to poll for status. +- `ping` — heartbeat every **15 s** of stream idle time, so proxies/clients + can distinguish "quiet" from "dead". +- **Resolved — ping `seq` semantics:** pings satisfy the "seq on every + line" rule by carrying the **last delivered event's seq** (a cursor + checkpoint: "you are current through N"). They are *not* stored in the + buffer and do *not* increment the counter — otherwise backfills would replay + stale pings, and cursor gaps would be indistinguishable from missed events. +- **Integrity contract:** within a connection, TCP already guarantees ordering + and delivery — `seq` exists for *cross-reconnect* continuity (the `after` + cursor) and buffer-overrun detection (`dropped`). Because the counter is + global, a *filtered* view legitimately skips values, so "seq increments by + exactly 1" is **not** a valid gap check on filtered streams. +- Clients must ignore unknown `type`s and unknown fields (forward + compatibility). + +- **Filter semantics:** `scope` and `session` are exact matches. `level` is a + **minimum severity** (`level=INFO` includes `WARNING`/`ERROR`) and only + constrains `log` events — `session` status events always pass a level filter. +- Because the counter is global, a *filtered* poll's `next` cursor may skip + seq values that belonged to filtered-out events; this is by design (they can + never match the same filter later). + +### `GET /api/v1/events?scope=play&session=play-3&level=INFO&after=10400&limit=500` + +Poll mode. All filters optional; `after` is exclusive; `limit` default 500. + +```json +{ + "events": [ { "…": "…" } ], + "next": 10424, + "dropped": false +} +``` + +`dropped: true` signals the cursor fell off the ring buffer's tail (client +should render a gap marker rather than assume continuity). + +### `GET /api/v1/events/stream?scope=play&after=10400` + +Streaming mode: `application/x-ndjson`, chunked. Backfills everything after +the cursor from the buffer, then stays live. Same filters as poll mode. +Reconnect = re-request with your last seen `seq`. + +> **Send `Accept-Encoding: identity` on stream requests.** A gzip layer (the +> app's own middleware stack, or any proxy) buffers compressed output, holding +> lines back indefinitely on a quiet stream. The Python client library does +> this automatically; hand-rolled clients (`curl` defaults to identity) must +> not request compression here. + +`/events/stream?scope=play` **is** the per-service notification stream: +session-status events ride alongside logs on the same connection, so a client +subscribed to a scope learns of completion/errors without polling. + +*(The Streamlit frontend runs one shared background reader per scope on the +stream endpoint — logs and status events feed queues that the rerun/fragment +loops drain, exactly like today's log-queue pattern. Poll mode serves simple +request/response clients and reconnect backfill.)* + +## 9. Schemas & client library + +Shared pydantic models live in the **client library component** (`lib/` at the +repo root) — the reference pattern, where the client lib is also the schema +home. The app installs it into its venv (editable in dev; built wheel for +release builds) and the FastAPI routes, services layer, and the Streamlit +frontend's API client all import the same classes. FastAPI's OpenAPI (at +`/api/openapi.json`, docs at `/api/docs`) is generated from these and is the +contract any future frontend builds against. + +``` +lib/ + pyproject.toml # wheel: mtconnect-explorer-client (version lockstep with the app) + src/mtconnect_explorer_client/ + __init__.py # ApiClient + schema re-exports + client.py # sync httpx ApiClient — flat per-endpoint methods (reference style) + events.py # events_stream() NDJSON iterator + exceptions.py # typed exceptions mapped from error codes + schemas/ + __init__.py + common.py # ErrorDetail, SessionInfo, SessionStatus, PortsInfo + capture.py # CaptureCreate, CaptureConfig + play.py # PlayCreate, PlayDevice, AgentSettings + simulate.py # SimulateCreate, ShdrSettings, EquipmentInfo, RoutineInfo + datasets.py # DatasetInfo, RenameRequest, DeviceInfo + events.py # LogEvent, SessionEvent, PingEvent, EventPage + system.py # VersionInfo, AgentInfo +``` + +## 10. Non-normative implementation notes + +*(Internal architecture — recorded here so the API's behavior guarantees have +a stated basis; not part of the wire contract.)* + +- Routes are thin wrappers over `src/services/` managers; all session mutation + happens under the manager lock (atomic validate-and-reserve). +- Workers run as **threads in v1** behind a host seam (`ThreadHost`), written + subprocess-ready: a worker is a pure function of `(config, event_sink)`. A + later `SubprocessHost` runs the same worker via self-exec + (`MTConnectExplorer --worker `, config on stdin, events as NDJSON on + stdout) — flipping a scope between hosts changes no API behavior. +- The hot data path (SHDR emission → agent → HTTP consumers; collector → + dataset file) lives entirely inside a worker and never crosses the API. +- The Streamlit frontend authenticates with the runtime token; its API base + URL and key are injected config, so pointing it at a remote backend later is + a flag, not a refactor. + +## 11. Decision log + +| # | Item | Status | +|---|---|---| +| 1 | Refuse to start when exposed (`--host` non-local) without `api_key` | ✅ Confirmed (§2) | +| 2 | `/health` unauthenticated | ✅ Confirmed (§2) | +| 3 | Terminal-session retention | ✅ Confirmed — event bus is the primary signal; retention is a REST safety net (§5) | +| 4 | Upload style | ✅ Resolved — multipart `POST /datasets/upload` per the AMT-DTP ISE/ISCP service convention; overwriting existing datasets is not allowed (§6) | +| 5 | Event ring buffer capacity / persistence | ✅ Accepted by default — 10,000, memory-only (§8) | +| 6 | Ping `seq` = checkpoint of last event, not stored | ✅ Confirmed (delegated) (§8) | +| 7 | Heartbeat interval 15 s | ✅ Accepted by default (§8) | +| 8 | One Swagger tag per service | ✅ Confirmed (§1) | +| 9 | Error shape | ✅ Resolved — typed object inside FastAPI-native `detail` (§3) | +| 10 | URL grammar | ✅ Resolved — resource-REST paths kept; deliberate deviation from the reference's RPC style (§1) | +| 11 | Client library | ✅ Resolved — `lib/` component at repo root is the schema home + sync httpx client; wheel `mtconnect-explorer-client`, version lockstep with the app (§9) | +| 12 | Per-equipment simulate creation | ✅ Resolved (2026-07-24) — `POST /simulate/start/{equipment_type}`, one dedicated body model per type (a deliberate RPC-style amendment to #10, chosen so each equipment's parameters are validated by its own schema as simulator types multiply). Axis limits: meters/degrees, `max > min`, defaults = native travel, limits **scale** the routine + 3D scene + published MTConnect Constraints. The generic `POST /simulate/sessions` was **removed** (not deprecated — pre-release). Equipment keys lowercase w/ axis orientation suffixes (`mill_5axis_trunnion_bc`); `tick_ms` renamed `interval_ms`; all routines named "Default Routine" (§7) | diff --git a/lib/.gitignore b/lib/.gitignore new file mode 100644 index 0000000..25aacff --- /dev/null +++ b/lib/.gitignore @@ -0,0 +1,3 @@ +build/ +dist/ +*.egg-info/ diff --git a/lib/pyproject.toml b/lib/pyproject.toml new file mode 100644 index 0000000..6c76112 --- /dev/null +++ b/lib/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mtconnect-explorer-client" +description = "Python client and API schemas for the MTConnectExplorer REST API" +readme = "readme.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +dependencies = [ + "pydantic>=2", + "httpx", +] +dynamic = ["version"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.dynamic] +# Read statically from src/mtconnect_explorer_client/__version__.py — the dotted +# submodule path keeps the build from importing the package (and its deps). +version = { attr = "mtconnect_explorer_client.__version__.__version__" } diff --git a/lib/readme.md b/lib/readme.md new file mode 100644 index 0000000..446ac80 --- /dev/null +++ b/lib/readme.md @@ -0,0 +1,32 @@ +# mtconnect-explorer-client + +Python client and pydantic schemas for the [MTConnectExplorer](../readme.md) REST API. +This package is the single source of truth for the API's request/response models — the +server imports its schemas from here. + +## Install + +```bash +pip install /mtconnect_explorer_client-*.whl +``` + +(Development installs use `pip install -e ./lib` from the repo root — wired automatically +by `dev.bat` / `make` via `requirements-src.txt`.) + +## Usage + +```python +from mtconnect_explorer_client import ApiClient, LinearAxisLimits, Mill3AxisCreate + +client = ApiClient("http://127.0.0.1:8501/api/v1", api_key="") + +print(client.version()) +print(client.datasets_list()) + +sid = client.simulate_start(Mill3AxisCreate(x=LinearAxisLimits(min=0.0, max=0.5))).id +for event in client.events_stream(scope="simulate"): + print(event.seq, event.type) +``` + +The full endpoint reference lives at `/api/docs` on a running instance, and in +[docs/api.md](../docs/api.md). diff --git a/lib/src/mtconnect_explorer_client/__init__.py b/lib/src/mtconnect_explorer_client/__init__.py new file mode 100644 index 0000000..c1a59e5 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/__init__.py @@ -0,0 +1,7 @@ +from .__version__ import __version__ +from .client import ApiClient +from .exceptions import ApiError +from .schemas import * # noqa: F401,F403 — the schema package defines __all__ +from . import schemas + +__all__ = ["__version__", "ApiClient", "ApiError", "schemas", *schemas.__all__] diff --git a/lib/src/mtconnect_explorer_client/__version__.py b/lib/src/mtconnect_explorer_client/__version__.py new file mode 100644 index 0000000..6a2b567 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/__version__.py @@ -0,0 +1,3 @@ +# PEP 440 mirror of the app version in src/__version__.py ("0.1.0-alpha.5"). +# Keep the two in lockstep manually when bumping. +__version__ = "0.1.0a5" diff --git a/lib/src/mtconnect_explorer_client/client.py b/lib/src/mtconnect_explorer_client/client.py new file mode 100644 index 0000000..1412a28 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/client.py @@ -0,0 +1,293 @@ +"""Sync httpx client for the MTConnectExplorer API (docs/api.md). + +Flat, reference-style methods — one per endpoint, named ``_``. +Create methods take the endpoint's pydantic model (capture/play also accept +its fields as keyword arguments):: + + from mtconnect_explorer_client import ApiClient, Mill3AxisCreate, LinearAxisLimits + + client = ApiClient("http://127.0.0.1:8501/api/v1", api_key="...") + sid = client.simulate_start(Mill3AxisCreate(x=LinearAxisLimits(max=0.5))).id + for event in client.events_stream(scope="simulate"): + print(event) +""" + +from typing import Any, BinaryIO, Dict, Iterator, List, Optional, Union + +import httpx + +from .events import AnyEvent, iter_events +from .exceptions import ApiError +from .schemas import ( + AgentInfo, + CaptureCreate, + CaptureSessionInfo, + DatasetDevices, + DatasetInfo, + EquipmentInfo, + EventPage, + HealthInfo, + PlayCreate, + PlaySessionInfo, + PortsInfo, + RenameRequest, + RoutineInfo, + SimulateCreate, + SimulateSessionInfo, + VersionInfo, +) + +DEFAULT_BASE_URL = "http://127.0.0.1:8501/api/v1" + + +class ApiClient: + """Synchronous client for a running MTConnectExplorer instance.""" + + def __init__( + self, + base_url: str = DEFAULT_BASE_URL, + api_key: Optional[str] = None, + timeout: float = 30.0, + transport: Optional[httpx.BaseTransport] = None, + ) -> None: + headers = {"X-API-Key": api_key} if api_key else {} + self._client = httpx.Client( + base_url=base_url.rstrip("/"), + headers=headers, + timeout=timeout, + transport=transport, + ) + + # -- plumbing -------------------------------------------------------------- + def close(self) -> None: + self._client.close() + + def __enter__(self) -> "ApiClient": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + response = self._client.request(method, path, **kwargs) + if response.status_code >= 400: + raise ApiError.from_response(response) + return response + + # -- System ---------------------------------------------------------------- + def health(self) -> HealthInfo: + """GET /health (unauthenticated liveness probe).""" + return HealthInfo.model_validate(self._request("GET", "/health").json()) + + def version(self) -> VersionInfo: + """GET /version.""" + return VersionInfo.model_validate(self._request("GET", "/version").json()) + + def agents_list(self) -> List[AgentInfo]: + """GET /agents — local agents run by live Play/Simulate sessions.""" + return [AgentInfo.model_validate(a) for a in self._request("GET", "/agents").json()] + + def ports(self, shdr_count: int = 1) -> PortsInfo: + """GET /ports — used ports plus free suggestions for launch forms.""" + return PortsInfo.model_validate( + self._request("GET", "/ports", params={"shdr_count": shdr_count}).json() + ) + + # -- Datasets ---------------------------------------------------------------- + def datasets_list(self) -> List[DatasetInfo]: + """GET /datasets.""" + return [DatasetInfo.model_validate(d) for d in self._request("GET", "/datasets").json()] + + def datasets_get(self, name: str) -> DatasetInfo: + """GET /datasets/{name}.""" + return DatasetInfo.model_validate(self._request("GET", f"/datasets/{name}").json()) + + def datasets_upload(self, name: str, file: Union[bytes, BinaryIO]) -> DatasetInfo: + """POST /datasets/upload (multipart; overwriting is not allowed).""" + response = self._request( + "POST", + "/datasets/upload", + files={"file": (name, file, "application/octet-stream")}, + timeout=None, # large uploads + ) + return DatasetInfo.model_validate(response.json()) + + def datasets_download(self, name: str) -> Iterator[bytes]: + """GET /datasets/{name}/content — yields the file in chunks.""" + with self._client.stream( + "GET", f"/datasets/{name}/content", timeout=None + ) as response: + if response.status_code >= 400: + response.read() + raise ApiError.from_response(response) + yield from response.iter_bytes(chunk_size=1024 * 1024) + + def datasets_rename(self, name: str, new_name: str) -> None: + """PATCH /datasets/{name}.""" + self._request( + "PATCH", f"/datasets/{name}", + json=RenameRequest(new_name=new_name).model_dump(), + ) + + def datasets_delete(self, name: str) -> None: + """DELETE /datasets/{name}.""" + self._request("DELETE", f"/datasets/{name}") + + def datasets_devices(self, name: str) -> DatasetDevices: + """GET /datasets/{name}/devices — devices from the probe frame.""" + return DatasetDevices.model_validate( + self._request("GET", f"/datasets/{name}/devices").json() + ) + + # -- Capture ----------------------------------------------------------------- + def capture_start( + self, create: Optional[CaptureCreate] = None, **fields: Any + ) -> CaptureSessionInfo: + """POST /capture/sessions.""" + body = create if create is not None else CaptureCreate(**fields) + return CaptureSessionInfo.model_validate( + self._request("POST", "/capture/sessions", json=body.model_dump()).json() + ) + + def capture_list(self) -> List[CaptureSessionInfo]: + """GET /capture/sessions.""" + return [ + CaptureSessionInfo.model_validate(s) + for s in self._request("GET", "/capture/sessions").json() + ] + + def capture_get(self, session_id: str) -> CaptureSessionInfo: + """GET /capture/sessions/{id}.""" + return CaptureSessionInfo.model_validate( + self._request("GET", f"/capture/sessions/{session_id}").json() + ) + + def capture_stop(self, session_id: str) -> None: + """DELETE /capture/sessions/{id} — stop and remove.""" + self._request("DELETE", f"/capture/sessions/{session_id}") + + def capture_stop_all(self) -> None: + """DELETE /capture/sessions — stop-all.""" + self._request("DELETE", "/capture/sessions") + + # -- Play ---------------------------------------------------------------------- + def play_start( + self, create: Optional[PlayCreate] = None, **fields: Any + ) -> PlaySessionInfo: + """POST /play/sessions.""" + body = create if create is not None else PlayCreate(**fields) + return PlaySessionInfo.model_validate( + self._request("POST", "/play/sessions", json=body.model_dump()).json() + ) + + def play_list(self) -> List[PlaySessionInfo]: + """GET /play/sessions.""" + return [ + PlaySessionInfo.model_validate(s) + for s in self._request("GET", "/play/sessions").json() + ] + + def play_get(self, session_id: str) -> PlaySessionInfo: + """GET /play/sessions/{id}.""" + return PlaySessionInfo.model_validate( + self._request("GET", f"/play/sessions/{session_id}").json() + ) + + def play_stop(self, session_id: str) -> None: + """DELETE /play/sessions/{id} — stop and remove.""" + self._request("DELETE", f"/play/sessions/{session_id}") + + def play_stop_all(self) -> None: + """DELETE /play/sessions — stop-all.""" + self._request("DELETE", "/play/sessions") + + # -- Simulate --------------------------------------------------------------------- + def simulate_equipment(self) -> List[EquipmentInfo]: + """GET /simulate/equipment — catalog for the launch form.""" + return [ + EquipmentInfo.model_validate(e) + for e in self._request("GET", "/simulate/equipment").json() + ] + + def simulate_start(self, create: SimulateCreate) -> SimulateSessionInfo: + """POST /simulate/start/{equipment_type} — dispatched from the create + model's type (Mill3AxisCreate, Mill5AxisTrunnionBcCreate, Robot6DofCreate).""" + return SimulateSessionInfo.model_validate( + self._request( + "POST", + f"/simulate/start/{create.equipment_type}", + json=create.model_dump(), + ).json() + ) + + def simulate_list(self) -> List[SimulateSessionInfo]: + """GET /simulate/sessions.""" + return [ + SimulateSessionInfo.model_validate(s) + for s in self._request("GET", "/simulate/sessions").json() + ] + + def simulate_get(self, session_id: str) -> SimulateSessionInfo: + """GET /simulate/sessions/{id}.""" + return SimulateSessionInfo.model_validate( + self._request("GET", f"/simulate/sessions/{session_id}").json() + ) + + def simulate_routine(self, session_id: str) -> RoutineInfo: + """GET /simulate/sessions/{id}/routine — feeds the phase-locked 3D view.""" + return RoutineInfo.model_validate( + self._request("GET", f"/simulate/sessions/{session_id}/routine").json() + ) + + def simulate_stop(self, session_id: str) -> None: + """DELETE /simulate/sessions/{id} — stop and remove.""" + self._request("DELETE", f"/simulate/sessions/{session_id}") + + def simulate_stop_all(self) -> None: + """DELETE /simulate/sessions — stop-all.""" + self._request("DELETE", "/simulate/sessions") + + # -- Events ----------------------------------------------------------------------- + def events( + self, + after: Optional[int] = None, + scope: Optional[str] = None, + session: Optional[str] = None, + level: Optional[str] = None, + limit: int = 500, + ) -> EventPage: + """GET /events — poll mode with an exclusive ``after`` cursor.""" + params: Dict[str, Any] = {"limit": limit} + if after is not None: + params["after"] = after + if scope is not None: + params["scope"] = scope + if session is not None: + params["session"] = session + if level is not None: + params["level"] = level + return EventPage.model_validate( + self._request("GET", "/events", params=params).json() + ) + + def events_stream( + self, + after: Optional[int] = None, + scope: Optional[str] = None, + session: Optional[str] = None, + level: Optional[str] = None, + reconnect: bool = True, + ) -> Iterator[AnyEvent]: + """GET /events/stream — NDJSON iterator with seq-based reconnect. + + Yields every event including pings (their ``seq`` is a cursor + checkpoint). Iterate forever, or pass ``reconnect=False`` to stop when + the connection drops.""" + return iter_events( + self._client, + after=after, + scope=scope, + session=session, + level=level, + reconnect=reconnect, + ) diff --git a/lib/src/mtconnect_explorer_client/events.py b/lib/src/mtconnect_explorer_client/events.py new file mode 100644 index 0000000..d6d2319 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/events.py @@ -0,0 +1,89 @@ +"""NDJSON event-stream iterator (docs/api.md §8). + +``iter_events`` consumes ``GET /events/stream``: it backfills after the cursor, +stays live, and transparently reconnects from the last seen ``seq`` (pings +count — their seq is a cursor checkpoint). Unknown event types and malformed +lines are skipped (forward compatibility). +""" + +import json +from typing import Any, Dict, Iterator, Optional, Union + +import httpx +from pydantic import TypeAdapter + +from .schemas import Event, LogEvent, PingEvent, SessionEvent + +AnyEvent = Union[LogEvent, SessionEvent, PingEvent] + +_EVENT_ADAPTER: TypeAdapter = TypeAdapter(Event) + +# The server pings every 15 s of idle time; the read timeout must sit above +# that so a quiet-but-alive stream is not treated as dead. +STREAM_READ_TIMEOUT_S = 45.0 + + +def parse_event(line: str) -> Optional[AnyEvent]: + """Parse one NDJSON line into an event model, or None if unknown/invalid.""" + line = line.strip() + if not line: + return None + try: + return _EVENT_ADAPTER.validate_json(line) + except Exception: + return None + + +def iter_events( + client: httpx.Client, + *, + after: Optional[int] = None, + scope: Optional[str] = None, + session: Optional[str] = None, + level: Optional[str] = None, + reconnect: bool = True, +) -> Iterator[AnyEvent]: + """Yield events (including pings) from /events/stream, forever. + + On a dropped connection, reconnects with ``after=`` so no + buffered event is missed. Stop by breaking out of the loop / closing the + client.""" + cursor = after + while True: + params: Dict[str, Any] = {} + if cursor is not None: + params["after"] = cursor + if scope is not None: + params["scope"] = scope + if session is not None: + params["session"] = session + if level is not None: + params["level"] = level + try: + with client.stream( + "GET", + "/events/stream", + params=params, + # identity: a gzip middleware/proxy would buffer the compressed + # stream, holding lines back indefinitely on a quiet feed. + headers={"Accept-Encoding": "identity"}, + timeout=httpx.Timeout(10.0, read=STREAM_READ_TIMEOUT_S), + ) as response: + if response.status_code >= 400: + response.read() + from .exceptions import ApiError + + raise ApiError.from_response(response) + for line in response.iter_lines(): + event = parse_event(line) + if event is None: + continue + cursor = event.seq + yield event + except (httpx.TransportError, httpx.TimeoutException): + if not reconnect: + return + continue + # Server closed the stream cleanly; reconnect from the cursor. + if not reconnect: + return diff --git a/lib/src/mtconnect_explorer_client/exceptions.py b/lib/src/mtconnect_explorer_client/exceptions.py new file mode 100644 index 0000000..02412e0 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/exceptions.py @@ -0,0 +1,52 @@ +"""Typed API errors raised by the MTConnectExplorer client (docs/api.md §3).""" + +from typing import Any, Dict, Optional + +import httpx + + +class ApiError(Exception): + """A non-2xx API response, carrying the typed error detail. + + ``code``/``message``/``context`` mirror the server's + ``{"detail": {"code", "message", "context"}}`` object. FastAPI-native 422 + validation errors (where ``detail`` is a list of field errors) are mapped + to code ``validation_error`` with the list in ``context["errors"]``. + """ + + def __init__( + self, + status_code: int, + code: str, + message: str, + context: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__(f"[{status_code} {code}] {message}") + self.status_code = status_code + self.code = code + self.message = message + self.context = context + + @classmethod + def from_response(cls, response: httpx.Response) -> "ApiError": + status = response.status_code + try: + detail = response.json().get("detail") + except Exception: + detail = None + if isinstance(detail, dict) and "code" in detail: + return cls( + status, + str(detail.get("code")), + str(detail.get("message", "")), + detail.get("context"), + ) + if isinstance(detail, list): # FastAPI-native 422 validation format + first = detail[0] if detail else {} + return cls( + status, + "validation_error", + str(first.get("msg", "request validation failed")), + {"errors": detail}, + ) + return cls(status, "http_error", response.text[:500] or f"HTTP {status}") diff --git a/lib/src/mtconnect_explorer_client/schemas/__init__.py b/lib/src/mtconnect_explorer_client/schemas/__init__.py new file mode 100644 index 0000000..22f8bf3 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/__init__.py @@ -0,0 +1,76 @@ +"""Pydantic request/response models for the MTConnectExplorer API (docs/api.md §9). + +This package is the single source of truth for the wire contract: the server's +services layer, the FastAPI routes, and the client all import these classes. +""" + +from .common import ( + TERMINAL_STATUSES, + ErrorDetail, + PortsInfo, + PortSuggestions, + Scope, + SessionInfo, + SessionPorts, + SessionStatus, +) +from .capture import CaptureCreate, CaptureSessionInfo +from .play import AgentSettings, PlayCreate, PlayDevice, PlaySessionInfo +from .simulate import ( + CREATE_MODELS, + EquipmentInfo, + LinearAxisLimits, + Mill3AxisCreate, + Mill5AxisTrunnionBcCreate, + Robot6DofCreate, + RotaryAxisLimits, + RoutineInfo, + ShdrSettings, + SimulateCreate, + SimulateSessionBase, + SimulateSessionInfo, +) +from .datasets import DatasetDevices, DatasetInfo, DeviceInfo, RenameRequest +from .events import Event, EventPage, LogEvent, PingEvent, SessionEvent +from .system import AgentInfo, HealthInfo, VersionInfo + +__all__ = [ + "TERMINAL_STATUSES", + "ErrorDetail", + "PortsInfo", + "PortSuggestions", + "Scope", + "SessionInfo", + "SessionPorts", + "SessionStatus", + "CaptureCreate", + "CaptureSessionInfo", + "AgentSettings", + "PlayCreate", + "PlayDevice", + "PlaySessionInfo", + "CREATE_MODELS", + "EquipmentInfo", + "LinearAxisLimits", + "Mill3AxisCreate", + "Mill5AxisTrunnionBcCreate", + "Robot6DofCreate", + "RotaryAxisLimits", + "RoutineInfo", + "ShdrSettings", + "SimulateCreate", + "SimulateSessionBase", + "SimulateSessionInfo", + "DatasetDevices", + "DatasetInfo", + "DeviceInfo", + "RenameRequest", + "Event", + "EventPage", + "LogEvent", + "PingEvent", + "SessionEvent", + "AgentInfo", + "HealthInfo", + "VersionInfo", +] diff --git a/lib/src/mtconnect_explorer_client/schemas/capture.py b/lib/src/mtconnect_explorer_client/schemas/capture.py new file mode 100644 index 0000000..42a2f72 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/capture.py @@ -0,0 +1,39 @@ +"""Capture-scope models (docs/api.md §7).""" + +from typing import Literal, Optional + +from pydantic import BaseModel, Field + +from .common import SessionInfo + + +class CaptureCreate(BaseModel): + """Body of POST /capture/sessions. Defaults mirror the app's client classes.""" + + base_url: str = Field( + min_length=1, + description="Base URL of the MTConnect agent to capture from.", + ) + mode: Literal["stream", "sample", "current"] = Field( + "stream", description="Data collection mode." + ) + interval_ms: int = Field( + 1000, ge=1, description="Milliseconds between agent stream buffer checks." + ) + timeout_s: float = Field( + 30.0, gt=0, description="Socket timeout in seconds for non-streaming requests." + ) + chunk_size: int = Field( + 4096, ge=1, description="Read chunk size in bytes for streaming responses." + ) + filename: Optional[str] = Field( + None, + description="Dataset filename to write. null = server generates one " + "(sanitized base_url + UTC timestamp, collision-checked).", + ) + + +class CaptureSessionInfo(SessionInfo): + """Capture session envelope with a typed config echo.""" + + config: CaptureCreate diff --git a/lib/src/mtconnect_explorer_client/schemas/common.py b/lib/src/mtconnect_explorer_client/schemas/common.py new file mode 100644 index 0000000..00e6833 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/common.py @@ -0,0 +1,72 @@ +"""Shared session/error/port models (docs/api.md §3–§5).""" + +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, Field + +# Session lifecycle: starting -> running -> (stopping ->) finished | error. +SessionStatus = Literal["starting", "running", "stopping", "finished", "error"] + +Scope = Literal["capture", "play", "simulate"] + +TERMINAL_STATUSES = ("finished", "error") + + +class ErrorDetail(BaseModel): + """Typed error object carried inside FastAPI's native ``detail`` field.""" + + code: str = Field(description="Machine-readable error code, e.g. 'port_conflict'.") + message: str = Field(description="Human-readable description of the error.") + context: Optional[Dict[str, Any]] = Field( + None, description="Optional structured context, e.g. the full port-conflict list." + ) + + +class SessionPorts(BaseModel): + """Ports held by a session (free once the session reaches a terminal state).""" + + agent: Optional[int] = Field( + None, description="MTConnect agent HTTP port, or null (capture runs no agent)." + ) + shdr: List[int] = Field( + default_factory=list, description="SHDR adapter TCP ports, one per adapter." + ) + + +class SessionInfo(BaseModel): + """Common session envelope shared by all three scopes (docs/api.md §5).""" + + id: str = Field(description="Opaque session id, prefixed by scope: cap-1, play-2, sim-3.") + scope: Scope = Field(description="Owning service: capture, play, or simulate.") + label: str = Field(description="Human-readable label used to tag log lines.") + status: SessionStatus = Field(description="Current lifecycle status.") + created_at: datetime = Field(description="When the session was created (UTC).") + started_at: Optional[datetime] = Field( + None, description="When the session reached 'running', or null." + ) + finished_at: Optional[datetime] = Field( + None, description="When the session reached a terminal status, or null." + ) + error: Optional[str] = Field( + None, description="Error message when status is 'error', else null." + ) + ports: SessionPorts = Field( + default_factory=SessionPorts, description="Ports held by this session." + ) + config: Dict[str, Any] = Field( + default_factory=dict, + description="Echo of the scope-specific create config (typed in scope subclasses).", + ) + + +class PortSuggestions(BaseModel): + agent: int = Field(description="Suggested free agent HTTP port.") + shdr: List[int] = Field(description="Suggested free SHDR ports (one per requested count).") + + +class PortsInfo(BaseModel): + """Used ports across all live sessions plus free suggestions (docs/api.md §4).""" + + used: List[int] = Field(description="Ports held by every non-terminal session.") + suggested: PortSuggestions = Field(description="Free ports for pre-filling launch forms.") diff --git a/lib/src/mtconnect_explorer_client/schemas/datasets.py b/lib/src/mtconnect_explorer_client/schemas/datasets.py new file mode 100644 index 0000000..5557e7c --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/datasets.py @@ -0,0 +1,38 @@ +"""Dataset models (docs/api.md §6).""" + +from datetime import datetime +from typing import List, Literal, Optional + +from pydantic import BaseModel, Field + + +class DatasetInfo(BaseModel): + """One dataset file in the datasets directory.""" + + name: str = Field(description="Dataset filename (always ends in .jsonl).") + size_bytes: int = Field(description="File size in bytes.") + modified: datetime = Field(description="Last-modified time (UTC).") + in_use: List[Literal["play", "capture"]] = Field( + default_factory=list, + description="Scopes currently using this dataset; empty when idle.", + ) + + +class RenameRequest(BaseModel): + """Body of PATCH /datasets/{name}.""" + + new_name: str = Field(min_length=1, description="New dataset filename (.jsonl).") + + +class DeviceInfo(BaseModel): + """One device from a dataset's probe frame.""" + + id: Optional[str] = Field(None, description="Device element id, if present.") + name: str = Field(description="Device name.") + uuid: Optional[str] = Field(None, description="Device uuid, if present.") + + +class DatasetDevices(BaseModel): + """GET /datasets/{name}/devices — all devices, including the Agent device.""" + + devices: List[DeviceInfo] = Field(description="Devices from the probe frame.") diff --git a/lib/src/mtconnect_explorer_client/schemas/events.py b/lib/src/mtconnect_explorer_client/schemas/events.py new file mode 100644 index 0000000..459eb55 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/events.py @@ -0,0 +1,63 @@ +"""Event-bus models (docs/api.md §8). + +Every line/event carries a globally monotonic ``seq`` and an RFC 3339 ``ts``. +Pings are cursor checkpoints: they carry the last delivered event's seq and are +never stored in the ring buffer. Clients must ignore unknown event types and +unknown fields (forward compatibility). +""" + +from datetime import datetime +from typing import Annotated, Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field + +from .common import Scope, SessionStatus + + +class LogEvent(BaseModel): + """A backend log record, attributed to a session when one emitted it.""" + + type: Literal["log"] = "log" + seq: int = Field(description="Globally monotonic sequence number.") + ts: datetime = Field(description="Event timestamp (UTC).") + scope: Optional[Scope] = Field(None, description="Owning scope, if session-attributed.") + session: Optional[str] = Field(None, description="Session id, if session-attributed.") + level: str = Field(description="Log level name, e.g. INFO.") + logger: str = Field(description="Logger name, e.g. src.converter.") + msg: str = Field(description="Log message.") + + +class SessionEvent(BaseModel): + """A session lifecycle transition (posted on every status change).""" + + type: Literal["session"] = "session" + seq: int = Field(description="Globally monotonic sequence number.") + ts: datetime = Field(description="Event timestamp (UTC).") + scope: Scope = Field(description="Owning scope.") + session: str = Field(description="Session id.") + event: str = Field("status", description="Event kind (currently always 'status').") + status: SessionStatus = Field(description="New session status.") + detail: Optional[Dict[str, Any]] = Field( + None, description="Extra transition data, e.g. {'start_epoch': ...} on simulate." + ) + + +class PingEvent(BaseModel): + """Stream heartbeat; ``seq`` is the last delivered event's seq (a checkpoint).""" + + type: Literal["ping"] = "ping" + seq: int = Field(description="Last delivered event's seq — 'current through N'.") + ts: datetime = Field(description="Ping timestamp (UTC).") + + +Event = Annotated[Union[LogEvent, SessionEvent, PingEvent], Field(discriminator="type")] + + +class EventPage(BaseModel): + """GET /events poll response.""" + + events: List[Event] = Field(description="Events after the cursor, oldest first.") + next: int = Field(description="Cursor for the next poll (pass as ?after=).") + dropped: bool = Field( + description="True if the cursor fell off the ring buffer's tail (gap)." + ) diff --git a/lib/src/mtconnect_explorer_client/schemas/play.py b/lib/src/mtconnect_explorer_client/schemas/play.py new file mode 100644 index 0000000..a23f032 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/play.py @@ -0,0 +1,51 @@ +"""Play-scope models (docs/api.md §7).""" + +from typing import List + +from pydantic import BaseModel, Field + +from .common import SessionInfo + + +class AgentSettings(BaseModel): + """Bundled MTConnect agent settings. Defaults mirror the app's AgentConfig.""" + + port: int = Field(5000, ge=1, le=65535, description="Agent HTTP port.") + mtconnect_version: str = Field("2.7", description="MTConnect version string.") + asset_buffer_size: int = Field(1024, ge=1, description="Agent asset buffer size.") + buffer_size: int = Field(131072, ge=1, description="Agent circular buffer size.") + + +class PlayDevice(BaseModel): + """One device selected for playback and the SHDR adapter that will feed it.""" + + device_name: str = Field(min_length=1, description="Device name from the dataset.") + shdr_host: str = Field("127.0.0.1", description="SHDR adapter host address.") + shdr_port: int = Field(7878, ge=1, le=65535, description="SHDR adapter TCP port.") + + +class PlayCreate(BaseModel): + """Body of POST /play/sessions.""" + + dataset: str = Field(min_length=1, description="Dataset filename to replay.") + devices: List[PlayDevice] = Field( + min_length=1, + description="Devices to replay; one SHDR adapter is spun up per entry.", + ) + agent: AgentSettings = Field( + default_factory=AgentSettings, description="Bundled agent settings." + ) + re_timestamp: bool = Field( + True, + description="Re-stamp observations with the current time on replay so the " + "data appears live; when false, original recorded timestamps are kept.", + ) + loop: bool = Field( + True, description="Repeat playback from the start once the dataset is exhausted." + ) + + +class PlaySessionInfo(SessionInfo): + """Play session envelope with a typed config echo.""" + + config: PlayCreate diff --git a/lib/src/mtconnect_explorer_client/schemas/simulate.py b/lib/src/mtconnect_explorer_client/schemas/simulate.py new file mode 100644 index 0000000..de80b68 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/simulate.py @@ -0,0 +1,229 @@ +"""Simulate-scope models (docs/api.md §7). + +Each equipment type has its own create model and its own endpoint +(``POST /simulate/start/{equipment_type}``), so every parameter set is fully +validated by its dedicated schema. Axis limits are **meters** for linear axes +and **degrees** for rotary axes; each limit defaults to the equipment's native +travel, so an empty body ``{}`` reproduces the canned routine identically. +Configured limits scale the routine (and the 3D scene, and the published +MTConnect Constraints) into the given range. +""" + +from datetime import datetime +from typing import Annotated, Any, Dict, List, Literal, Optional, Type, Union + +from pydantic import BaseModel, Field, model_validator + +from .common import SessionInfo +from .play import AgentSettings + + +class ShdrSettings(BaseModel): + """SHDR adapter bind settings.""" + + host: str = Field("127.0.0.1", description="SHDR adapter host address.") + port: int = Field(7878, ge=1, le=65535, description="SHDR adapter TCP port.") + + +class LinearAxisLimits(BaseModel): + """Travel limits of a linear axis, in meters.""" + + min: float = Field(0.0, description="Lower travel limit in meters.") + max: float = Field(1.0, description="Upper travel limit in meters.") + + @model_validator(mode="after") + def _ordered(self) -> "LinearAxisLimits": + if not self.max > self.min: + raise ValueError(f"max ({self.max}) must be greater than min ({self.min})") + return self + + +class RotaryAxisLimits(BaseModel): + """Rotation limits of a rotary axis or joint, in degrees.""" + + min: float = Field(-180.0, description="Lower rotation limit in degrees.") + max: float = Field(180.0, description="Upper rotation limit in degrees.") + + @model_validator(mode="after") + def _ordered(self) -> "RotaryAxisLimits": + if not self.max > self.min: + raise ValueError(f"max ({self.max}) must be greater than min ({self.min})") + return self + + +class SimulateSessionBase(BaseModel): + """Fields common to every simulate create body.""" + + interval_ms: int = Field( + 10, ge=1, le=60000, + description="Interval in milliseconds at which generated observations " + "are fed to the SHDR adapter. Lower = smoother motion, higher data rate.", + ) + agent: AgentSettings = Field( + default_factory=AgentSettings, description="Bundled MTConnect agent settings." + ) + shdr: ShdrSettings = Field( + default_factory=ShdrSettings, description="SHDR adapter bind settings." + ) + + +class Mill3AxisCreate(SimulateSessionBase): + """Body of POST /simulate/start/mill_3axis — a 3-axis vertical mill. + + Axis limit defaults are the machine's native travel (0–1 m per axis).""" + + equipment_type: Literal["mill_3axis"] = Field( + "mill_3axis", + description="Equipment type (constant; determined by the endpoint).", + ) + device_name: str = Field( + "Mill", min_length=1, description="MTConnect device name." + ) + routine: Literal["Default Routine"] = Field( + "Default Routine", + description="Routine to run: a canned facing raster + circular boring cycle.", + ) + x: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="X-axis travel limits in meters.", + ) + y: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Y-axis travel limits in meters.", + ) + z: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Z-axis travel limits in meters.", + ) + + +class Mill5AxisTrunnionBcCreate(SimulateSessionBase): + """Body of POST /simulate/start/mill_5axis_trunnion_bc — a 5-axis trunnion + mill: the head carries X/Y/Z; the table tilts about Y (B) and rotates about + Z (C). C is continuous and therefore has no limits. + + Axis limit defaults are the machine's native travel (0–1 m linear, ±120° B).""" + + equipment_type: Literal["mill_5axis_trunnion_bc"] = Field( + "mill_5axis_trunnion_bc", + description="Equipment type (constant; determined by the endpoint).", + ) + device_name: str = Field( + "TrunnionMill", min_length=1, description="MTConnect device name." + ) + routine: Literal["Default Routine"] = Field( + "Default Routine", + description="Routine to run: canned 3+2 indexed facing passes followed " + "by a simultaneous 5-axis helical contour.", + ) + x: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="X-axis travel limits in meters.", + ) + y: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Y-axis travel limits in meters.", + ) + z: LinearAxisLimits = Field( + default=LinearAxisLimits(min=0.0, max=1.0), + description="Z-axis travel limits in meters.", + ) + b: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-120.0, max=120.0), + description="Trunnion tilt about Y (B axis) limits in degrees.", + ) + + +class Robot6DofCreate(SimulateSessionBase): + """Body of POST /simulate/start/robot_6dof — a generic 6-DOF articulated + robot running a pick-and-place cycle in joint space. + + Joint limit defaults are the arm's native travel; link lengths are fixed + physical structure.""" + + equipment_type: Literal["robot_6dof"] = Field( + "robot_6dof", + description="Equipment type (constant; determined by the endpoint).", + ) + device_name: str = Field( + "Robot", min_length=1, description="MTConnect device name." + ) + routine: Literal["Default Routine"] = Field( + "Default Routine", + description="Routine to run: a canned pick-and-place cycle.", + ) + j1: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-170.0, max=170.0), + description="J1 (base yaw, about Z) limits in degrees.", + ) + j2: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-100.0, max=135.0), + description="J2 (shoulder pitch, about Y) limits in degrees.", + ) + j3: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-80.0, max=155.0), + description="J3 (elbow pitch, about Y) limits in degrees.", + ) + j4: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-180.0, max=180.0), + description="J4 (forearm roll, about Z) limits in degrees.", + ) + j5: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-120.0, max=120.0), + description="J5 (wrist pitch, about Y) limits in degrees.", + ) + j6: RotaryAxisLimits = Field( + default=RotaryAxisLimits(min=-180.0, max=180.0), + description="J6 (wrist roll, about Z) limits in degrees.", + ) + + +# The discriminated union of every create body — the session envelope's typed +# config echo, and the type the client's simulate_start() dispatches on. +SimulateCreate = Annotated[ + Union[Mill3AxisCreate, Mill5AxisTrunnionBcCreate, Robot6DofCreate], + Field(discriminator="equipment_type"), +] + +# equipment_type key -> create model (form scaffolding, client dispatch). +CREATE_MODELS: Dict[str, Type[SimulateSessionBase]] = { + "mill_3axis": Mill3AxisCreate, + "mill_5axis_trunnion_bc": Mill5AxisTrunnionBcCreate, + "robot_6dof": Robot6DofCreate, +} + + +class EquipmentInfo(BaseModel): + """One entry of the GET /simulate/equipment catalog.""" + + type: str = Field(description="Equipment type key, e.g. mill_3axis.") + label: str = Field(description="Human-readable equipment label.") + default_device_name: str = Field(description="Default MTConnect device name.") + routines: List[str] = Field(description="Available routine labels.") + create_path: str = Field( + description="Endpoint that starts a session of this type, e.g. " + "/api/v1/simulate/start/mill_3axis; its request schema is that " + "operation's body in the OpenAPI document." + ) + + +class RoutineInfo(BaseModel): + """GET /simulate/sessions/{id}/routine — feeds the phase-locked 3D view.""" + + spec: Dict[str, Any] = Field( + description="RoutineSpec.to_dict(): equipment, coord_names, start, cycle_time, " + "geometry, moves — scaled into the session's configured axis limits." + ) + start_epoch: Optional[float] = Field( + None, + description="Unix time emission began; null until the session reaches 'running'.", + ) + server_time: datetime = Field( + description="Server clock at response time, for client clock-offset correction." + ) + + +class SimulateSessionInfo(SessionInfo): + """Simulate session envelope with a typed config echo.""" + + config: SimulateCreate diff --git a/lib/src/mtconnect_explorer_client/schemas/system.py b/lib/src/mtconnect_explorer_client/schemas/system.py new file mode 100644 index 0000000..bc0b270 --- /dev/null +++ b/lib/src/mtconnect_explorer_client/schemas/system.py @@ -0,0 +1,25 @@ +"""System models (docs/api.md §4).""" + +from typing import List + +from pydantic import BaseModel, Field + + +class HealthInfo(BaseModel): + """GET /health — the only unauthenticated endpoint.""" + + status: str = Field("ok", description="Liveness indicator.") + + +class VersionInfo(BaseModel): + """GET /version.""" + + version: str = Field(description="Application version, e.g. 0.1.0-alpha.5.") + api: str = Field("v1", description="API version.") + + +class AgentInfo(BaseModel): + """One locally-running MTConnect agent (run by a live Play/Simulate session).""" + + base_url: str = Field(description="Agent base URL, e.g. http://localhost:5000/.") + sessions: List[str] = Field(description="Session ids running this agent.") diff --git a/makefile b/makefile index 9b7f9db..057880d 100644 --- a/makefile +++ b/makefile @@ -2,6 +2,11 @@ all: bin bin: build +# Build the client wheel and install it over the editable install so the +# frozen binary uses (and proves) the real wheel + rm -rf lib/dist + venv/bin/python -m build --wheel lib + venv/bin/pip install --force-reinstall lib/dist/*.whl venv/bin/pyinstaller installer.spec --clean build: src/bin/agent venv/touchfile @@ -23,8 +28,8 @@ venv/touchfile: requirements-src.txt venv/bin/pip install -Ur requirements-src.txt # Signal that venv has been updated due to pyproject.toml change touch venv/touchfile -# Update requirements.txt - venv/bin/pip freeze > requirements.txt +# Update requirements.txt (exclude the editable lib install - machine-specific path) + venv/bin/pip freeze --exclude mtconnect-explorer-client > requirements.txt src/bin/agent: # Create temporary build directory diff --git a/readme.md b/readme.md index 0271258..7662628 100644 --- a/readme.md +++ b/readme.md @@ -218,6 +218,50 @@ The `Datasets` page allows you to view, upload, download, rename, and delete dat ![Datasets Page](.github/Datasets1.png) +## HTTP API + +Everything the backend owns — capture, play, and simulate sessions; datasets; +events/logs; ports; running agents — is drivable over a REST API served by the +same process as the UI (UI at `/`, API at `/api/v1`, interactive docs at +`/api/docs`). The full contract lives in [docs/api.md](docs/api.md). + +- **Auth:** send `X-API-Key: ` on every request, where `` is the + `api_key` field in `config.json`. `GET /api/v1/health` needs no key. +- **Bind address:** the server binds `127.0.0.1` by default; pass + `--host ` to expose it. Exposing without an `api_key` configured is + refused at startup. +- **Events:** `GET /api/v1/events/stream` streams logs and session status + changes as NDJSON with cursor-based reconnect (see docs/api.md §8). + +### Configuration + +`config.json` is optional and lives in the working directory (next to the +binary, or the repo root in development). It is parsed as JSONC, so `//` and +`/* */` comments are allowed: + +```jsonc +{ + // Logging verbosity: debug, info, warning, error, or critical (default: info). + "log_level": "info", + + // API key accepted in the X-API-Key header. Required to bind a non-local + // --host; without it the API is only reachable by the built-in UI. + "api_key": "replace-with-a-long-random-secret" +} +``` + +A Python client ships as the `mtconnect-explorer-client` wheel (built from +[lib/](lib/), which is also the home of the API's pydantic schemas): + +```python +from mtconnect_explorer_client import ApiClient + +client = ApiClient("http://127.0.0.1:8501/api/v1", api_key="") +session = client.simulate_start(equipment_type="MILL_3AXIS", device_name="Mill") +for event in client.events_stream(scope="simulate"): + print(event) +``` + ## Play The `Play` page allows you to play a `.jsonl` dataset. diff --git a/requirements-src.txt b/requirements-src.txt index 1930efe..75ab5e8 100644 --- a/requirements-src.txt +++ b/requirements-src.txt @@ -4,3 +4,9 @@ pydantic psutil streamlit streamlit-tree-select + +fastapi +build +pytest + +-e ./lib diff --git a/requirements.txt b/requirements.txt index b1306e9..cccec0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,25 @@ altair==6.1.0 altgraph==0.17.5 +annotated-doc==0.0.4 annotated-types==0.7.0 anyio==4.13.0 attrs==26.1.0 blinker==1.9.0 +build==1.5.0 cachetools==7.1.2 certifi==2026.4.22 charset-normalizer==3.4.7 click==8.4.0 colorama==0.4.6 +fastapi==0.139.2 gitdb==4.0.12 GitPython==3.1.50 h11==0.16.0 +httpcore==1.0.9 httptools==0.7.1 +httpx==0.28.1 idna==3.15 +iniconfig==2.3.0 itsdangerous==2.2.0 Jinja2==3.1.6 jsonschema==4.26.0 @@ -25,14 +31,18 @@ packaging==26.2 pandas==3.0.3 pefile==2024.8.26 pillow==12.2.0 +pluggy==1.6.0 protobuf==7.34.1 psutil==7.2.2 pyarrow==24.0.0 pydantic==2.13.4 pydantic_core==2.46.4 pydeck==0.9.2 +Pygments==2.20.0 pyinstaller==6.21.0 pyinstaller-hooks-contrib==2026.6 +pyproject_hooks==1.2.0 +pytest==9.1.1 python-dateutil==2.9.0.post0 python-multipart==0.0.29 pywin32-ctypes==0.2.3 diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000..cfe0eed --- /dev/null +++ b/src/api/__init__.py @@ -0,0 +1 @@ +"""FastAPI backend (docs/api.md). Routes are thin wrappers over src/services/.""" diff --git a/src/api/app.py b/src/api/app.py new file mode 100644 index 0000000..4c3db3f --- /dev/null +++ b/src/api/app.py @@ -0,0 +1,50 @@ +"""FastAPI application factory (docs/api.md §1–§3). + +Mounted under ``/api`` in main.py, so routes live at ``/api/v1/...`` and the +interactive docs at ``/api/docs`` / ``/api/redoc`` / ``/api/openapi.json``. +""" + +import logging + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from src.__version__ import __version__ +from src.services.errors import ServiceError + +from .routers import routers + +logger = logging.getLogger(__name__) + + +def create_api() -> FastAPI: + api = FastAPI( + title="MTConnectExplorer", + version=__version__, + description=( + "REST API for MTConnectExplorer: capture, play, and simulate " + "sessions; datasets; events/logs; ports; running agents. " + "Authenticate with the `X-API-Key` header (config.json `api_key`); " + "`GET /v1/health` is the only unauthenticated endpoint." + ), + docs_url="/docs", + redoc_url="/redoc", + openapi_url="/openapi.json", + ) + + @api.exception_handler(ServiceError) + async def service_error_handler(request: Request, exc: ServiceError) -> JSONResponse: + return JSONResponse(status_code=exc.status, content={"detail": exc.detail()}) + + @api.exception_handler(Exception) + async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: + logger.error(f"unhandled API error on {request.url.path}: {exc}") + return JSONResponse( + status_code=500, + content={"detail": {"code": "internal_error", "message": str(exc)}}, + ) + + for router in routers: + api.include_router(router, prefix="/v1") + + return api diff --git a/src/api/errors.py b/src/api/errors.py new file mode 100644 index 0000000..b6a7910 --- /dev/null +++ b/src/api/errors.py @@ -0,0 +1,25 @@ +"""HTTP error helpers (docs/api.md §3): every non-2xx response carries a typed +object inside FastAPI's native ``detail`` field.""" + +from typing import Any, Dict, Optional + +from fastapi import HTTPException + +from src.services.errors import ServiceError + + +def http_error( + status: int, + code: str, + message: str, + context: Optional[Dict[str, Any]] = None, +) -> HTTPException: + detail: Dict[str, Any] = {"code": code, "message": message} + if context is not None: + detail["context"] = context + return HTTPException(status_code=status, detail=detail) + + +def from_service_error(ex: ServiceError) -> HTTPException: + """Map a services-layer error 1:1 onto its HTTP form.""" + return HTTPException(status_code=ex.status, detail=ex.detail()) diff --git a/src/api/routers/__init__.py b/src/api/routers/__init__.py new file mode 100644 index 0000000..9b8f30f --- /dev/null +++ b/src/api/routers/__init__.py @@ -0,0 +1,13 @@ +"""One APIRouter per service — one Swagger tag each (docs/api.md §1).""" + +from . import capture, datasets, events, play, simulate, system + +# Included in this order; also the tag order in /api/docs. +routers = [ + system.router, + datasets.router, + capture.router, + play.router, + simulate.router, + events.router, +] diff --git a/src/api/routers/capture.py b/src/api/routers/capture.py new file mode 100644 index 0000000..9cffbb5 --- /dev/null +++ b/src/api/routers/capture.py @@ -0,0 +1,63 @@ +"""Capture session endpoints (docs/api.md §5, §7).""" + +from typing import List + +from fastapi import APIRouter, Depends + +from mtconnect_explorer_client.schemas import CaptureCreate, CaptureSessionInfo + +from src.services.capture import capture_manager + +from ..security import require_api_key + +router = APIRouter( + prefix="/capture", + tags=["Capture"], + dependencies=[Depends(require_api_key)], +) + + +@router.post( + "/sessions", + summary="Start a capture session (validated atomically)", + response_model=CaptureSessionInfo, + status_code=201, +) +def create_session(body: CaptureCreate) -> CaptureSessionInfo: + return capture_manager.start(body).to_info(CaptureSessionInfo) + + +@router.get( + "/sessions", + summary="List capture sessions (including not-yet-purged terminal ones)", + response_model=List[CaptureSessionInfo], +) +def list_sessions() -> List[CaptureSessionInfo]: + return capture_manager.list_info() + + +@router.get( + "/sessions/{session_id}", + summary="One capture session", + response_model=CaptureSessionInfo, +) +def get_session(session_id: str) -> CaptureSessionInfo: + return capture_manager.get_info(session_id) + + +@router.delete( + "/sessions/{session_id}", + summary="Stop and remove a capture session", + status_code=204, +) +def delete_session(session_id: str) -> None: + capture_manager.delete(session_id) + + +@router.delete( + "/sessions", + summary="Stop and remove every capture session", + status_code=204, +) +def delete_all_sessions() -> None: + capture_manager.delete_all() diff --git a/src/api/routers/datasets.py b/src/api/routers/datasets.py new file mode 100644 index 0000000..acc83da --- /dev/null +++ b/src/api/routers/datasets.py @@ -0,0 +1,100 @@ +"""Dataset endpoints (docs/api.md §6).""" + +from typing import List + +from fastapi import APIRouter, Depends, UploadFile +from fastapi.responses import FileResponse + +from mtconnect_explorer_client.schemas import ( + DatasetDevices, + DatasetInfo, + RenameRequest, +) + +from src.services import datasets + +from ..errors import http_error +from ..security import require_api_key + +router = APIRouter( + prefix="/datasets", + tags=["Datasets"], + dependencies=[Depends(require_api_key)], +) + + +@router.get( + "", + summary="List all datasets (newest first) with in-use status", + response_model=List[DatasetInfo], +) +def list_datasets() -> List[DatasetInfo]: + return datasets.list_datasets() + + +@router.post( + "/upload", + summary="Upload a dataset (multipart; overwriting is not allowed)", + response_model=DatasetInfo, + status_code=201, +) +def upload(file: UploadFile) -> DatasetInfo: + name = file.filename or "" + if not name.lower().endswith(datasets.EXT): + raise http_error( + 415, "unsupported_type", + f"upload must be a {datasets.EXT} dataset (got '{name}')", + ) + # Starlette has already spooled the part to a temp file; stream it in 1 MB + # chunks to a temp file beside the datasets dir and atomically move it in. + return datasets.save_upload(name, file.file) + + +@router.get( + "/{name}", + summary="Dataset metadata", + response_model=DatasetInfo, +) +def get_dataset(name: str) -> DatasetInfo: + return datasets.get_dataset(name) + + +@router.get( + "/{name}/content", + summary="Download the dataset file (streamed)", + response_class=FileResponse, +) +def download(name: str) -> FileResponse: + path = datasets.existing_path(name) + return FileResponse( + path, + media_type="application/octet-stream", + filename=name, + ) + + +@router.get( + "/{name}/devices", + summary="Devices from the dataset's probe frame (including the Agent device)", + response_model=DatasetDevices, +) +def devices(name: str) -> DatasetDevices: + return datasets.get_devices(name) + + +@router.patch( + "/{name}", + summary="Rename a dataset", + status_code=204, +) +def rename(name: str, body: RenameRequest) -> None: + datasets.rename(name, body.new_name) + + +@router.delete( + "/{name}", + summary="Delete a dataset", + status_code=204, +) +def delete(name: str) -> None: + datasets.delete(name) diff --git a/src/api/routers/events.py b/src/api/routers/events.py new file mode 100644 index 0000000..9ca0833 --- /dev/null +++ b/src/api/routers/events.py @@ -0,0 +1,106 @@ +"""Event endpoints (docs/api.md §8): cursor polling and NDJSON streaming.""" + +import json +import queue +import time +from typing import Iterator, Optional + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import StreamingResponse + +from mtconnect_explorer_client.schemas import EventPage + +from src.services.events import BUS, rfc3339_now + +from ..security import require_api_key + +router = APIRouter( + prefix="/events", + tags=["Events"], + dependencies=[Depends(require_api_key)], +) + +PING_INTERVAL_S = 15.0 + +_SCOPE_QUERY = Query(None, description="Filter: capture | play | simulate.") +_SESSION_QUERY = Query(None, description="Filter: session id, e.g. play-3.") +_LEVEL_QUERY = Query( + None, description="Filter log events to this minimum level (e.g. INFO)." +) +_AFTER_QUERY = Query(None, ge=0, description="Exclusive seq cursor to resume from.") + + +@router.get( + "", + summary="Poll events after a seq cursor", + response_model=EventPage, +) +def poll( + after: Optional[int] = _AFTER_QUERY, + scope: Optional[str] = _SCOPE_QUERY, + session: Optional[str] = _SESSION_QUERY, + level: Optional[str] = _LEVEL_QUERY, + limit: int = Query(500, ge=1, le=10_000, description="Maximum events to return."), +) -> EventPage: + events, next_cursor, dropped = BUS.query( + after=after, scope=scope, session=session, level=level, limit=limit + ) + return EventPage(events=events, next=next_cursor, dropped=dropped) + + +def _ndjson(event: dict) -> str: + return json.dumps(event, separators=(",", ":")) + "\n" + + +def _event_stream( + after: Optional[int], + scope: Optional[str], + session: Optional[str], + level: Optional[str], +) -> Iterator[str]: + """Backfill after the cursor, then stay live. A sync generator — Starlette + runs it on the threadpool, so the blocking queue.get needs no asyncio + bridging. Pings fire every 15 s of idle time and carry the checkpoint seq + ("you are current through N"); they are never stored in the buffer.""" + backfill, _dropped, live = BUS.subscribe( + after=after, scope=scope, session=session, level=level + ) + try: + checkpoint = after or 0 + last_emit = time.monotonic() + for event in backfill: + checkpoint = event["seq"] + yield _ndjson(event) + last_emit = time.monotonic() + while True: + try: + event = live.get(timeout=1.0) + except queue.Empty: + if time.monotonic() - last_emit >= PING_INTERVAL_S: + yield _ndjson({"type": "ping", "seq": checkpoint, "ts": rfc3339_now()}) + last_emit = time.monotonic() + continue + # The live queue is unfiltered; skipped events still advance the + # checkpoint — under this filter they would never be delivered. + checkpoint = event["seq"] + if BUS._matches(event, scope, session, level): + yield _ndjson(event) + last_emit = time.monotonic() + finally: + BUS.unsubscribe(live) + + +@router.get( + "/stream", + summary="Stream events as NDJSON (backfill after cursor, then live)", +) +def stream( + after: Optional[int] = _AFTER_QUERY, + scope: Optional[str] = _SCOPE_QUERY, + session: Optional[str] = _SESSION_QUERY, + level: Optional[str] = _LEVEL_QUERY, +) -> StreamingResponse: + return StreamingResponse( + _event_stream(after, scope, session, level), + media_type="application/x-ndjson", + ) diff --git a/src/api/routers/play.py b/src/api/routers/play.py new file mode 100644 index 0000000..498c9f4 --- /dev/null +++ b/src/api/routers/play.py @@ -0,0 +1,63 @@ +"""Play session endpoints (docs/api.md §5, §7).""" + +from typing import List + +from fastapi import APIRouter, Depends + +from mtconnect_explorer_client.schemas import PlayCreate, PlaySessionInfo + +from src.services.play import play_manager + +from ..security import require_api_key + +router = APIRouter( + prefix="/play", + tags=["Play"], + dependencies=[Depends(require_api_key)], +) + + +@router.post( + "/sessions", + summary="Start a playback session (dataset + ports validated atomically)", + response_model=PlaySessionInfo, + status_code=201, +) +def create_session(body: PlayCreate) -> PlaySessionInfo: + return play_manager.start(body).to_info(PlaySessionInfo) + + +@router.get( + "/sessions", + summary="List play sessions (including not-yet-purged terminal ones)", + response_model=List[PlaySessionInfo], +) +def list_sessions() -> List[PlaySessionInfo]: + return play_manager.list_info() + + +@router.get( + "/sessions/{session_id}", + summary="One play session", + response_model=PlaySessionInfo, +) +def get_session(session_id: str) -> PlaySessionInfo: + return play_manager.get_info(session_id) + + +@router.delete( + "/sessions/{session_id}", + summary="Stop and remove a play session", + status_code=204, +) +def delete_session(session_id: str) -> None: + play_manager.delete(session_id) + + +@router.delete( + "/sessions", + summary="Stop and remove every play session", + status_code=204, +) +def delete_all_sessions() -> None: + play_manager.delete_all() diff --git a/src/api/routers/simulate.py b/src/api/routers/simulate.py new file mode 100644 index 0000000..96e2755 --- /dev/null +++ b/src/api/routers/simulate.py @@ -0,0 +1,121 @@ +"""Simulate session endpoints (docs/api.md §5, §7). + +Session creation is one endpoint per equipment type +(``POST /simulate/start/{equipment_type}``), each with a dedicated body model +so every equipment parameter set is fully validated by its own schema — the +catalog's ``create_path`` tells clients where to post. The session lifecycle +endpoints (list/get/delete/routine) are shared across types. +""" + +from datetime import datetime, timezone +from typing import List + +from fastapi import APIRouter, Depends + +from mtconnect_explorer_client.schemas import ( + EquipmentInfo, + Mill3AxisCreate, + Mill5AxisTrunnionBcCreate, + Robot6DofCreate, + RoutineInfo, + SimulateSessionInfo, +) + +from src.services.simulate import equipment_catalog, simulate_manager + +from ..security import require_api_key + +router = APIRouter( + prefix="/simulate", + tags=["Simulate"], + dependencies=[Depends(require_api_key)], +) + + +@router.get( + "/equipment", + summary="Equipment catalog: types, labels, routines, and create endpoints", + response_model=List[EquipmentInfo], +) +def equipment() -> List[EquipmentInfo]: + return equipment_catalog() + + +@router.post( + "/start/mill_3axis", + summary="Start a 3-axis mill simulation", + response_model=SimulateSessionInfo, + status_code=201, +) +def start_mill_3axis(body: Mill3AxisCreate) -> SimulateSessionInfo: + return simulate_manager.start(body).to_info(SimulateSessionInfo) + + +@router.post( + "/start/mill_5axis_trunnion_bc", + summary="Start a 5-axis trunnion mill (B/C) simulation", + response_model=SimulateSessionInfo, + status_code=201, +) +def start_mill_5axis_trunnion_bc(body: Mill5AxisTrunnionBcCreate) -> SimulateSessionInfo: + return simulate_manager.start(body).to_info(SimulateSessionInfo) + + +@router.post( + "/start/robot_6dof", + summary="Start a 6-DOF robot simulation", + response_model=SimulateSessionInfo, + status_code=201, +) +def start_robot_6dof(body: Robot6DofCreate) -> SimulateSessionInfo: + return simulate_manager.start(body).to_info(SimulateSessionInfo) + + +@router.get( + "/sessions", + summary="List simulate sessions (including not-yet-purged terminal ones)", + response_model=List[SimulateSessionInfo], +) +def list_sessions() -> List[SimulateSessionInfo]: + return simulate_manager.list_info() + + +@router.get( + "/sessions/{session_id}", + summary="One simulate session", + response_model=SimulateSessionInfo, +) +def get_session(session_id: str) -> SimulateSessionInfo: + return simulate_manager.get_info(session_id) + + +@router.get( + "/sessions/{session_id}/routine", + summary="Routine spec + start epoch for the phase-locked 3D view", + response_model=RoutineInfo, +) +def routine(session_id: str) -> RoutineInfo: + sess = simulate_manager.get(session_id) + return RoutineInfo( + spec=sess.spec.to_dict(), + start_epoch=simulate_manager.start_epoch_of(sess), + server_time=datetime.now(timezone.utc), + ) + + +@router.delete( + "/sessions/{session_id}", + summary="Stop and remove a simulate session", + status_code=204, +) +def delete_session(session_id: str) -> None: + simulate_manager.delete(session_id) + + +@router.delete( + "/sessions", + summary="Stop and remove every simulate session", + status_code=204, +) +def delete_all_sessions() -> None: + simulate_manager.delete_all() diff --git a/src/api/routers/system.py b/src/api/routers/system.py new file mode 100644 index 0000000..169dc19 --- /dev/null +++ b/src/api/routers/system.py @@ -0,0 +1,60 @@ +"""System endpoints (docs/api.md §4).""" + +from typing import List + +from fastapi import APIRouter, Depends, Query + +from mtconnect_explorer_client.schemas import ( + AgentInfo, + HealthInfo, + PortsInfo, + VersionInfo, +) + +from src.__version__ import __version__ +from src.services import ports + +from ..security import require_api_key + +router = APIRouter(tags=["System"]) + + +@router.get( + "/health", + summary="Liveness probe (the only unauthenticated endpoint)", + response_model=HealthInfo, +) +def health() -> HealthInfo: + return HealthInfo(status="ok") + + +@router.get( + "/version", + summary="Application and API version", + response_model=VersionInfo, + dependencies=[Depends(require_api_key)], +) +def version() -> VersionInfo: + return VersionInfo(version=__version__, api="v1") + + +@router.get( + "/agents", + summary="Local MTConnect agents run by live Play/Simulate sessions", + response_model=List[AgentInfo], + dependencies=[Depends(require_api_key)], +) +def agents() -> List[AgentInfo]: + return ports.running_agents() + + +@router.get( + "/ports", + summary="Used ports plus free suggestions for pre-filling launch forms", + response_model=PortsInfo, + dependencies=[Depends(require_api_key)], +) +def ports_info( + shdr_count: int = Query(1, ge=0, le=64, description="How many SHDR ports to suggest."), +) -> PortsInfo: + return ports.suggest_ports(shdr_count) diff --git a/src/api/security.py b/src/api/security.py new file mode 100644 index 0000000..edf69a6 --- /dev/null +++ b/src/api/security.py @@ -0,0 +1,27 @@ +"""X-API-Key authentication dependency (docs/api.md §2). + +Attached at router level to every router; the sole exception is +``GET /v1/health`` (liveness probes can't send keys). Using a Security +dependency (rather than middleware) also registers the scheme in OpenAPI, so +Swagger's Authorize button can set the key for try-it-out calls. +""" + +from typing import Optional + +from fastapi import Security +from fastapi.security import APIKeyHeader + +from src.services import auth + +from .errors import http_error + +_api_key_header = APIKeyHeader( + name="X-API-Key", + auto_error=False, + description="config.json api_key, or the in-process runtime token.", +) + + +def require_api_key(api_key: Optional[str] = Security(_api_key_header)) -> None: + if not api_key or api_key not in auth.valid_keys(): + raise http_error(401, "unauthorized", "missing or invalid API key") diff --git a/src/main.py b/src/main.py index 31b5529..0225ce2 100644 --- a/src/main.py +++ b/src/main.py @@ -1,13 +1,12 @@ -import os import argparse import logging -import streamlit.web.bootstrap -from typing import Literal -from pydantic import BaseModel, Field +import sys from pathlib import Path from .__version__ import __version__ -from .test import run_mirror_test, run_loopback_test, run_play_test + +# Hosts considered local for the refuse-to-start check (docs/api.md §2). +_LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"} def main(): @@ -30,9 +29,20 @@ def main(): parser.add_argument("--device", metavar="NAME", help="--convert-shdr-log: restrict output to a single device") parser.add_argument("--port", type=int, default=8501, help="Port to run the web server on (default: 8501)") + parser.add_argument("--host", default="127.0.0.1", + help="Address to bind the web server to (default: 127.0.0.1). " + "Binding a non-local address requires an api_key in config.json.") + # Hidden: self-exec worker mode (spawned by services.hosts.SubprocessHost; + # config on stdin, NDJSON events on stdout). Dispatched before any + # streamlit import so worker processes start lean and PyInstaller + # re-entry is safe. + parser.add_argument("--worker", choices=["capture", "play", "simulate"], + help=argparse.SUPPRESS) args = parser.parse_args() - #_signal_register() + if args.worker: + from .services.worker_main import run_worker + sys.exit(run_worker(args.worker)) if args.convert_shdr_log: if not args.devices: parser.error("--convert-shdr-log requires --devices") @@ -43,47 +53,91 @@ def main(): ) return if args.loopback_test: + from .test import run_loopback_test run_loopback_test() elif args.mirror_test: + from .test import run_mirror_test run_mirror_test() elif args.play_test: + from .test import run_play_test run_play_test() else: - app_path = str(Path(__file__).parent / "app.py") - flag_options = { - "global.developmentMode": False, - "server.port": args.port, - "server.headless": True, - "server.showEmailPrompt": False, - "server.maxUploadSize": 5120, # 5 GB dataset uploads - "browser.gatherUsageStats": False, - "client.toolbarMode": "minimal", - # Suppress Streamlit's "You can now view your Streamlit app..." banner - # (and its URL list) so we can print our own startup message below. - "logger.hideWelcomeMessage": True, - } - streamlit.web.bootstrap.load_config_options(flag_options=flag_options) - flag_options["_is_running_with_streamlit"] = True - - # Silence urllib3's connection DEBUG logs (emitted by the external-IP - # lookup below, which fetches the public IP over HTTP). - logging.getLogger("urllib3").setLevel(logging.WARNING) - - # Print our own startup banner (Streamlit's is suppressed above), reusing - # Streamlit's URL helpers so the Network/External URLs match its output. - from streamlit import net_util - from streamlit.web.server import server_util - print(f" Open MTConnectExplorer v{__version__} in your browser:") - print(f" Local URL: {server_util.get_url('localhost')}") + _run_server(host=args.host, port=args.port) + + +def _run_server(host: str, port: int) -> None: + """Serve the Streamlit UI at / and the FastAPI backend at /api — one + process, one port, via streamlit.starlette.App with the API mounted.""" + from starlette.routing import Mount + + import streamlit.web.bootstrap + from streamlit.starlette import App + + from .api.app import create_api + from .services import auth + from .services.config import load_config, setup_logging + + config = load_config() + setup_logging(config) + + exposed = host not in _LOCAL_HOSTS + if exposed and not config.api_key: + print( + f"Error: refusing to bind '{host}' without an API key.\n" + "Exposing the server requires an 'api_key' field in config.json " + "so the HTTP API is not open to the network unauthenticated.", + file=sys.stderr, + ) + sys.exit(1) + + # The in-process frontend authenticates with the runtime token; external + # clients use the config.json api_key. + auth.configure(api_key=config.api_key, server_port=port) + + flag_options = { + "global.developmentMode": False, + "server.port": port, + "server.address": host, + "server.headless": True, + "server.showEmailPrompt": False, + "server.maxUploadSize": 5120, # 5 GB dataset uploads + "browser.gatherUsageStats": False, + "client.toolbarMode": "minimal", + # Suppress Streamlit's "You can now view your Streamlit app..." banner + # (and its URL list) so we can print our own startup message below. + "logger.hideWelcomeMessage": True, + } + # Load options now so the URL helpers below read the right port/address; + # App.run() re-applies the same mapping. + streamlit.web.bootstrap.load_config_options(flag_options=flag_options) + + # Silence urllib3's connection DEBUG logs (emitted by the external-IP + # lookup below, which fetches the public IP over HTTP). + logging.getLogger("urllib3").setLevel(logging.WARNING) + + # Print our own startup banner (Streamlit's is suppressed above), reusing + # Streamlit's URL helpers so the URLs match its output. Network/External + # URLs are only meaningful (and only printed) when actually exposed. + from streamlit import net_util + from streamlit.web.server import server_util + print(f" Open MTConnectExplorer v{__version__} in your browser:") + print(f" Local URL: {server_util.get_url('localhost')}") + if exposed: internal_ip = net_util.get_internal_ip() if internal_ip: print(f" Network URL: {server_util.get_url(internal_ip)}") external_ip = net_util.get_external_ip() if external_ip: print(f" External URL: {server_util.get_url(external_ip)}") - print() + print(f" API Docs: {server_util.get_url('localhost')}/api/docs") + print() + + app = App( + str(Path(__file__).parent / "app.py"), + routes=[Mount("/api", app=create_api())], + ) + app.run(config=flag_options) - streamlit.web.bootstrap.run(app_path, False, [], flag_options) if __name__ == "__main__": main() diff --git a/src/pages/api.py b/src/pages/api.py new file mode 100644 index 0000000..f28a3ca --- /dev/null +++ b/src/pages/api.py @@ -0,0 +1,98 @@ +"""Frontend → backend wiring. + +The Streamlit pages talk to the backend exclusively through the lib's +``ApiClient`` against this process's own API (loopback), authenticating with +the runtime token — the single place the frontend touches ``src.services``. +Pointing the frontend at a remote backend later is a config change here, not a +refactor. + +Each scope gets one shared background reader on ``/events/stream`` (a +``ScopeFeed``) that renders backend log events into the page's log pane, +merged with frontend-generated ``[ui]`` lines (e.g. API-call failures). +""" + +import threading +from collections import deque +from datetime import datetime +from typing import Dict, List, Optional + +from mtconnect_explorer_client import ApiClient +from mtconnect_explorer_client.schemas import SessionInfo + +from src.services import auth + +_lock = threading.RLock() +_client: Optional[ApiClient] = None + + +def get_client() -> ApiClient: + """The process-wide client for this app's own API (thread-safe).""" + global _client + with _lock: + if _client is None: + _client = ApiClient( + f"http://127.0.0.1:{auth.server_port()}/api/v1", + api_key=auth.runtime_token, + ) + return _client + + +def _fmt_ts(ts: datetime) -> str: + """Local-time log prefix matching the classic '%(asctime)s' look.""" + local = ts.astimezone() + return local.strftime("%Y-%m-%d %H:%M:%S") + f",{local.microsecond // 1000:03d}" + + +class ScopeFeed: + """Background reader of /events/stream for one scope. + + Log events are rendered into ``lines`` tagged with the emitting session's + label (ids are mapped via ``update_labels``, fed by the page's session + list). The deque bounds what was an unbounded log list.""" + + def __init__(self, scope: str) -> None: + self.scope = scope + self.lines: deque = deque(maxlen=2000) + self.labels: Dict[str, str] = {} + self._thread: Optional[threading.Thread] = None + + def ensure_started(self) -> None: + with _lock: + if self._thread is None or not self._thread.is_alive(): + self._thread = threading.Thread( + target=self._run, name=f"event-feed-{self.scope}", daemon=True + ) + self._thread.start() + + def update_labels(self, sessions: List[SessionInfo]) -> None: + for sess in sessions: + self.labels[sess.id] = sess.label + + def post_ui(self, message: str, level: str = "ERROR") -> None: + """Append a frontend-generated line (API-call failures etc.).""" + self.lines.append(f"{_fmt_ts(datetime.now().astimezone())} [{level}] [ui] {message}") + + def _run(self) -> None: + try: + for event in get_client().events_stream(scope=self.scope): + if event.type != "log": + continue + who = self.labels.get(event.session, event.session or "-") + self.lines.append( + f"{_fmt_ts(event.ts)} [{event.level}] [{who}] {event.msg}" + ) + except Exception as ex: + self.post_ui(f"event stream for '{self.scope}' stopped: {ex}") + + +_feeds: Dict[str, ScopeFeed] = {} + + +def get_feed(scope: str) -> ScopeFeed: + """The process-wide event feed for a scope (reader thread lazily started).""" + with _lock: + feed = _feeds.get(scope) + if feed is None: + feed = _feeds[scope] = ScopeFeed(scope) + feed.ensure_started() + return feed diff --git a/src/pages/capture.py b/src/pages/capture.py index b2aba5f..eb764fa 100644 --- a/src/pages/capture.py +++ b/src/pages/capture.py @@ -1,107 +1,79 @@ -import logging -import re -import time -from datetime import datetime, timezone - import streamlit as st -from src.client import ( - MtConnectClientConfiguration, - MtConnectDataCollector, - MtConnectDataCollectorConfiguration, -) +from mtconnect_explorer_client import ApiError +from mtconnect_explorer_client.schemas import CaptureCreate + +from src.pages.api import get_client, get_feed from src.pages.core import ( - DATASETS_DIR, CaptureConfig, - CaptureSession, - CapturePageState, GlobalState, - agent_url, - attach_session_log, - detach_log, - drain, get_capture_state, get_global_state, get_state_lock, log_area, - running_agent_urls, - session_thread_name, ) -_log = logging.getLogger(__name__) - - -def _filename(base_url: str) -> str: - cleaned = re.sub( - r"_+", "_", - re.sub(r"[^a-zA-Z0-9._-]", "_", base_url.replace("://", "_").rstrip("/")) - ).strip("_") - return f"{cleaned}-{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H-%M-%SZ')}.jsonl" - - -def _unique_filename(base_url: str, state: CapturePageState) -> str: - """A dataset filename for this capture that doesn't collide with an existing - file or another in-progress capture.""" - active = {s.filename for s in state.sessions.values()} - name = _filename(base_url) - if not (DATASETS_DIR / name).exists() and name not in active: - return name - stem, _, ext = name.rpartition(".") - i = 2 - while True: - candidate = f"{stem}-{i}.{ext}" - if not (DATASETS_DIR / candidate).exists() and candidate not in active: - return candidate - i += 1 - - -def _capture_label(filename: str, base_url: str) -> str: - """Human-readable session label used to tag log lines and label the grid.""" - return f"{filename} | {base_url}" - - -def _sync_log_handler(state: CapturePageState) -> None: - """Detach the consolidated log handler once no captures remain. Caller holds the lock.""" - if not state.sessions and state.handler is not None: - detach_log(state.handler) - state.handler = None - - -def _remove_session(state: CapturePageState, sid: str) -> None: - """Stop a capture's collector and drop it. Caller holds the lock.""" - sess = state.sessions.get(sid) - if sess is None: +def _backend_fingerprint() -> tuple: + """What this page renders from the backend: capture sessions (id, status) + plus the running-agent quick-fill URLs.""" + client = get_client() + return ( + tuple(sorted((s.id, s.status) for s in client.capture_list())), + tuple(sorted(a.base_url for a in client.agents_list())), + ) + + +@st.fragment(run_every=1.0) +def _watch_backend_fragment() -> None: + """The API can change sessions behind this page's back — poll the backend + and trigger a full-page rerun only when what the page rendered has actually + changed, so idle pages pick up API-driven starts/stops without constant + rerun churn.""" + try: + current = _backend_fingerprint() + except ApiError: return - if sess.collector is not None: - try: - sess.collector.stop() - except Exception: - pass - state.sessions.pop(sid, None) - _sync_log_handler(state) + if current != st.session_state.get("cap_backend_fp"): + st.rerun(scope="app") -def _reap_finished(state: CapturePageState) -> None: - """Drop captures whose collector exited on its own. Caller holds the lock.""" - for sid, sess in list(state.sessions.items()): - if sess.collector is None or not sess.collector.is_alive: - state.sessions.pop(sid, None) - _sync_log_handler(state) +@st.fragment(run_every=0.5) +def _log_output_fragment() -> None: + """Live-refresh the consolidated log via a fragment (partial) rerun.""" + log_area("cap_log_area", get_feed("capture").lines) def capture_page() -> None: st.set_page_config(page_title="MTConnectExplorer", layout="wide") st.title("Capture") + client = get_client() + feed = get_feed("capture") global_state = get_global_state() state = get_capture_state() - drain(state.log_q, state.log_lines) - - # Reap captures whose collector has finished/died so the grid stays accurate. - with get_state_lock(): - _reap_finished(state) + # Auto-delete terminal sessions (finished/error) so the grid reaps + # instantly, exactly like the pre-API behavior. + sessions = [] + try: + for sess in client.capture_list(): + if sess.status in ("finished", "error"): + try: + client.capture_stop(sess.id) + except ApiError: + pass + else: + sessions.append(sess) + except ApiError as ex: + st.error(f"API error: {ex.message}") + feed.update_labels(sessions) + # Snapshot what this render is based on; the watcher fragment reruns the + # page when the backend diverges from it (e.g. API-driven start/stop). + try: + st.session_state["cap_backend_fp"] = _backend_fingerprint() + except ApiError: + pass config: CaptureConfig = state.config @@ -113,15 +85,16 @@ def capture_page() -> None: form_container = st.container() # --- Capturing --- - if state.sessions: + if sessions: with capturing_container: head_l, head_r = st.columns([3, 1]) head_l.subheader("Capturing") with head_r: if st.button("Stop All", width="stretch"): - with get_state_lock(): - for sid in list(state.sessions): - _remove_session(state, sid) + try: + client.capture_stop_all() + except ApiError as ex: + feed.post_ui(f"stop all failed: {ex.message}") st.rerun() h = st.columns([4, 3, 2, 2, 1]) @@ -130,17 +103,18 @@ def capture_page() -> None: h[2].markdown("**Mode**") h[3].markdown("**Status**") h[4].markdown("**Stop**") - for sid, sess in list(state.sessions.items()): + for sess in sessions: c = st.columns([4, 3, 2, 2, 1]) - c[0].write(sess.filename) - c[1].write(sess.base_url) + c[0].write(sess.config.filename) + c[1].write(sess.config.base_url) c[2].write(sess.config.mode) - alive = sess.collector is not None and sess.collector.is_alive - c[3].write("capturing" if alive else "stopping") + c[3].write("stopping" if sess.status == "stopping" else "capturing") with c[4]: - if st.button("Stop", key=f"cap_stop_{sid}", width="stretch"): - with get_state_lock(): - _remove_session(state, sid) + if st.button("Stop", key=f"cap_stop_{sess.id}", width="stretch"): + try: + client.capture_stop(sess.id) + except ApiError as ex: + feed.post_ui(f"stop failed: {ex.message}") st.rerun() st.divider() @@ -152,7 +126,10 @@ def capture_page() -> None: # as entered — tab, enter, or clicking Start all commit it. Currently- # running local agents (Play and Simulator sessions) are offered as # one-click quick-fills below. - playing_urls = running_agent_urls() + try: + playing_urls = [a.base_url for a in client.agents_list()] + except ApiError: + playing_urls = [] default_url = GlobalState.model_fields["base_url"].default if "cap_base_url" not in st.session_state: st.session_state["cap_base_url"] = global_state.base_url or default_url @@ -221,52 +198,30 @@ def _use_base_url(url: str) -> None: if start_clicked and str(base_url).strip(): with get_state_lock(): global_state.base_url = str(base_url) - new_config = CaptureConfig( + state.config = CaptureConfig( interval_ms=int(stream_interval_ms), mode=str(mode), ) - state.config = new_config - - DATASETS_DIR.mkdir(exist_ok=True) - filename = _unique_filename(str(base_url), state) - label = _capture_label(filename, str(base_url)) - - if state.handler is None: - state.handler = attach_session_log(state.log_q, state.sessions) - - sid = str(state.next_id) - state.next_id += 1 - try: - collector = MtConnectDataCollector( - MtConnectDataCollectorConfiguration( - configuration=MtConnectClientConfiguration( - base_url=str(base_url), - timeout=new_config.timeout, - chunk_size=new_config.chunk_size, - ), - stream_interval_ms=new_config.interval_ms, - mode=new_config.mode, - ), - thread_name=session_thread_name(label, "collector"), - ).start(str(DATASETS_DIR / filename)) - except Exception as ex: - _sync_log_handler(state) - st.error(f"Failed to start capture: {ex}") - else: - state.sessions[sid] = CaptureSession( - id=sid, - label=label, - filename=filename, - base_url=str(base_url), - config=new_config, - collector=collector, - ) - st.rerun() + try: + client.capture_start(CaptureCreate( + base_url=str(base_url), + mode=state.config.mode, + interval_ms=state.config.interval_ms, + timeout_s=state.config.timeout, + chunk_size=state.config.chunk_size, + )) + except ApiError as ex: + feed.post_ui(f"start capture failed: {ex.message}") + st.error(f"Failed to start capture: {ex.message}") + except Exception as ex: + st.error(f"Failed to start capture: {ex}") + else: + st.rerun() # --- Log Output (full width) --- st.subheader("Log Output") - log_area("cap_log_area", state.log_lines) + _log_output_fragment() - if state.sessions: - time.sleep(0.5) - st.rerun() + # Keep the page in sync with API-driven changes (change-triggered rerun — + # a quiet backend never disturbs the form). + _watch_backend_fragment() diff --git a/src/pages/core.py b/src/pages/core.py index 0aeb159..14715a9 100644 --- a/src/pages/core.py +++ b/src/pages/core.py @@ -1,99 +1,21 @@ -import inspect -import logging -import os -import queue -import re +"""Frontend view state and shared UI helpers. + +Everything here is *view* state (form defaults, tree selections, log-pane +widgets). Domain state — sessions, datasets, ports, events — lives behind the +HTTP API and is reached through the lib client (see src/pages/api.py). The one +carve-out is Explore, which keeps a direct ``MtConnectExplorer`` (an MTConnect +agent is itself a REST API; see docs/api.md). +""" + import threading -from pathlib import Path -from typing import Dict, List, Literal, Optional +from typing import List, Optional import streamlit as st from pydantic import BaseModel, ConfigDict, Field -from src.agent import AgentConfig, AgentAdapter -from src.client import ( - MtConnectClientConfiguration, - MtConnectDataCollector, - MtConnectDataCollectorConfiguration, - MtConnectExplorer, -) -from src.shdr import ShdrDataPlayer -from src.simulator import EquipmentType, EquipmentSimulator - -DATASETS_DIR = Path("datasets") -_LOG_FMT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" - - -# --------------------------------------------------------------------------- -# Logging setup — mirrors src/main.py Config class and config.json logic -# --------------------------------------------------------------------------- -class _Config(BaseModel): - log_level: Literal["debug", "info", "warning", "error", "critical"] = Field("info") - - def get_log_level_value(self) -> int: - return logging._nameToLevel[self.log_level.upper()] - - -def _strip_json_comments(text: str) -> str: - """Strip // line and /* */ block comments from JSON text so config.json can - be commented like JSONC. Comment sequences inside string values (e.g. a - "http://..." URL) are preserved: string literals are matched first and kept.""" - return re.sub( - r'"(?:\\.|[^"\\])*"|//[^\n]*|/\*.*?\*/', - lambda m: m.group(0) if m.group(0).startswith('"') else "", - text, - flags=re.DOTALL, - ) - - -_log_level = ( - _Config.model_validate_json( - _strip_json_comments(Path("config.json").read_text()) - ).get_log_level_value() - if os.path.isfile("config.json") - else _Config().get_log_level_value() -) -logging.basicConfig(level=_log_level, format=_LOG_FMT) -logging.getLogger("src").setLevel(_log_level) +from mtconnect_explorer_client.schemas import AgentSettings, CaptureCreate, ShdrSettings - -# --------------------------------------------------------------------------- -# Logging helpers -# --------------------------------------------------------------------------- -class _QueueHandler(logging.Handler): - def __init__(self, q: queue.Queue) -> None: - super().__init__() - self.queue = q - - def emit(self, record: logging.LogRecord) -> None: - try: - self.queue.put_nowait(self.format(record)) - except Exception: - self.handleError(record) - - -def attach_log(q: queue.Queue) -> _QueueHandler: - h = _QueueHandler(q) - h.setFormatter(logging.Formatter(_LOG_FMT)) - logging.getLogger("src").addHandler(h) - return h - - -def detach_log(h: Optional[_QueueHandler]) -> None: - if h is not None: - logging.getLogger("src").removeHandler(h) - - -# A play/capture session tags its worker/collector threads as "