diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index f0822fd..207e265 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -5,7 +5,7 @@ pipeline { booleanParam(name: 'v1_26', defaultValue: false, description: 'maintenance/v1.26 branch (ryzen-5950x-1)') booleanParam(name: 'v1_27', defaultValue: false, description: 'maintenance/v1.27 branch (ryzen-5950x-1)') - booleanParam(name: 'master', defaultValue: false, description: 'master branch (ryzen-5950x-1)') + booleanParam(name: 'master', defaultValue: false, description: 'master branch, simulated with the solver each model asks for, filling master (ryzen-5950x-1). Ticking it together with cvode, gbode or ida runs one job that builds every model once and simulates it with each of them.') booleanParam(name: 'fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI, simulated by OMSimulator, filling v1.26-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_v1_26 runs one job that builds every FMU once and simulates it with both.') booleanParam(name: 'fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI, simulated by OMSimulator, filling v1.27-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_v1_27 runs one job that builds every FMU once and simulates it with both.') @@ -31,9 +31,9 @@ pipeline { booleanParam(name: 'report_ryzen_5950x_2', defaultValue: false, description: 'Generate a report for ryzen-5950x-2 without running the tests') booleanParam(name: 'conversion_script', defaultValue: false, description: 'master branch with conversion script from MSL 3 to 4 (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') - booleanParam(name: 'cvode', defaultValue: false, description: 'master branch, with -d=newInst and -s cvode (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') - booleanParam(name: 'gbode', defaultValue: false, description: 'master branch, with -d=newInst and -s gbode (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') - booleanParam(name: 'ida', defaultValue: false, description: 'master branch, with -d=newInst and -s ida (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') + booleanParam(name: 'cvode', defaultValue: false, description: 'master branch, simulated with -s cvode, filling cvode (ryzen-5950x-1). Ticking it together with master, gbode or ida runs one job that builds every model once and simulates it with each of them. This is an experimental job that does not run on a fixed schedule.') + booleanParam(name: 'gbode', defaultValue: false, description: 'master branch, simulated with -s gbode -gbm=radauIIA3, filling gbode (ryzen-5950x-1). Ticking it together with master, cvode or ida runs one job that builds every model once and simulates it with each of them. This is an experimental job that does not run on a fixed schedule.') + booleanParam(name: 'ida', defaultValue: false, description: 'master branch, simulated with -s ida, filling ida (ryzen-5950x-1). Ticking it together with master, cvode or gbode runs one job that builds every model once and simulates it with each of them. This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'generateSymbolicJacobian', defaultValue: false, description: 'master branch, with --generateSymbolicJacobian (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'wasm_jit', defaultValue: false, description: 'master branch, with --simCodeTarget=wasm-jit (ryzen-9950x). Every model is exported once as a wasm artifact and simulated three ways, filling wasm-jit (the runtime inside the artifact), wasm-jit-me (FMI 3.0 Model Exchange) and wasm-jit-cs (FMI 3.0 Co-Simulation). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'heavy_tests', defaultValue: false, description: 'master branch, runs one test at a time. That is, no parallel launching of tests. omc will use multiple threads for each test (-n=1 is not set unlike the other regression tests.), (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') @@ -98,10 +98,11 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.master } + expression { params.master || params.cvode || params.gbode || params.ida } } steps { - runRegressiontest('master', 'master', '', '', false, '', '', false, false) + runRegressiontest('master', 'master', '', '', false, '', '', false, false, 0, 'configs/conf.json', '', '.CI/testing', null, + solvers(params.master, params.cvode, params.gbode, params.ida)) } } @@ -323,54 +324,6 @@ pipeline { runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', false, '', '', false, false) } } - stage('cvode') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cvode } - } - steps { - runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s cvode', '', false, false) - } - } - stage('gbode') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.gbode } - } - steps { - runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s gbode -gbm=radauIIA3', '', false, false) - } - } - stage('ida') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.ida } - } - steps { - runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s ida', '', false, false) - } - } stage('wasm-jit') { agent { node { @@ -827,7 +780,20 @@ def fmiSimulators(boolean omsimulator, boolean fmpy) { return simulators } -def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '.CI/testing', fmiSimulators = null) { +/* The solvers a job runs, from the parameters that used to start one job each. + * Every model is built once and simulated with each of them, filling master, + * cvode, gbode and ida as the four jobs did; a library is only run for the + * solvers missing it, so one job serves master daily and cvode weekly. */ +def solvers(boolean dflt, boolean cvode, boolean gbode, boolean ida) { + def chosen = [] + if (dflt) chosen << 'default' + if (cvode) chosen << 'cvode' + if (gbode) chosen << 'gbode' + if (ida) chosen << 'ida' + return chosen +} + +def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '.CI/testing', fmiSimulators = null, solvers = null) { sh ''' find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true @@ -892,6 +858,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla simulators = name.contains('fmpy') ? ['fmpy'] : (omsHash ? ['OMSimulator'] : []) } FMI_TESTING_FLAG = "" + SOLVER_FLAG = (solvers ?: []).collect { " --solver=${it}" }.join('') if (simulators.contains('OMSimulator') && omsHash) { // In the image rather than on the node: test.py runs this binary from inside it, and one built // against the node's libraries need not load there. @@ -1171,7 +1138,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla ${cgroupReport} cd OpenModelicaLibraryTesting # Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions - stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 + stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG}${SOLVER_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 """) sh 'date' // In the image: the script talks to the results database through psycopg2, diff --git a/README.md b/README.md index 1dd2b38..7709558 100644 --- a/README.md +++ b/README.md @@ -18,23 +18,20 @@ are publicly available. The configuration file for the regular library nightly testsuite is [conf.json](configs/conf.json). Additional old and non-standard libraries are listed in [conf-old.json](configs/conf-old.json) and -[conf-nonstandard.json](configs/conf-nonstandard.json), note that failures in -those libraries may be due to the fact that they are not fully complying with -the Modelica standard, rather than to OpenModelica issues. The setup of the -configuration files is discussed in [conf-howto.md](conf-howto.md). - -Test results reports are collected in the -[https://libraries.openmodelica.org/branches/](libraries.openmodelica.org/branches) -directory. The -[overview.html](https://libraries.openmodelica.org/branches/overview.html) -report gives the results of the regular testsuite with the default C runtime and -solvers. Other reports contain the results using the C++ runtime, FMI, daeMode, -and the old frontend. Combined reports also include results from the old and -nonstandard libraries. The -[https://libraries.openmodelica.org/branches/history/](libraries.openmodelica.org/branches/history) -directory contains regression reports and plots using different versions -(including master) and simulation runtime configurations (C++, daeMode, FMI, old -frontend) of OpenModelica. +[conf-nonstandard.json](configs/conf-nonstandard.json); failures in those may be +due to the libraries not fully complying with the Modelica standard rather than +to OpenModelica issues. The setup of the configuration files is discussed in +[conf-howto.md](conf-howto.md). + +The reports are collected in +[libraries.openmodelica.org/branches](https://libraries.openmodelica.org/branches/). +[overview.html](https://libraries.openmodelica.org/branches/overview.html) gives +the results of the regular testsuite with the default C runtime and solvers; +others cover the C++ runtime, FMI, daeMode and the old frontend, and the +combined ones include the old and nonstandard libraries. +[history](https://libraries.openmodelica.org/branches/history/) holds regression +reports and plots across versions (master included) and runtime +configurations. If you want to include your open-source library in the testsuite, please open a pull request on [conf.json](configs/conf.json), or open an issue on the @@ -45,22 +42,19 @@ and ask us to do it for you. ### The image the OSMC jobs run in Every job of [.CI/Jenkinsfile](.CI/Jenkinsfile) runs in one docker image, built -from [.CI/testing/Dockerfile](.CI/testing/Dockerfile) at the start of the job -and with `--pull`, so a rebuilt base is picked up on its own. What a node needs -is then docker, git and an ssh key to publish the results with; the compilers, -the python environment, the `omc` that generates the mos files, FMPy and the -packages the tested libraries need are the image's, not the machine's, and a job -moved to another machine tests what it tested before. +with `--pull` from [.CI/testing/Dockerfile](.CI/testing/Dockerfile) at the start +of the job. A node needs docker, git and an ssh key to publish the results with; +the compilers, the python environment, the `omc` that generates the mos files, +FMPy and the packages the tested libraries need are the image's, so a job moved +to another machine tests what it tested before. The omc build, the OMSimulator build and `test.py` run inside the image; the -steps that use the node's own `omc` - installing the libraries and preparing the -reference files - stay outside it. The node's home directory is mounted, so the -cached omc build in `~/saved_omc`, the installed libraries and the ssh key are -the files they always were, and so is `/mnt/ReferenceFiles`, where the -maintenance job installs the reference results: the container reads the same -directory the node does, and writes the hash `test.py` keeps next to a reference -file back into it. The container is capped at 85% of the node's memory, which a -job running on the node itself was not. +steps using the node's own `omc` - installing the libraries and preparing the +reference files - stay outside. The node's home directory and +`/mnt/ReferenceFiles` are mounted, so the cached omc build in `~/saved_omc`, the +installed libraries, the ssh key and the reference results are the node's own +files, the hash `test.py` keeps next to a reference file included. The container +is capped at 85% of the node's memory. A dependency a library needs is therefore a line in the Dockerfile rather than an `apt-get install` repeated on every machine: @@ -197,6 +191,8 @@ Options: that one artifact several ways, see [One wasm artifact, three ways to simulate it](#one-wasm-artifact-three-ways-to-simulate-it) +- `--solver=[]`: Build every model once and simulate it once per solver, see + [One build, several solvers](#one-build-several-solvers) - `--ulimitvmem=8388608`: Virtual memory limit (in kB) - `--default=[]`: Add a default value for some configuration key, such as `--default=ulimitExe=60`. The equals sign is mandatory @@ -207,14 +203,10 @@ Options: ### Testing FMI with several simulators -Building an FMU costs far more than simulating it. Measured on the twelve -models of ExternData: 178 seconds building the FMUs, 0.7 simulating them with -OMSimulator and 2.6 with FMPy. Testing the same FMUs with a second tool -therefore used to cost almost twice as much as testing them with one, because -each job built its own copy of them. - -Give `--fmisimulator` once per tool and the FMUs are built once and simulated -with each of them: +Building an FMU costs far more than simulating it: on the twelve models of +ExternData, 178 seconds building the FMUs against 0.7 simulating them with +OMSimulator and 2.6 with FMPy. Give `--fmisimulator` once per tool and the FMUs +are built once and simulated with each of them: ```bash ./test.py --branch=v1.27-fmi --fmi=true \ @@ -231,20 +223,20 @@ own derived from it, so the run above fills | OMSimulator | `v1.27-fmi` | `branches/v1.27-fmi` | | FMPy | `v1.27-fmi-fmpy` | `branches/v1.27-fmi-fmpy` | -which is where those results have always been. OMSimulator keeps the plain -`-fmi` branch; every other tool adds its name. The branch a tool fills depends -on the tool and not on the order, so asking for FMPy alone still fills -`v1.27-fmi-fmpy` and leaves `v1.27-fmi` alone. +OMSimulator keeps the plain `-fmi` branch and every other tool adds its name. +The branch depends on the tool and not on the order, so asking for FMPy alone +still fills `v1.27-fmi-fmpy` and leaves `v1.27-fmi` alone. A library is only run +for the tools missing it: asking for both when `v1.27-fmi` already has that +library builds its FMUs and simulates them with FMPy alone. -A branch directory looks the same as it always did, file names included. The -`.err` of a model is written by the build, so every simulator of it publishes -the same one; the `.sim` and the difference files are the ones that simulator -produced. +A branch directory looks the same as it always did, file names included: the +`.err` is written by the build and shared, the `.sim` and the difference files +are the ones that simulator produced. -Only the simulator may differ between the results that share an FMU. Anything -that changes the FMU itself - a different compiler, a different library, a -different `--fmuType` or `--fmiFlags` - is a different job, which is why the -Co-Simulation jobs with CVODE are not merged with the Model Exchange ones. +Only the simulator may differ between results that share an FMU. Anything that +changes the FMU itself - a different compiler, a different library, a different +`--fmuType` or `--fmiFlags` - is a different job, which is why the Co-Simulation +jobs with CVODE are not merged with the Model Exchange ones. In Jenkins the parameters keep their meaning: `fmi_v1_27` asks for OMSimulator and `fmpy_fmi_v1_27` for FMPy. Ticking both runs one job that builds every FMU @@ -268,9 +260,9 @@ Adding one is an entry there and no change to any script: `result`, `requestedResult`, `tempDir`, `startTime`, `stopTime`, `tolerance`, `timeout`, `stepSize` and anything named in `optionalArguments`. - `optionalArguments` are the flags that have to disappear when there is nothing - to put in them. OMSimulator hangs on `--stepSize=0` rather than ignoring it, - so its step size flag lives here, while FMPy wants `--output-interval 0` all - the same and writes the value straight into its `arguments`. + to put in them: OMSimulator hangs on `--stepSize=0` rather than ignoring it, + while FMPy wants `--output-interval 0` all the same and writes it straight + into its `arguments`. - `command` is how the tool is invoked, `{simulator}` by default. FMPy needs a subcommand, `{simulator} simulate`. - `resultExtension` is what the tool writes, so that the comparison against the @@ -284,18 +276,17 @@ Adding one is an entry there and no change to any script: Then run it with `--fmisimulator=fmusim=/path/to/fmusim`, or just `--fmisimulator=/path/to/fmusim` if the command contains the name. -A tool that is a Python package rather than a command line needs a small driver -script that takes the arguments its entry passes, simulates, writes the result -file and exits non-zero when it fails; the entry then points `command` at it. +A tool that is a Python package rather than a command line needs a driver script +that takes the arguments its entry passes, writes the result file and exits +non-zero when it fails; `command` then points at it. ### One wasm artifact, three ways to simulate it `--simCodeTarget=wasm-jit` can export a model as a single WebAssembly artifact -that carries three things at once: the model's own simulation runtime, an FMI -3.0 Model Exchange interface and an FMI 3.0 Co-Simulation interface. Exporting -it costs one translation and one compilation; simulating it three ways then -costs three simulations and nothing else, which is the same bargain the FMI -simulators above strike. +carrying three things at once: the model's own simulation runtime, an FMI 3.0 +Model Exchange interface and an FMI 3.0 Co-Simulation interface. Exporting it +costs one translation and one compilation, and simulating it three ways then +costs three simulations - the same bargain the FMI simulators above strike. ```bash ./test.py --branch=master-wasm-jit --wasmjitrunner=sim,me,cs \ @@ -318,17 +309,54 @@ neither packed nor loaded: | `me` | `master-wasm-jit-me` | FMI 3.0 Model Exchange, integrated by omc with DASKR | | `cs` | `master-wasm-jit-cs` | FMI 3.0 Co-Simulation, the artifact integrating itself with DASKR | -Every run reports what loading and linking the artifact cost, so the `.sim` -files say how much of a short simulation is the artifact and how much is the -model. +Every run reports what loading and linking the artifact cost, so a short +simulation's `.sim` says how much of it was the artifact. The runners live in [configs/wasm-jit-runners.json](configs/wasm-jit-runners.json); adding one is an entry there (`simflags` is what is appended to the model's simulation flags, `branchSuffix` overrides the `-` it adds to the branch). -`--wasmjitrunner` and `--fmisimulator` both fan one build out into several -result branches, so a job uses one of them, not both. +`--wasmjitrunner`, `--fmisimulator` and `--solver` each fan one build out into +several result branches, so a job uses one of them, not several. + +### One build, several solvers + +Which solver integrates a model is a simulation flag, so testing another one +costs a simulation and not a build. `--solver` builds every model once and runs +it once per solver: + +```bash +./test.py --branch=master --solver=default,cvode,gbode,ida configs/myConf.json +./report.py --branches="cvode master" +# the overview.html it writes is published as overview-cvode.html +``` + +`default` is the model's own solver - DASSL unless the model says otherwise - +and every other name is what the run is given as `-s`. A solver fills the table +its entry names, which is the one each of them has always had: + +| solver | branch | simulated with | +| --- | --- | --- | +| `default` | `master` | the solver the model asks for | +| `cvode` | `cvode` | `-s cvode` | +| `gbode` | `gbode` | `-s gbode -gbm=radauIIA3` | +| `ida` | `ida` | `-s ida` | + +The branches are not filled at the same rate - master is tested twice a day, +cvode and gbode once a week, ida now and then - so a library is only run for the +solvers missing it: a weekly `--solver=default,cvode` builds a library master +already has and simulates it with CVODE alone, and one master has not been run +for with both. + +The solvers live in [configs/solvers.json](configs/solvers.json); adding one is +an entry there (`simflags` is what is appended to the model's simulation flags, +`branch` is the table it fills, a template over `{branch}` and `{name}` that +defaults to `{name}`). + +In Jenkins the parameters keep their meaning: `master` asks for the model's own +solver, `cvode`, `gbode` and `ida` for theirs. Ticking several runs one job that +builds every model once and simulates it with each. ### Testing a pull request @@ -352,12 +380,11 @@ git checkout -f --detach FETCH_HEAD ./test.py --branch=pr/ configs/conf.json ``` -`pr/` rather than `pr-`: the results of a pull request are stored and -published under `pr/`, so that `branches/` holds branches and the pull requests -sit together in one directory of it. It is the one job name that keeps the -directory part of its name - `maintenance/v1.27` is tested as `v1.27`. +`pr/` rather than `pr-`: the pull requests sit together in one directory +of `branches/`, which otherwise holds branches. It is the one job name that +keeps the directory part of its name - `maintenance/v1.27` is tested as `v1.27`. -and the report compares that run against the newest run of `master`: +The report compares that run against the newest run of `master`: ```bash ./pr-report.py # --baseline=master by default @@ -368,22 +395,20 @@ kind of page as the nightly regression reports, next to `00_comment.md`, a summary to comment on the pull request with. Both are published with the other reports. -`--comment` posts that summary on the pull request, and replaces it rather than -adding to it when the same pull request is tested again. It posts as whoever the -token belongs to: `GITHUB_TOKEN` or `GH_TOKEN` in the environment, or the account -[`gh`](https://cli.github.com) is logged in as. In Jenkins it is the -`pull_request_comment` parameter, which takes the token from an -`OpenModelica-Hudson` credential; without one the report is still written and -published, and the summary is in the build log. +`--comment` posts that summary on the pull request, replacing the one an earlier +run posted. It posts as whoever the token belongs to: `GITHUB_TOKEN` or +`GH_TOKEN` in the environment, or the account [`gh`](https://cli.github.com) is +logged in as. In Jenkins it is the `pull_request_comment` parameter, which takes +the token from an `OpenModelica-Hudson` credential; without one the report is +still written and published, and the summary is in the build log. Two things make a difference mean something other than "the pull request did -this", and the report says so when they apply: **the machine**, since two runs -produced on different hardware compare the hardware as much as the change, and -**the libraries**, since two runs that tested different library versions, or -verified against different reference files, differ for reasons of their own. The -baseline is also the newest `master` run rather than the commit the pull request -is based on, so a difference can come from anything merged since it was -branched - a reason to rebase before believing a surprising result. +this", and the report says so when they apply: **the machine**, since runs on +different hardware compare the hardware as much as the change, and **the +libraries**, since runs that tested different library versions or verified +against different reference files differ for reasons of their own. The baseline +is the newest `master` run rather than the commit the pull request is based on, +so a difference can also come from anything merged since it was branched. A full run takes days, so testing every pull request this way is not the idea; point `--branch=pr/` at a smaller configuration file when the question is diff --git a/configs/solvers.json b/configs/solvers.json new file mode 100644 index 0000000..8f12e32 --- /dev/null +++ b/configs/solvers.json @@ -0,0 +1,30 @@ +{ + "_comment": [ + "The solvers one built model can be simulated with. Adding one is an entry here and nothing else.", + "", + "A model is translated and compiled once and its executable is then run once per solver, so a job", + "asked for several of them costs one build and one simulation per solver.", + "", + "simflags what is appended to the model's simulation flags; -s picks the solver.", + "description what the solver is, for the .sim files and the reports.", + "branch the table this solver fills, a template over {branch} (the branch of the job) and", + " {name}. The default is {name}: a solver has always had a table of its own name." + ], + "default": { + "branch": "{branch}", + "simflags": "", + "description": "the solver the model asks for, DASSL unless it says otherwise" + }, + "cvode": { + "simflags": "-s cvode", + "description": "SUNDIALS CVODE" + }, + "gbode": { + "simflags": "-s gbode -gbm=radauIIA3", + "description": "the generic bi-rate ODE solver, with the radauIIA3 method" + }, + "ida": { + "simflags": "-s ida", + "description": "SUNDIALS IDA" + } +} diff --git a/shared.py b/shared.py index eed9dc5..4ea90c2 100644 --- a/shared.py +++ b/shared.py @@ -254,6 +254,46 @@ def branchForWasmJitRunner(branch, name): """Where the results of one wasm-jit runner are stored: --branch, then -.""" return branch + wasmJitRunner(name).get("branchSuffix", "-%s" % name) +# The solvers one built model can be run with, in configs/solvers.json. Like the +# wasm-jit runners, a solver is not a tool but a set of simulation flags, so +# cvode, gbode and ida share one translation and one compilation. +SOLVERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "configs", "solvers.json") +_solvers = None + +def solvers(path=None): + """Everything the testing knows about the solvers.""" + global _solvers + if _solvers is None or path: + with open(path or SOLVERS_FILE) as fin: + _solvers = dict((k, v) for (k, v) in json.load(fin).items() if not k.startswith("_")) + return _solvers + +def solver(name): + known = solvers() + if name not in known: + raise Exception("Unknown solver %s; known are %s. Adding one is an entry in %s." + % (name, ", ".join(sorted(known)), SOLVERS_FILE)) + return known[name] + +def parseSolvers(names): + """The --solver values as an ordered list of (name, simflags).""" + res = [] + for spec in names or []: + for name in spec.split(","): + name = name.strip() + if name: + res.append((name, solver(name).get("simflags") or "")) + seen = [n for (n, _) in res] + if len(set(seen)) != len(seen): + raise Exception("The same solver name is used twice: %s" % ", ".join(seen)) + return res + +def branchForSolver(branch, name): + """Where the results of one solver are stored: the table its entry names, as a + template over the branch of the job and the name of the solver.""" + return (solver(name).get("branch") or "{name}").format(branch=branch, name=name) + def branchForSimulator(branch, name): """Where the results of one FMI simulator of a run are stored. diff --git a/test.py b/test.py index 32f397e..57b018a 100755 --- a/test.py +++ b/test.py @@ -38,6 +38,7 @@ parser.add_argument('--coldhot', action="store_true", help="Simulate each model twice in the same omc; the second run reuses the compiled module. Both times are printed, but only the hot one is stored. Only used by simCodeTarget=wasm-jit.", default=False) parser.add_argument('--fmisimulator', action='append', default=[], help="FMI simulator to run the FMUs with, as 'name=command' or just the command. Repeat it to simulate every FMU with several tools without building it more than once; the first one stores its results in --branch and each further one in -, so --branch=master-fmi with OMSimulator and fmpy fills master-fmi and master-fmi-fmpy." ) parser.add_argument('--wasmjitrunner', action='append', default=[], help="Export every model once as a wasm artifact (buildModelFMU with fmuType=me_cs, platforms={wasm,}) and simulate that one artifact each of these ways: 'sim' runs the translated model the way simulate() does, 'me' and 'cs' the artifact's FMI 3.0 interfaces. Comma-separated or repeated; the first fills --branch and each further one -, so --branch=master-wasm-jit with sim,me,cs fills master-wasm-jit, master-wasm-jit-me and master-wasm-jit-cs. See configs/wasm-jit-runners.json. Only for simCodeTarget=wasm-jit.") +parser.add_argument('--solver', action='append', default=[], help="Build every model once and simulate it once per solver, so that testing another solver costs a simulation rather than a build. 'default' is the model's own solver and each further name is a -s the simulation is given; comma-separated or repeated. Every solver stores its results in the table its entry names, so --branch=master with default,cvode,gbode fills master, cvode and gbode. See configs/solvers.json.") parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024) parser.add_argument('--default', action='append', help="Add a default value for some configuration key, such as --default=ulimitExe=60. The equals sign is mandatory.", default=[]) parser.add_argument('-j', '--jobs', default=0, help="Ignored and deprecated, use procOMC:0 or procOMC:1 in the config") @@ -73,26 +74,25 @@ # The first simulator is the one the single-simulator code paths use. fmisimulator = fmisimulators[0][1] if fmisimulators else None wasmjitrunners = shared.parseWasmJitRunners(args.wasmjitrunner) -if fmisimulators and wasmjitrunners: - raise Exception("--fmisimulator and --wasmjitrunner both fan one build out into several result " - "branches; a job runs one of them, not both.") -# Everything one build is simulated by, whichever of the two it is. -runnerNames = [n for (n, _) in fmisimulators or wasmjitrunners] +solvers = shared.parseSolvers(args.solver) +if len([x for x in (fmisimulators, wasmjitrunners, solvers) if x]) > 1: + raise Exception("--fmisimulator, --wasmjitrunner and --solver each fan one build out into several " + "result branches; a job runs one of them, not several.") +# Everything one build is simulated by, whichever of the three it is. +runnerNames = [n for (n, _) in fmisimulators or wasmjitrunners or solvers] def branchForRunner(name): - return (shared.branchForSimulator(branch, name) if fmisimulators - else shared.branchForWasmJitRunner(branch, name)) - -# One branch per simulator. The first reports itself in the results of the -# model and the others under their own name, but every one of them stores its -# results where its own simulator belongs: a job given only FMPy on -# --branch=v1.27-fmi fills v1.27-fmi-fmpy, not v1.27-fmi. -resultBranches = [(branch, None)] -if runnerNames: - resultBranches = [(branchForRunner(runnerNames[0]), None)] - for simulatorName in runnerNames[1:]: - resultBranches.append((branchForRunner(simulatorName), simulatorName)) -# What the run asks about when it looks for results it already has. + if fmisimulators: + return shared.branchForSimulator(branch, name) + if wasmjitrunners: + return shared.branchForWasmJitRunner(branch, name) + return shared.branchForSolver(branch, name) + +# One branch per runner, where its own runner belongs: a job given only FMPy on +# --branch=v1.27-fmi fills v1.27-fmi-fmpy, not v1.27-fmi. Which of them a library +# is run for is decided per library, see runnersToRun. +resultBranches = [(branchForRunner(name), name) for name in runnerNames] or [(branch, None)] +# What the run asks about when it has no library to ask about. primaryBranch = resultBranches[0][0] jobOutput = result_location @@ -603,6 +603,43 @@ def hashReferenceFiles(s): res = "".join([getmd5(f) for f in files]) return res+"fixCorruptBuilds-2017-06-21" +def haveResultsFor(resultBranch, libName, conf): + """The newest run of that library in that branch, or None when there is none.""" + return cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM libversion NATURAL JOIN omcversion + WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", + (conf["libraryLastChange"], libName, resultBranch, omc_version, conf["confighash"])).fetchone() + +def runnersToRun(libName, conf): + """The runners of the job that still owe this library its results. + + The branches are not filled at the same rate - master is tested twice a day, + cvode and gbode once a week, ida now and then - so a library with anything + left to do is built again and simulated only by the runners missing it. + """ + return [name for (resultBranch, name) in resultBranches + if execAllTests or haveResultsFor(resultBranch, libName, conf) is None] + +def restrictRunners(conf, names): + """Tell testmodel.py which runners of the job this library is to be run by.""" + if fmisimulators: + chosen = [(n, c) for (n, c) in fmisimulators if n in names] + conf["fmisimulator"] = chosen[0][1] + conf["fmisimulators"] = ["%s=%s" % (n, c) for (n, c) in chosen] + elif wasmjitrunners: + conf["wasmjitrunners"] = names + elif solvers: + conf["solvers"] = names + +def ranRunner(libname, runner): + """Whether the run owed that library the results of that runner.""" + return runner is None or runner in stats_by_libname[libname]["runners"] + +def simulatorKey(libname, runner): + """How that runner's results are stored in a model's data: the first runner of + a library reports itself in the keys of the model, the others under their name.""" + runners = stats_by_libname[libname]["runners"] + return None if not runner or runner == runners[0] else runner + stats_by_libname = {} skipped_libs = {} # A library that did not load cannot be told apart from one with no models left. @@ -659,6 +696,8 @@ def hashReferenceFiles(s): conf["fmuType"] = fmuType if wasmjitrunners: conf["wasmjitrunners"] = [n for (n, _) in wasmjitrunners] + if solvers: + conf["solvers"] = [n for (n, _) in solvers] if (not canChangeOptLevel) and "optlevel" in conf: print("Deleting optlevel") del conf["optlevel"] @@ -773,22 +812,26 @@ def hashReferenceFiles(s): prefix = conf["ignoreModelPrefix"] res=list(filter(lambda x: not x.startswith(prefix), res)) libName=shared.libname(library, conf) - v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM libversion NATURAL JOIN omcversion - WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,primaryBranch,omc_version,confighash)).fetchone() + todo = runnersToRun(libName, conf) if libName in stats_by_libname or libName in skipped_libs: raise Exception("Duplicate libName found: %s" % libName) - if v is None or execAllTests: + if todo: # On the shared database another machine may already be running this exact - # job; claiming it is what keeps the two from testing the same thing. - if not db.claim(primaryBranch, libName, conf["libraryLastChange"], omc_version, confighash): - (host, since) = db.claimedBy(primaryBranch, libName, conf["libraryLastChange"], omc_version, confighash) + # job; claiming it is what keeps the two from testing the same thing. The + # library is built for the first runner missing it, so that is the branch. + claimBranch = branchForRunner(todo[0]) if todo[0] else branch + if not db.claim(claimBranch, libName, conf["libraryLastChange"], omc_version, confighash): + (host, since) = db.claimedBy(claimBranch, libName, conf["libraryLastChange"], omc_version, confighash) print("Skipping %s as %s has been testing it since %s" % (libName, host, since)) skipped_libs[libName] = None continue - stats_by_libname[libName] = {"conf":conf, "stats":[]} + restrictRunners(conf, todo) + stats_by_libname[libName] = {"conf":conf, "stats":[], "runners":todo} tests = tests + [(r,library,libName,libName+"_"+r,conf) for r in res] - print("Running library %s (%d tests)" % (libName, len(res))) + print("Running library %s (%d tests)%s" % (libName, len(res), + (" with %s" % ", ".join(todo)) if todo[0] else "")) else: + v = haveResultsFor(primaryBranch, libName, conf) print("Skipping %s as we already have results for it: %s" % (libName,str(v))) skipped_libs[libName] = v[0] @@ -1080,15 +1123,24 @@ def cpu_name(): for (name,model,libname,data) in stats.values(): testedModels[libname].add(model) -for (resultBranch, simulator) in resultBranches: +for key in stats.keys(): + (name,model,libname,data)=stats[key] + stats_by_libname[libname]["stats"].append(stats[key]) + +# A branch is filled by the libraries the run owed it; the others kept the +# results they had, and nothing of them is written or published again. +for (resultBranch, runner) in resultBranches: + libnames = [l for l in stats_by_libname.keys() if ranRunner(l, runner)] + if not libnames: + continue db.createTables(resultBranch) for key in stats.keys(): (name,model,libname,data)=stats[key] - if simulator is None: - stats_by_libname[libname]["stats"].append(stats[key]) + if not ranRunner(libname, runner): + continue cursor.execute("INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)%s" % (db.quote(resultBranch), db.insertIgnore()), - resultValues(model, libname, data, simulator)) - for libname in stats_by_libname.keys(): + resultValues(model, libname, data, simulatorKey(libname, runner))) + for libname in libnames: for model in removedModels(resultBranch, libname, testedModels[libname]): cursor.execute("INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)%s" % (db.quote(resultBranch), db.insertIgnore()), @@ -1182,15 +1234,15 @@ def dataForSimulator(data, simulator): return merged def artifactSuffix(simulator): - """What tells the files of one simulator from those of another.""" - return "_%s" % simulator if simulator and len(runnerNames) > 1 else "" - -# Every simulator publishes its own results - .sim and diff files included - to -# the directory of its own branch; the .err of the build is shared, so each of -# them gets a copy of it. -for (resultBranch, simulator) in resultBranches: + """What tells the files of one runner from those of another; the first runner + publishes the workspace itself, so only the others have one.""" + return "_%s" % simulator if simulator else "" + +# Every runner publishes its own results - .sim and diff files included - to the +# directory of its own branch; the .err of the build is shared, so each of them +# gets a copy of it. +for (resultBranch, runner) in resultBranches: result_location = outputFor(resultBranch) - suffix = artifactSuffix(simulator) if result_location != "" and (isWin or noSync): # Unlike the rsync path, this one is given the directory the branches live # in and appends the branch itself. @@ -1201,9 +1253,11 @@ def artifactSuffix(simulator): htmltpl=open("library.html.tpl").read() for libname in stats_by_libname.keys(): - if libname in skipped_libs: + if libname in skipped_libs or not ranRunner(libname, runner): continue s = None # Make sure I don't use this + simulator = simulatorKey(libname, runner) + suffix = artifactSuffix(simulator) stageRoot = stageRootFor(simulator, suffix) filesList = open(os.path.join(stageRoot, libname + ".files"), "w") filesList.write("/\n") diff --git a/testmodel.py b/testmodel.py index ae26b45..9555f21 100755 --- a/testmodel.py +++ b/testmodel.py @@ -240,8 +240,8 @@ def target(res): "simcold":None, "diff":None, "phase":0, - # One entry per FMI simulator beyond the first, which reports itself in the - # keys above; see configs/fmi-simulators.json. + # One entry per runner beyond the first, which reports itself in the keys + # above; see configs/fmi-simulators.json, wasm-jit-runners.json, solvers.json. "simulators":{} } simulators = execstat["simulators"] @@ -643,6 +643,9 @@ def simulateCmd(resimulate): if conf.get("fmi") and not fmisimulators and fmisimulator: fmisimulators = shared.parseFmiSimulators([fmisimulator]) wasmjitrunners = shared.parseWasmJitRunners(conf.get("wasmjitrunners")) if useArtifact else [] +# The model is built once and its executable run once per solver; the solvers are +# described in configs/solvers.json. +solverRunners = shared.parseSolvers(conf.get("solvers")) if not conf.get("fmi") and not isWasmJit else [] if daeMode: meRunners = [name for (name, flags) in wasmjitrunners if ":me:" in flags or flags.endswith(":me")] if meRunners: @@ -650,36 +653,31 @@ def simulateCmd(resimulate): fp.write("The model asks for --daeMode, which has no FMI Model Exchange interface: " "not running %s\n" % ", ".join(meRunners)) wasmjitrunners = [r for r in wasmjitrunners if r[0] not in meRunners] -# One build, several runs: an FMU simulated by several tools and a wasm artifact -# run several ways fan out the same way, so they share the naming below. -runners = fmisimulators or wasmjitrunners +# One build, several runs: an FMU simulated by several tools, a wasm artifact run +# several ways and a model run by several solvers fan out the same way. +runners = fmisimulators or wasmjitrunners or solverRunners -def resultFile(name=None): - """Where a simulator writes its results. +def runnerSuffix(name): + """What tells the files of one runner from those of another. The first one + publishes the workspace itself, so its files keep the plain names.""" + return "_%s" % name if name and runners and name != runners[0][0] else "" - Every tool writes what its entry says, and they only need to be told apart - when more than one of them runs on the same FMU. - """ - if not name: - return "%s_res.%s" % (conf["fileName"], outputFormat) - # A wasm-jit runner is omc, which writes the format the model was translated for. - extension = shared.fmiSimulator(name)["resultExtension"] if fmisimulators else outputFormat - if len(runners) < 2: - return "%s_res.%s" % (conf["fileName"], extension) - return "%s_%s_res.%s" % (conf["fileName"], name, extension) +def resultFile(name=None): + """Where a simulator writes its results.""" + # Only an FMI tool decides the format; the others write what the model was + # translated for. + extension = shared.fmiSimulator(name)["resultExtension"] if (name and fmisimulators) else outputFormat + return "%s%s_res.%s" % (conf["fileName"], runnerSuffix(name), extension) def artifactPrefix(name=None): """The files a simulator's results are written to, under files/.""" - if not name or len(runners) < 2: - return os.path.abspath("../files/%s" % conf["fileName"]).replace('\\','/') - return os.path.abspath("../files/%s_%s" % (conf["fileName"], name)).replace('\\','/') + return os.path.abspath("../files/%s%s" % (conf["fileName"], runnerSuffix(name))).replace('\\','/') resFile = resultFile(runners[0][0]) if runners else resultFile() def simulateFmu(name, command, resFile, simFile): """Run the FMU with one simulator, writing what it says to simFile.""" - # Only tell the runs apart when more than one of them shares the directory. - suffix = "_%s" % name if len(fmisimulators) > 1 else "" + suffix = runnerSuffix(name) fmitmpdir = "temp_%s%s_fmu" % (conf["fileName"].replace(".","_"), suffix) with open("%s.tmpfiles" % conf["fileName"], "a+") as fp: fp.write("%s\n" % fmitmpdir) @@ -731,6 +729,28 @@ def simulateArtifact(name, runnerFlags, resFile, simFile): fp.write(res.get("messages") or "") return res +def simulateExecutable(name, solverFlags, resFile, simFile): + """Run the built executable once, with the flags of one solver.""" + exe = ".\\%s.bat" % conf["fileName"] if isWin else "./%s" % conf["fileName"] + # A run sharing the directory with other solvers needs a result file of its own. + resultArgument = "-r=%s" % resFile if runnerSuffix(name) and outputFormat != "empty" else "" + cmd = " ".join(x for x in (exe, annotationSimFlags, conf["simFlags"], emit_protected, + "-lv LOG_STATS" if conf["simCodeTarget"]=="C" else "", + resultArgument, solverFlags) if x.strip()) + with open(simFile,"w") as fp: + fp.write("Environment - simulationEnvironment:\n") + for e in conf["environmentSimulation"]: + fp.write("%s = %s\n" % (e[0], e[1])) + fp.write("startTime=%g\nstopTime=%g\ntolerance=%g\nnumberOfIntervals=%d\nstepSize=%g\n" % (startTime,stopTime,tolerance,numberOfIntervals,stepSize)) + if name: + fp.write("Regular simulation (%s: %s): %s\n" % (name, shared.solver(name).get("description") or "", cmd)) + else: + fp.write("Regular simulation: %s\n" % cmd) + if isWin: + return checkOutputTimeout("%s >> %s" % (cmd,simFile), conf["ulimitExe"], conf) + pipe = "%s%s" % (conf["fileName"], runnerSuffix(name)) + return checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" % (pipe,pipe,pipe,simFile,cmd,pipe), conf["ulimitExe"], conf) + def simElapsed(): # omc's own time: the wall clock here covers the wrong run for these flags if useSimulate or useColdHot or useArtifact: @@ -784,29 +804,14 @@ def simElapsed(): with open(errFile, 'a+') as fp: fp.write("The hot simulation failed; keeping the cold time\n") else: - if isWin: - cmd = (".\\%s.bat %s %s %s" % (conf["fileName"],annotationSimFlags,conf["simFlags"],emit_protected)).strip() - else: - cmd = ("./%s %s %s %s" % (conf["fileName"],annotationSimFlags,conf["simFlags"],emit_protected)).strip() - - if conf["simCodeTarget"]=="C": - cmd = cmd + " -lv LOG_STATS" executable = os.path.normpath("%s.bat" % conf["fileName"] if isWin else conf["fileName"]) if not os.path.exists(executable): with open(errFile, 'a+') as fp: fp.write("The simulation executable %s does not exist\n" % executable) execstat["sim"] = monotonic()-start writeResultAndExit(0, False, omc, omc_new) - with open(simFile,"w") as fp: - fp.write("Environment - simulationEnvironment:\n") - for e in conf["environmentSimulation"]: - fp.write("%s = %s\n" % (e[0], e[1])) - fp.write("startTime=%g\nstopTime=%g\ntolerance=%g\nnumberOfIntervals=%d\nstepSize=%g\n" % (startTime,stopTime,tolerance,numberOfIntervals,stepSize)) - fp.write("Regular simulation: %s\n" % cmd) - if isWin: - res = checkOutputTimeout("%s >> %s" % (cmd,simFile), conf["ulimitExe"], conf) - else: - res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,cmd,conf["fileName"]), conf["ulimitExe"], conf) + (name, solverFlags) = solverRunners[0] if solverRunners else (None, "") + res = simulateExecutable(name, solverFlags, resFile, simFile) execstat["sim"] = simElapsed() execstat["simwall"] = monotonic()-start execstat["phase"] = 6 @@ -816,7 +821,7 @@ def simElapsed(): # checkOutputTimeout raises TimeoutError for a command that fails as well as # for one that runs out of time, so this covers both. if len(runners) > 1: - # The FMU is built and the other simulators are about to run it. What the + # The build is done and the other runners are about to use it. What the # first one did says nothing about them, so record the failure and let them # have their turn. Ending the model here would write them all down as having # failed at this phase without ever being started, which is what the loop @@ -943,8 +948,8 @@ def verifyAgainstReference(resFile, prefix, stat): if not firstSimulatorFailed: verifyAgainstReference(resFile, artifactPrefix(runners[0][0] if runners else None) + ".diff", execstat) -# The FMU is built; every other simulator the job asked for is now only a -# simulation and a comparison. A tool that times out or fails takes its own +# The build is done; every other runner the job asked for is now only a +# simulation and a comparison. One that times out or fails takes its own # results down with it and leaves the others alone - the first one included, # see the TimeoutError handler above. for (name, command) in runners[1:]: @@ -962,7 +967,10 @@ def verifyAgainstReference(resFile, prefix, stat): writeResult() continue else: - simulateFmu(name, command, other, simFileOther) + if solverRunners: + simulateExecutable(name, command, other, simFileOther) + else: + simulateFmu(name, command, other, simFileOther) stat["sim"] = monotonic()-start stat["simwall"] = stat["sim"] stat["phase"] = 6 @@ -971,7 +979,7 @@ def verifyAgainstReference(resFile, prefix, stat): stat["sim"] = monotonic()-start stat["simwall"] = stat["sim"] with open(errFile, 'a+') as fp: - fp.write("%s timed out simulating the %s\n" % (name, "artifact" if useArtifact else "FMU")) + fp.write("%s timed out simulating the %s\n" % (name, "artifact" if useArtifact else ("model" if solverRunners else "FMU"))) writeResult() # quit omc_new: every verification needed it