diff --git a/exist-core/pom.xml b/exist-core/pom.xml index 677e244476..57fcc2ff3f 100644 --- a/exist-core/pom.xml +++ b/exist-core/pom.xml @@ -584,6 +584,12 @@ + + de.bottlecaps + markup-blitz + 1.12 + + xyz.elemental.fork.org.exist-db diff --git a/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java b/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java index 6183cc35b2..067cbd0ec1 100644 --- a/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java +++ b/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java @@ -328,6 +328,9 @@ public enum W3CErrorCode implements IErrorCode { FOXT0003 ("XSLT transformation failed"), FOXT0004 ("XSLT transformation has been disabled"), FOXT0006 ("XSLT output contains non-accepted characters"), + FOIX0001 ("Invalid Invisible XML grammar."), + FOIX0002 ("Input provided could not be parsed successfully."), + FOIX0003 ("No Invisible XML processor is available."), XTSE0165 ("It is a static error if the processor is not able to retrieve the resource identified by the URI reference [ in the href attribute of xsl:include or xsl:import] , or if the resource that is retrieved does not contain a stylesheet module conforming to this specification."); private final ErrorCode errorCode; @@ -1537,6 +1540,24 @@ public DynamicErrorCode(final QName qname, @Nullable final String description) { @Deprecated public static final ErrorCode FOXT0006 = W3CErrorCode.FOXT0006.errorCode; + /** + * @deprecated Use {@link W3CErrorCode#FOIX0001}. + */ + @Deprecated + public static final ErrorCode FOIX0001 = W3CErrorCode.FOIX0001.errorCode; + + /** + * @deprecated Use {@link W3CErrorCode#FOIX0002}. + */ + @Deprecated + public static final ErrorCode FOIX0002 = W3CErrorCode.FOIX0002.errorCode; + + /** + * @deprecated Use {@link W3CErrorCode#FOIX0003}. + */ + @Deprecated + public static final ErrorCode FOIX0003 = W3CErrorCode.FOIX0003.errorCode; + /** * @deprecated Use {@link W3CErrorCode#XTSE0165}. */ diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java new file mode 100644 index 0000000000..96064da2c1 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java @@ -0,0 +1,204 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.exist.xquery.functions.fn; + +import static com.evolvedbinary.j8fu.Either.Left; +import static com.evolvedbinary.j8fu.Either.Right; +import static org.exist.xquery.FunctionDSL.optParam; +import static org.exist.xquery.functions.fn.FnModule.functionSignature; +import static org.exist.xquery.FunctionDSL.param; +import static org.exist.xquery.FunctionDSL.returns; + +import com.evolvedbinary.j8fu.Either; +import org.exist.Namespaces; +import org.apache.commons.io.output.StringBuilderWriter; +import org.exist.dom.memtree.SAXAdapter; +import org.exist.util.XMLReaderPool; +import org.exist.util.serializer.XQuerySerializer; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; +import org.w3c.dom.Element; +import org.xml.sax.*; + +import de.bottlecaps.markup.Blitz; +import de.bottlecaps.markup.BlitzException; +import de.bottlecaps.markup.blitz.Parser; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; + +public class FnInvisibleXml extends BasicFunction { + + private static final String FS_INVISIBLE_XML_NAME = "invisible-xml"; + + final static FunctionSignature FS_INVISIBLE_XML = functionSignature( + FS_INVISIBLE_XML_NAME, + "Evaluates invisible XML.", + returns(Type.FUNCTION, "The iXML parsing function"), + optParam("grammar", Type.ITEM, "The iXML grammar"), + optParam("options", Type.MAP_ITEM, "Options for the iXML parser")); + + public FnInvisibleXml(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence optionsArg = args[1]; + final MapType options = optionsArg.isEmpty() + ? new MapType(context) + : (MapType) optionsArg.itemAt(0); + + final IxmlParserFunction fn; + final Sequence grammarArg = args[0]; + if (grammarArg.isEmpty()) { + // no grammar provided + fn = new IxmlParserFunction(context, (StringValue) null, options); + + } else if (grammarArg.getItemType() == Type.STRING) { + // grammar is a string + final StringValue grammarString = grammarArg.itemAt(0).toJavaObject(StringValue.class); + fn = new IxmlParserFunction(context, grammarString, options); + + } else { + // grammar is an element + final Element grammarItem = grammarArg.itemAt(0).toJavaObject(Element.class); + fn = new IxmlParserFunction(context, grammarItem, options); + } + + final FunctionCall invisibleXmlFunctionCall = new FunctionCall(context, fn); + return new FunctionReference(invisibleXmlFunctionCall); + } + + private static class IxmlParserFunction extends UserDefinedFunction { + + private static final String FS_PARSE_INVISIBLE_XML_NAME = "parse-invisible-xml"; + private static final FunctionSignature FS_PARSE_INVISIBLE_XML = functionSignature( + FS_PARSE_INVISIBLE_XML_NAME, + "Parses the input using the given iXML grammar.", + returns(Type.DOCUMENT, "The parsed document"), + param("input", Type.STRING, "The input to parse")); + + private static final StringValue FAIL_ON_ERROR_KEY = new StringValue("fail-on-error"); + + @Nullable + final Either grammar; + final MapType options; + + IxmlParserFunction(final XQueryContext context, @Nullable final StringValue grammar, + final MapType options) { + super(context, FS_PARSE_INVISIBLE_XML); + this.grammar = Left(grammar); + this.options = options; + } + + IxmlParserFunction(final XQueryContext context, @Nullable final Element grammar, + final MapType options) { + super(context, FS_PARSE_INVISIBLE_XML); + this.grammar = Right(grammar); + this.options = options; + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + + // get the input + final Sequence inputArg = getCurrentArguments()[0]; + final String input = inputArg.getStringValue(); + + // generate the default ixml grammar + final String ixmlGrammar; + // the null check here is wrong + if (grammar == null) { + // something went horabily wrong + throw new XPathException("idk how it can be null"); + } + if (grammar.isLeft()) { + if (grammar.left().get() == null) { + ixmlGrammar = Blitz.ixmlGrammar(); + } else { + ixmlGrammar = grammar.left().get().getStringValue(); + } + } else { + // grammar is an element: serialize it to a String + try (final StringBuilderWriter writer = new StringBuilderWriter()) { + final XQuerySerializer xqSerializer = new XQuerySerializer( + context.getBroker(), new Properties(), writer); + xqSerializer.serialize((Sequence) grammar.right().get()); + ixmlGrammar = writer.toString(); + } catch (final SAXException e) { + throw new XPathException(this, ErrorCodes.FOIX0001, e.getMessage(), e); + } + } + + final boolean failOnError = options.contains(FAIL_ON_ERROR_KEY) + && options.get(FAIL_ON_ERROR_KEY).effectiveBooleanValue(); + + final Parser parser; + try { + parser = Blitz.generate(ixmlGrammar); + } catch (final BlitzException e) { + throw new XPathException(this, ErrorCodes.FOIX0001, e.getMessage(), e); + } + + // parse the input using the ixml grammar + final String generatedXML; + try { + generatedXML = failOnError + ? parser.parse(input, Blitz.Option.FAIL_ON_ERROR) + : parser.parse(input); + } catch (final BlitzException e) { + throw new XPathException(this, ErrorCodes.FOIX0002, e.getMessage(), e); + } + + return parse(generatedXML); + } + + @Override + public void accept(final ExpressionVisitor visitor) { + if (visited) { + return; + } + visited = true; + } + + private Sequence parse(final String xmlContent) throws XPathException { + final XMLReaderPool pool = context.getBroker().getBrokerPool().getParserPool(); + final SAXAdapter adapter = new SAXAdapter(context); + final XMLReader reader = pool.borrowXMLReader(); + try (final StringReader stringReader = new StringReader(xmlContent)) { + reader.setContentHandler(adapter); + reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, adapter); + reader.parse(new InputSource(stringReader)); + return adapter.getDocument(); + } catch (final SAXException | IOException e) { + throw new XPathException(this, e.getMessage(), e); + } finally { + pool.returnXMLReader(reader); + } + + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java index e801e9ce75..16367be1c4 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java @@ -299,7 +299,8 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FnRandomNumberGenerator.FS_RANDOM_NUMBER_GENERATOR[0], FnRandomNumberGenerator.class), new FunctionDef(FnRandomNumberGenerator.FS_RANDOM_NUMBER_GENERATOR[1], FnRandomNumberGenerator.class), new FunctionDef(FunContainsToken.FS_CONTAINS_TOKEN[0], FunContainsToken.class), - new FunctionDef(FunContainsToken.FS_CONTAINS_TOKEN[1], FunContainsToken.class) + new FunctionDef(FunContainsToken.FS_CONTAINS_TOKEN[1], FunContainsToken.class), + new FunctionDef(FnInvisibleXml.FS_INVISIBLE_XML, FnInvisibleXml.class) }; static { diff --git a/exist-core/src/test/java/org/exist/xquery/functions/fn/FnInvisibleXmlTest.java b/exist-core/src/test/java/org/exist/xquery/functions/fn/FnInvisibleXmlTest.java new file mode 100644 index 0000000000..838223cde3 --- /dev/null +++ b/exist-core/src/test/java/org/exist/xquery/functions/fn/FnInvisibleXmlTest.java @@ -0,0 +1,157 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.test.ExistXmldbEmbeddedServer; +import org.junit.ClassRule; +import org.junit.Test; +import org.xmldb.api.base.XMLDBException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FnInvisibleXmlTest { + + // xquery version "3.1"; + + // let $date-grammar := " date = year, -'-', month, -'-', day . + // year = d, d, d, d . + // month = '0', d | '1', ['0'|'1'|'2'] . + // day = ['0'|'1'|'2'], d | '3', ['0'|'1'] . + // -d = ['0'-'9'] ." + + // let $valid-date-input := "2023-10-31" + + // let $invalid-date-input := "2023-10-32" + + // let $alphabit-grammar := "S=A. A='a'." + + // let $alphabit-valid-input := "a" + // (:expected a :) + // let $alphabit-invalid-input := "b" + // (:let $result := $parser("b"):) + // (:return $result/*/@*:state = 'failed':) + // (:expected true() :) + // (:FOIX0002 when fail-on is true:) + + // (:let $parser := fn:invisible-xml($grammar, map { }):) + // let $parser-fail-on := fn:invisible-xml($alphabit-grammar, map { + // "fail-on-error": true() }) + + // return $parser-fail-on($alphabit-invalid-input) + + @ClassRule + public static final ExistXmldbEmbeddedServer existEmbeddedServer = new ExistXmldbEmbeddedServer(false, true, true); + + private static final String DATE_GRAMMAR = " date = year, -'-', month, -'-', day .\n" + + " year = d, d, d, d .\n" + + "month = '0', d | '1', ['0'|'1'|'2'] .\n" + + " day = ['0'|'1'|'2'], d | '3', ['0'|'1'] .\n" + + " -d = ['0'-'9'] ."; + private static final String DATE_VALID_INPUT = "2023-10-31"; + private static final String DATE_INVALID_INPUT = "2023-10-32"; + + private static final String ALPHABIT_GRAMMAR = "S=A. A='a'."; + private static final String ALPHABIT_VALID_INPUT = "a"; + private static final String ALPHABIT_INVALID_INPUT = "b"; + + private static final String NO_OPTIONS = "map { }"; + private static final String FAIL_ON_ERROR_OPTIONS = "map { \"fail-on-error\": true() }"; + + @Test + public void dateValidInput() throws XMLDBException { + final String result = existEmbeddedServer + .executeOneValue(parseQuery(DATE_GRAMMAR, NO_OPTIONS, DATE_VALID_INPUT)); + assertEquals("20231031", result); + } + + @Test + public void dateInvalidInput() throws XMLDBException { + final String result = existEmbeddedServer + .executeOneValue(failedStateQuery(DATE_GRAMMAR, NO_OPTIONS, DATE_INVALID_INPUT)); + assertEquals("true", result); + } + + @Test + public void dateInvalidInputFailOnError() { + assertError("FOIX0002", parseQuery(DATE_GRAMMAR, FAIL_ON_ERROR_OPTIONS, DATE_INVALID_INPUT)); + } + + @Test + public void alphabitValidInput() throws XMLDBException { + final String result = existEmbeddedServer + .executeOneValue(parseQuery(ALPHABIT_GRAMMAR, NO_OPTIONS, ALPHABIT_VALID_INPUT)); + assertEquals("a", result); + } + + @Test + public void alphabitValidInputFailOnError() throws XMLDBException { + final String result = existEmbeddedServer + .executeOneValue(parseQuery(ALPHABIT_GRAMMAR, FAIL_ON_ERROR_OPTIONS, ALPHABIT_VALID_INPUT)); + assertEquals("a", result); + } + + @Test + public void alphabitInvalidInput() throws XMLDBException { + final String result = existEmbeddedServer + .executeOneValue(failedStateQuery(ALPHABIT_GRAMMAR, NO_OPTIONS, ALPHABIT_INVALID_INPUT)); + assertEquals("true", result); + } + + @Test + public void alphabitInvalidInputFailOnError() { + assertError("FOIX0002", parseQuery(ALPHABIT_GRAMMAR, FAIL_ON_ERROR_OPTIONS, ALPHABIT_INVALID_INPUT)); + } + + /** + * Builds a query that creates a parser from the grammar and options, and + * applies it to the input. + */ + private static String parseQuery(final String grammar, final String options, final String input) { + return "let $parser := fn:invisible-xml(" + stringLiteral(grammar) + ", " + options + ")\n" + + "return fn:serialize($parser(" + stringLiteral(input) + "))"; + } + + /** + * Builds a query that checks whether parsing the input produced a failed state. + */ + private static String failedStateQuery(final String grammar, final String options, final String input) { + return "let $parser := fn:invisible-xml(" + stringLiteral(grammar) + ", " + options + ")\n" + + "let $result := $parser(" + stringLiteral(input) + ")\n" + + "return $result/*/@*:state = 'failed'"; + } + + private static String stringLiteral(final String value) { + return "\"" + value.replace("\"", "\"\"") + "\""; + } + + private static void assertError(final String errorCode, final String query) { + try { + existEmbeddedServer.executeOneValue(query); + } catch (final XMLDBException e) { + assertTrue(e.getMessage(), e.getMessage().contains(errorCode)); + return; + } + + fail("Expected XPathException: err:" + errorCode); + } +}