`highlevel_action_parser` does not parse # properly

[xhluca]

Reproduce code

import pyparsing as pp
from typing import Any

from browsergym.core.action.parsers import highlevel_action_parser, NamedArgument

highlevel_code = 'send_msg_to_user("Personalization is a powerful driver of sales in the retail industry, especially for regional players like Lee\'s Market that serve diverse and culturally \\"specific\\" communities. Here’s how personalization can drive sales and actionable strategies tailored for Lee’s Market:\n\n## How Personalization Drives Sales in Retail\n")'

print("highlevel (original):", highlevel_action_parser.search_string(highlevel_code))

Results

highlevel (original): []

[xhluca]

Here's a fix:

import ast
import pyparsing as pp
from dataclasses import dataclass
from typing import Any

from browsergym.core.action.parsers import highlevel_action_parser

highlevel_code = 'send_msg_to_user("Personalization is a powerful driver of sales in the retail industry, especially for regional players like Lee\'s Market that serve diverse and culturally \\"specific\\" communities. Here’s how personalization can drive sales and actionable strategies tailored for Lee’s Market:\n\n## How Personalization Drives Sales in Retail\n")'


print("highlevel (original):", highlevel_action_parser.search_string(highlevel_code))

@dataclass
class NamedArgument:
    name: str
    value: Any
    def __repr__(self):
        return f"{self.name}={repr(self.value)}"

def _build_highlevel_action_parser_fixed() -> pp.ParserElement:
    def make_keyword(kwd_str, kwd_value):
        return pp.Keyword(kwd_str).set_parse_action(pp.replace_with(kwd_value))

    TRUE = make_keyword("True", True)
    FALSE = make_keyword("False", False)
    NONE = make_keyword("None", None)

    LBRACK, RBRACK, LBRACE, RBRACE, LPAREN, RPAREN, COLON = map(pp.Suppress, "[]{}():")

    # strings: support multiline contents inside quotes (messages often span lines)
    string = (
        pp.QuotedString('"', escChar='\\', unquoteResults=True, multiline=True)
        | pp.QuotedString("'", escChar='\\', unquoteResults=True, multiline=True)
    )

    # numbers and other python-like literals
    number = pp.pyparsing_common.number()
    dict_ = pp.Forward().set_name("dict")
    list_ = pp.Forward().set_name("list")
    _tuple = pp.Forward().set_name("tuple")

    element = (string | number | dict_ | list_ | _tuple | TRUE | FALSE | NONE).set_name("element")

    list_items = pp.DelimitedList(element, allow_trailing_delim=True).set_name(None)
    list_ << pp.Group(LBRACK + pp.Optional(list_items) + RBRACK, aslist=True)
    _tuple << pp.Group(LPAREN + pp.Optional(list_items) + RPAREN, aslist=True).set_parse_action(
        lambda tokens: tuple(tokens[0])
    )

    dict_item = pp.Group(string + COLON + element, aslist=True).set_name("dict item")
    dict_items = pp.DelimitedList(dict_item, allow_trailing_delim=True).set_name(None)
    dict_ << pp.Dict(LBRACE + pp.Optional(dict_items) + RBRACE, asdict=True)

    # named arg -> NamedArgument; use Suppress on '=' so token list is stable
    named_arg = (pp.pyparsing_common.identifier() + pp.Suppress(pp.Literal("=")) + element).set_parse_action(
        lambda tokens: NamedArgument(name=tokens[0], value=tokens[1])
    )
    arg_item = named_arg | element
    args = pp.Optional(pp.DelimitedList(arg_item, allow_trailing_delim=True)).set_name("args")

    # function call: group name and a grouped args-list
    function_call = pp.Group(pp.pyparsing_common.identifier("fname") + pp.Group(LPAREN + args + RPAREN)("args_grp"))

    # accept one or more function calls
    multiple_function_calls = pp.OneOrMore(function_call)
    multiple_function_calls.ignore(pp.python_style_comment())

    return multiple_function_calls

# this one will be used to extract python-like function calls
highlevel_action_parser_fixed: pp.ParserElement = _build_highlevel_action_parser_fixed()

print("highlevel (fixed):", highlevel_action_parser_fixed.search_string(highlevel_code))

Results:

highlevel (original): []
highlevel (fixed): [[['send_msg_to_user', ['Personalization is a powerful driver of sales in the retail industry, especially for regional players like Lee\'s Market that serve diverse and culturally "specific" communities. Here’s how personalization can drive sales and actionable strategies tailored for Lee’s Market:\n\n## How Personalization Drives Sales in Retail\n']]]]

[amanjaiswal73892]

Hey! @xhluca, Thank you for reporting this. It would be great if you can make a simple unit test for this.

[xhluca]

@amanjaiswal73892 i'm quite busy until end of november, happy to revisit this later

[shreyas1903]

Hi @xhluca @amanjaiswal73892 , I'd like to pick this up since the fix is already written and verified, I'll write the unit test covering the repro case (the # and escaped-quote parsing in multiline strings) and open a PR with the fix + test together. Will reference this issue in the PR.

[shreyas1903]

I've opened a PR with the fix + regression test: ServiceNow/BrowserGym#394. Since highlevel_action_parser is defined in BrowserGym, I filed it there. Used @xhluca suggested approach (explicit QuotedString variants handling multiline, #, and escaped quotes) and added the unit test @amanjaiswal73892 requested.