From 3ab2658cd9c90c2ae0c3c1f4cad8aabc9ea188fd Mon Sep 17 00:00:00 2001 From: LeizhenZhang Date: Fri, 20 Nov 2026 15:16:03 -0600 Subject: [PATCH 1/2] [None][fix] MuseSpark tool parser: scan a whitespace run once in the header patterns The routing-header patterns tried every start inside a whitespace run, and the streaming hold check matched every suffix of a held run. Both took time quadratic in the run's length. On a 64k-character run in an answer, detect_and_parse took 11.5 s and streaming took 107 s, in the server's event loop. Start the header whitespace only where a run starts, skip suffixes that start inside a run, and accept an all-whitespace hold without the regex. Signed-off-by: LeizhenZhang --- .../serve/tool_parser/musespark_parser.py | 19 +++++-- .../llmapi/apps/test_musespark_tool_parser.py | 49 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/serve/tool_parser/musespark_parser.py b/tensorrt_llm/serve/tool_parser/musespark_parser.py index 376acea90931..ad49c70f6032 100644 --- a/tensorrt_llm/serve/tool_parser/musespark_parser.py +++ b/tensorrt_llm/serve/tool_parser/musespark_parser.py @@ -133,9 +133,14 @@ class MuseSparkToolParser(BaseToolParser): EOS = "<|end_of_text|>" STRUCTURAL_TOKENS = (START, MESSAGE, EOM, EOT, EOS) - _HEADER_RE = re.compile(r"(?:<\|start\|>assistant)?(\s+to=(" + _NAME + r"))<\|message\|>") + # A header's whitespace begins where a whitespace run begins. A match that + # starts inside a run also matches from the start of the run, and trying + # every start in a long run takes time quadratic in its length. + _HEADER_RE = re.compile( + r"(?:<\|start\|>assistant)?(?" + ) # Header still growing at the end of the buffer (no <|message|> yet). - _HEADER_TAIL_RE = re.compile(r"(?:<\|start\|>assistant\s+|\s+)to=" + _NAME + r"\s*$") + _HEADER_TAIL_RE = re.compile(r"(?:<\|start\|>assistant\s+|(?]*>") @@ -507,9 +512,17 @@ def _header_hold_len(cls, safe: str) -> int: """ hold = max(cls._partial_suffix_len(safe, tok) for tok in cls.STRUCTURAL_TOKENS) for n in range(len(safe), hold, -1): + if n < len(safe) and safe[-n].isspace() and safe[-n - 1].isspace(): + # A suffix that starts inside a whitespace run matches only if + # the longer one starting a character earlier did. + continue suffix = safe[-n:] core = suffix[: len(suffix) - cls._partial_suffix_len(suffix, cls.MESSAGE)] - if "<|start|>assistant".startswith(core) or cls._HEADER_PREFIX_RE.match(core): + if ( + core.isspace() + or "<|start|>assistant".startswith(core) + or cls._HEADER_PREFIX_RE.match(core) + ): return n return hold diff --git a/tests/unittest/llmapi/apps/test_musespark_tool_parser.py b/tests/unittest/llmapi/apps/test_musespark_tool_parser.py index bdea1d91b176..5d622ca43e5b 100644 --- a/tests/unittest/llmapi/apps/test_musespark_tool_parser.py +++ b/tests/unittest/llmapi/apps/test_musespark_tool_parser.py @@ -16,6 +16,7 @@ import json import random +import time import pytest @@ -611,6 +612,54 @@ def test_musespark_held_whitespace_is_not_rescanned_per_delta(monkeypatch, muses assert len(scanned) <= 4 +class _MatchRecorder: + """Wraps a compiled pattern and counts its `match` calls.""" + + def __init__(self, pattern): + self._pattern, self.calls = pattern, 0 + + def match(self, text, *args): + self.calls += 1 + return self._pattern.match(text, *args) + + +def test_musespark_held_whitespace_is_measured_once_per_delta(monkeypatch, musespark_sample_tools): + """A held whitespace run is matched against the header prefix once per delta. + + The word that ends the run releases it without a match for every + suffix that starts inside the run. + """ + recorder = _MatchRecorder(MuseSparkToolParser._HEADER_PREFIX_RE) + monkeypatch.setattr(MuseSparkToolParser, "_HEADER_PREFIX_RE", recorder) + text = "Answer:" + " " * 4000 + "done." + normal, calls = musespark_stream_chunks( + MuseSparkToolParser(), musespark_sample_tools, text, [4] * (len(text) // 4) + ) + assert normal == text and not calls + assert recorder.calls <= 2 * (len(text) // 4) + + +@pytest.mark.parametrize( + "text", + [ + "Answer:" + " " * 100_000 + "done.", + "set to=x" + " " * 100_000 + "done.", + f" to=user{MS_M}Answer:" + "\n" * 100_000 + "done.", + ], + ids=["spaces", "spaces_after_to", "newlines_in_answer"], +) +def test_musespark_header_patterns_are_linear_in_a_whitespace_run(text, musespark_sample_tools): + """The header patterns try a whitespace run from its start only. + + Trying every start inside a run of this length takes minutes. + """ + parser = MuseSparkToolParser() + start = time.monotonic() + result = parser.detect_and_parse(text, musespark_sample_tools) + assert time.monotonic() - start < 5 + assert not result.calls and result.normal_text.endswith("done.") + + def test_musespark_long_open_argument_is_parsed_once_per_closed_parameter( monkeypatch, musespark_sample_tools ): From 7f83da978c6e76a651433d51622404a88a4f2f3c Mon Sep 17 00:00:00 2001 From: LeizhenZhang Date: Sat, 21 Nov 2026 04:01:55 -0600 Subject: [PATCH 2/2] [None][fix] MuseSpark tool parser: set aside whitespace deltas while a whitespace run is held A whitespace run that could still precede a routing header stays in the streaming buffer, and every delta rescanned the whole buffer. Streaming a 64k-character run through the reasoning and tool parsers took 2.0 to 2.2 s, and 43 s after ` to=x`, growing three to four times per doubling. When the held buffer ends in whitespace, a whitespace delta cannot change what it holds, so it is kept aside and joined to the buffer with the next other delta. A 128k run now streams in 0.6 to 0.7 s. Streamed content and calls are unchanged. Add a streaming test for the header tail search over a held run, and a test that counts the characters the stream rescans at two run lengths. Signed-off-by: LeizhenZhang --- .../serve/tool_parser/musespark_parser.py | 29 ++++++ .../llmapi/apps/test_musespark_tool_parser.py | 96 +++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/tensorrt_llm/serve/tool_parser/musespark_parser.py b/tensorrt_llm/serve/tool_parser/musespark_parser.py index ad49c70f6032..1d4a7095271d 100644 --- a/tensorrt_llm/serve/tool_parser/musespark_parser.py +++ b/tensorrt_llm/serve/tool_parser/musespark_parser.py @@ -172,6 +172,10 @@ def __init__(self): def _reset_streaming_state(self) -> None: self._buffer = "" + # Whitespace deltas that arrived while the buffer was held and ended in + # whitespace. They belong after `_buffer`; see parse_streaming_increment. + self._held_whitespace: List[str] = [] + self._holds_whitespace_run = False self._in_tool_block = False self._in_reasoning_block = False self._block_text = "" @@ -368,6 +372,18 @@ def _may_hold_header(text: str) -> bool: # ------------------------------------------------------------- streaming def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult: + if self._holds_whitespace_run and new_text.isspace(): + # Outside tool and reasoning blocks, a held buffer that ends in + # whitespace is all whitespace, ` to=NAME` and whitespace, or a + # `<|start|>` header with no `<|message|>` yet. More whitespace + # leaves it held, so the delta is set aside unscanned: rescanning + # the held run on every delta takes time quadratic in its length. + self._held_whitespace.append(new_text) + return StreamingParseResult() + self._holds_whitespace_run = False + if self._held_whitespace: + self._buffer += "".join(self._held_whitespace) + self._held_whitespace = [] self._buffer += new_text content_parts: List[str] = [] calls: List[ToolCallItem] = [] @@ -450,6 +466,16 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami break normal_text = "".join(self._clean(p) for p in content_parts) + # `$` in _HEADER_PREFIX_RE also matches before a final newline, so + # `t\n` is held as a header prefix and `t\n\n` is not: a held + # buffer that ends in one newline after other text is excluded. + end = self._buffer[-2:] + self._holds_whitespace_run = ( + not self._in_tool_block + and not self._in_reasoning_block + and end[-1:].isspace() + and (end.isspace() or end[-1] != "\n") + ) return StreamingParseResult(normal_text=normal_text, calls=calls) except Exception as e: # pragma: no cover - defensive, mirrors upstream logger.error(f"Error in MuseSpark parse_streaming_increment: {e}") @@ -595,6 +621,9 @@ def finish(self, tools: List[Tool]) -> StreamingParseResult: """ calls: List[ToolCallItem] = [] normal_text = "" + self._buffer += "".join(self._held_whitespace) + self._held_whitespace = [] + self._holds_whitespace_run = False if self._in_tool_block: self._block_text += self._buffer self._buffer = "" diff --git a/tests/unittest/llmapi/apps/test_musespark_tool_parser.py b/tests/unittest/llmapi/apps/test_musespark_tool_parser.py index 5d622ca43e5b..8864d0c5c115 100644 --- a/tests/unittest/llmapi/apps/test_musespark_tool_parser.py +++ b/tests/unittest/llmapi/apps/test_musespark_tool_parser.py @@ -660,6 +660,102 @@ def test_musespark_header_patterns_are_linear_in_a_whitespace_run(text, musespar assert not result.calls and result.normal_text.endswith("done.") +def test_musespark_held_header_tail_is_linear_in_a_whitespace_run( + monkeypatch, musespark_sample_tools +): + """A held ` to=x` and the run after it are searched for a header tail from the run's start. + + The stream holds the text from ` to=x` on while it could still grow into + a header, and searches it with `_HEADER_TAIL_RE`. Trying every start + inside a run of this length takes minutes. + """ + parser = MuseSparkToolParser() + scanned = [] + monkeypatch.setattr( + parser, + "_HEADER_TAIL_RE", + _ScanRecorder(MuseSparkToolParser._HEADER_TAIL_RE, scanned), + ) + text = "set to=x" + " " * 100_000 + "done." + start = time.monotonic() + normal, calls = musespark_stream_chunks( + parser, musespark_sample_tools, text, [4] * (len(text) // 4) + ) + assert time.monotonic() - start < 5 + assert normal == text and not calls + # The search reached the held run. + assert max(scanned) > 100_000 + + +def _stream_through_reasoning_parser(tool_parser, tools, text, chunk=4): + """Stream `text` as the server does: reasoning parser content into the tool parser.""" + rp = MuseSparkReasoningParser() + results = [] + for pos in range(0, len(text), chunk): + content = rp.parse_delta(text[pos : pos + chunk]).content + if content: + results.append(tool_parser.parse_streaming_increment(content, tools)) + content = rp.finish().content + if content: + results.append(tool_parser.parse_streaming_increment(content, tools)) + results.append(tool_parser.finish(tools)) + normal = "".join(r.normal_text for r in results) + return normal, [c.name for r in results for c in r.calls if c.name] + + +_WHITESPACE_RUN_STREAMS = { + # id: (text around a run of n characters, normal text, call names) + "answer_spaces": ( + lambda run: f" to=user{MS_M}Answer:{run}done.{MS_EOT}", + lambda run: f"Answer:{run}done.", + [], + ), + "held_header_tail": ( + lambda run: f" to=user{MS_M}set to=x{run}done.{MS_EOT}", + lambda run: f"set to=x{run}done.", + [], + ), + "commentary_before_call": ( + lambda run: f" to=commentary{MS_M}Checking.{run}{MS_EOM}" + f"{MS_S}assistant to=get_weather{MS_M}{MS_WEATHER}{MS_EOT}", + lambda run: f"Checking.{run}", + ["get_weather"], + ), +} + + +@pytest.mark.parametrize("run_char", [" ", "\n"], ids=["spaces", "newlines"]) +@pytest.mark.parametrize("stream", list(_WHITESPACE_RUN_STREAMS)) +def test_musespark_streamed_whitespace_run_is_scanned_a_bounded_number_of_times( + monkeypatch, musespark_sample_tools, stream, run_char +): + """A streamed whitespace run is rescanned a bounded number of times. + + The run is held while a header could still follow it. Rescanning the held + run on every delta made the stream quadratic in the run's length: + doubling the run quadrupled the characters scanned. + """ + make_text, make_normal, expected_calls = _WHITESPACE_RUN_STREAMS[stream] + scanned = {} + for n in (8_000, 16_000): + parser = MuseSparkToolParser() + lengths = [] + strip_header = parser._strip_header + + def recording_strip_header(safe, lengths=lengths, strip_header=strip_header): + lengths.append(len(safe)) + return strip_header(safe) + + monkeypatch.setattr(parser, "_strip_header", recording_strip_header) + text = make_text(run_char * n) + normal, calls = _stream_through_reasoning_parser(parser, musespark_sample_tools, text) + assert normal == make_normal(run_char * n) + assert calls == expected_calls + scanned[n] = sum(lengths) + assert scanned[n] <= 3 * len(text) + assert scanned[16_000] <= 2.5 * scanned[8_000] + + def test_musespark_long_open_argument_is_parsed_once_per_closed_parameter( monkeypatch, musespark_sample_tools ):