fix

Author: sufubao

lightllm/server/function_call_parser.py

@@ -239,18 +239,38 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
 
         try:
             try:
-                tool_call_pos = current_text.find(self.bot_token)
-                if tool_call_pos != -1:
-                    start_idx = tool_call_pos + len(self.bot_token)
-                elif self.current_tool_id > 0 and current_text.startswith(self.tool_call_separator):
+                # Priority check: if we're processing a subsequent tool (current_tool_id > 0),
+                # first check if text starts with the tool separator. This is critical for
+                # parallel tool calls because the bot_token (e.g., '[') can also
+                # appear inside array parameters of the current tool, and we must not
+                # mistakenly identify that as the start of a new tool.
+                used_separator_branch = False
+                if self.current_tool_id > 0 and current_text.startswith(self.tool_call_separator):
                     start_idx = len(self.tool_call_separator)
+                    used_separator_branch = True
                 else:
-                    start_idx = 0
+                    tool_call_pos = current_text.find(self.bot_token)
+                    if tool_call_pos != -1:
+                        start_idx = tool_call_pos + len(self.bot_token)
+                    else:
+                        start_idx = 0
 
                 if start_idx >= len(current_text):
                     return StreamingParseResult()
 
-                obj, end_idx = _partial_json_loads(current_text[start_idx:], flags)
+                try:
+                    obj, end_idx = _partial_json_loads(current_text[start_idx:], flags)
+                except (MalformedJSON, json.JSONDecodeError):
+                    # Separator landed on non-JSON markup; fall back to
+                    # bot_token which skips past all inter-object markup.
+                    # e.g. Qwen25: separator "\n" matches between eot/bot tags.
+                    if used_separator_branch and self.bot_token in current_text:
+                        start_idx = current_text.find(self.bot_token) + len(self.bot_token)
+                        if start_idx >= len(current_text):
+                            return StreamingParseResult()
+                        obj, end_idx = _partial_json_loads(current_text[start_idx:], flags)
+                    else:
+                        raise
 
                 is_current_complete = _is_complete_json(current_text[start_idx : start_idx + end_idx])
 
@@ -272,7 +292,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
 
                 current_tool_call = obj
 
-            except MalformedJSON:
+            except (MalformedJSON, json.JSONDecodeError):
                 return StreamingParseResult()
 
             if not current_tool_call:
@@ -331,19 +351,24 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
                         # Only remove the processed portion, keep unprocessed content
                         self._buffer = current_text[start_idx + end_idx :]
 
-                        if self.current_tool_id < len(self.prev_tool_call_arr):
-                            self.prev_tool_call_arr[self.current_tool_id].clear()
-                        self.current_tool_name_sent = False
-                        self.streamed_args_for_tool[self.current_tool_id] = ""
-                        self.current_tool_id += 1
-
                     # If the tool is still being parsed, send incremental changes
                     elif prev_arguments:
                         prev_args_json = json.dumps(prev_arguments, ensure_ascii=False)
                         if cur_args_json != prev_args_json:
                             prefix = _find_common_prefix(prev_args_json, cur_args_json)
                             argument_diff = prefix[sent:]
 
+                    # Update prev_tool_call_arr BEFORE advancing current_tool_id
+                    if self.current_tool_id >= 0:
+                        while len(self.prev_tool_call_arr) <= self.current_tool_id:
+                            self.prev_tool_call_arr.append({})
+                        self.prev_tool_call_arr[self.current_tool_id] = current_tool_call
+
+                    # Advance to next tool if complete
+                    if is_current_complete:
+                        self.current_tool_name_sent = False
+                        self.current_tool_id += 1
+
                     # Send the argument diff if there's something new
                     if argument_diff is not None:
                         # Use the correct tool_index: completing_tool_id for completed tools,
@@ -357,15 +382,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
                                 )
                             ],
                         )
-                        if not is_current_complete:
-                            self.streamed_args_for_tool[self.current_tool_id] += argument_diff
-
-            # Update prev_tool_call_arr with current state
-            if self.current_tool_id >= 0:
-                # Ensure prev_tool_call_arr is large enough
-                while len(self.prev_tool_call_arr) <= self.current_tool_id:
-                    self.prev_tool_call_arr.append({})
-                self.prev_tool_call_arr[self.current_tool_id] = current_tool_call
+                        self.streamed_args_for_tool[tool_index_to_use] += argument_diff
 
             return res
 
@@ -396,8 +413,8 @@ def __init__(self):
         Initializes the detector with necessary state variables.
         """
         super().__init__()
-        self.bot_token = "<tool_call>"
-        self.eot_token = "</tool_call>"
+        self.bot_token = "<tool_call>\n"
+        self.eot_token = "\n</tool_call>"
         self.tool_call_separator = "\n"
         self._normal_text_buffer = ""  # Buffer for handling partial end tokens
 
@@ -443,7 +460,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
             self._normal_text_buffer += result.normal_text
 
             # Check if buffer contains complete end token (without leading newline)
-            end_token_without_newline = self.eot_token  # "</tool_call>"
+            end_token_without_newline = self.eot_token[1:]  # "</tool_call>" (strip leading \n)
             if end_token_without_newline in self._normal_text_buffer:
                 cleaned_text = self._normal_text_buffer.replace(end_token_without_newline, "")
                 self._normal_text_buffer = ""
@@ -1890,7 +1907,15 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
                 self._buffer = current_text[last_end + len(self.eot_token) :].lstrip()
             else:
                 self._buffer = ""
-            self.current_tool_id = -1
+
+            # Reassign tool_index sequentially so parallel calls using the same
+            # tool get distinct indices (detect_and_parse uses definition-position
+            # indices which collide when the same tool is called twice).
+            if self.current_tool_id == -1:
+                self.current_tool_id = 0
+            for call in result.calls:
+                call.tool_index = self.current_tool_id
+                self.current_tool_id += 1
             self.current_tool_name_sent = False
             return result