feat(genai): Support multimodal file inputs and display_name in function responses

libs/genai/langchain_google_genai/chat_models.py

@@ -33,6 +33,9 @@
     FunctionCall,
     FunctionDeclaration,
     FunctionResponse,
+    FunctionResponseBlob,
+    FunctionResponseFileData,
+    FunctionResponsePart,
     GenerateContentConfig,
     GenerateContentResponse,
     GenerationConfig,
@@ -656,17 +659,43 @@ def _convert_tool_message_to_parts(
     name = message.name or name or message.additional_kwargs.get("name")
     response: Any
     parts: list[Part] = []
+    function_response_parts: list[FunctionResponsePart] = []
     if isinstance(message.content, list):
         media_blocks = []
         other_blocks = []
         for block in message.content:
             if isinstance(block, dict) and (
-                is_data_content_block(block) or is_openai_data_block(block)
+                is_data_content_block(block)
+                or is_openai_data_block(block)
+                or block.get("type") in ("media", "file", "image_url")
             ):
                 media_blocks.append(block)
             else:
                 other_blocks.append(block)
-        parts.extend(_convert_to_parts(media_blocks, model=model))
+
+        base_parts = _convert_to_parts(media_blocks, model=model)
+        for i, p in enumerate(base_parts):
+            original_block = media_blocks[i]
+            display_name = original_block.get("display_name")
+            if p.inline_data:
+                blob = FunctionResponseBlob(
+                    data=p.inline_data.data,
+                    mime_type=p.inline_data.mime_type,
+                )
+                if display_name:
+                    blob.display_name = display_name
+                function_response_parts.append(FunctionResponsePart(inline_data=blob))
+            elif p.file_data:
+                file_data = FunctionResponseFileData(
+                    file_uri=p.file_data.file_uri,
+                    mime_type=p.file_data.mime_type,
+                )
+                if display_name:
+                    file_data.display_name = display_name
+                function_response_parts.append(
+                    FunctionResponsePart(file_data=file_data)
+                )
+
         response = other_blocks
 
     elif not isinstance(message.content, str):
@@ -676,14 +705,15 @@ def _convert_tool_message_to_parts(
             response = json.loads(message.content)
         except json.JSONDecodeError:
             response = message.content  # leave as str representation
-    part = Part(
-        function_response=FunctionResponse(
-            name=name,
-            response=(
-                {"output": response} if not isinstance(response, dict) else response
-            ),
-        )
-    )
+    response_dict = {"output": response} if not isinstance(response, dict) else response
+    function_response_kwargs: dict[str, Any] = {
+        "name": name,
+        "response": response_dict,
+    }
+    if function_response_parts:
+        function_response_kwargs["parts"] = function_response_parts
+
+    part = Part(function_response=FunctionResponse(**function_response_kwargs))
     parts.append(part)
     return parts
 

libs/genai/tests/unit_tests/test_chat_models.py

@@ -3623,13 +3623,41 @@ def test_convert_tool_message_to_parts_list_content_with_media() -> None:
         tool_call_id="123",
     )
     result = _convert_tool_message_to_parts(message)
-    assert len(result) == 2
-    # First part should be the media (image)
-    assert result[0].inline_data is not None
-    # Second part should be the function response
-    assert result[1].function_response is not None
-    assert result[1].function_response.name == "test_tool"
-    assert result[1].function_response.response == {"output": ["Text response"]}
+    assert len(result) == 1
+    # Function response should contain parts with media
+    assert result[0].function_response is not None
+    assert result[0].function_response.name == "test_tool"
+    assert result[0].function_response.response == {"output": ["Text response"]}
+    assert result[0].function_response.parts is not None
+    assert len(result[0].function_response.parts) == 1
+    assert result[0].function_response.parts[0].inline_data is not None
+
+
+def test_convert_tool_message_to_parts_with_display_name() -> None:
+    """Test `_convert_tool_message_to_parts` preserves `display_name`."""
+    message = ToolMessage(
+        name="test_tool",
+        content=[
+            {
+                "type": "media",
+                "mime_type": "application/pdf",
+                "file_uri": "gs://bucket/file.pdf",
+                "display_name": "My Document",
+            },
+        ],
+        tool_call_id="123",
+    )
+    result = _convert_tool_message_to_parts(message)
+    assert len(result) == 1
+    assert result[0].function_response is not None
+    assert result[0].function_response.parts is not None
+    assert len(result[0].function_response.parts) == 1
+    assert result[0].function_response.parts[0].file_data is not None
+    assert (
+        result[0].function_response.parts[0].file_data.file_uri
+        == "gs://bucket/file.pdf"
+    )
+    assert result[0].function_response.parts[0].file_data.display_name == "My Document"
 
 
 def test_convert_tool_message_to_parts_with_name_parameter() -> None: