From 4432b1581b9bb3ced176c36767ecfa2b97932f8b Mon Sep 17 00:00:00 2001 From: Thuraabtech <97426541+Thuraabtech@users.noreply.github.com> Date: Wed, 29 Oct 2025 11:55:15 -0500 Subject: [PATCH 01/10] updated socket protocol implementation to be compatible with 1.0v (#72) * socket protocol updated to be compatible with 1.0v utcp * cubic fixes done * pinned mcp-use to use langchain 0.3.27 * removed mcp denpendency on langchain * adding the langchain dependency for testing (temporary) * remove langchain-core pin to resolve dependency conflict --------- Co-authored-by: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Co-authored-by: Salman Mohammed --- .github/workflows/test.yml | 3 +- .../mcp/pyproject.toml | 3 +- .../communication_protocols/socket/README.md | 45 ++- .../socket/pyproject.toml | 5 +- .../socket/src/utcp_socket/__init__.py | 18 ++ .../src/utcp_socket/tcp_call_template.py | 18 +- .../utcp_socket/tcp_communication_protocol.py | 192 +++++++------ .../src/utcp_socket/udp_call_template.py | 16 ++ .../utcp_socket/udp_communication_protocol.py | 191 +++++++------ .../tests/test_tcp_communication_protocol.py | 178 ++++++++++++ .../tests/test_udp_communication_protocol.py | 176 ++++++++++++ scripts/socket_sanity.py | 265 ++++++++++++++++++ socket_plugin_test.py | 40 +++ 13 files changed, 952 insertions(+), 198 deletions(-) create mode 100644 plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py create mode 100644 plugins/communication_protocols/socket/tests/test_udp_communication_protocol.py create mode 100644 scripts/socket_sanity.py create mode 100644 socket_plugin_test.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9394330..bf677f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,10 +31,11 @@ jobs: pip install -e plugins/communication_protocols/http[dev] pip install -e plugins/communication_protocols/mcp[dev] pip install -e plugins/communication_protocols/text[dev] + pip install -e plugins/communication_protocols/socket[dev] - name: Run tests with pytest run: | - pytest core/tests/ plugins/communication_protocols/cli/tests/ plugins/communication_protocols/http/tests/ plugins/communication_protocols/mcp/tests/ plugins/communication_protocols/text/tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=core/src/utcp --cov-report=xml --cov-report=html + pytest core/tests/ plugins/communication_protocols/cli/tests/ plugins/communication_protocols/http/tests/ plugins/communication_protocols/mcp/tests/ plugins/communication_protocols/text/tests/ plugins/communication_protocols/socket/tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=core/src/utcp --cov-report=xml --cov-report=html - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/plugins/communication_protocols/mcp/pyproject.toml b/plugins/communication_protocols/mcp/pyproject.toml index 36bb48e..9232cd5 100644 --- a/plugins/communication_protocols/mcp/pyproject.toml +++ b/plugins/communication_protocols/mcp/pyproject.toml @@ -15,7 +15,8 @@ dependencies = [ "pydantic>=2.0", "mcp>=1.12", "utcp>=1.0", - "mcp-use>=1.3" + "mcp-use>=1.3", + "langchain==0.3.27", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/socket/README.md b/plugins/communication_protocols/socket/README.md index 8febb5a..3e695c9 100644 --- a/plugins/communication_protocols/socket/README.md +++ b/plugins/communication_protocols/socket/README.md @@ -1 +1,44 @@ -Find the UTCP readme at https://github.com/universal-tool-calling-protocol/python-utcp. \ No newline at end of file +# UTCP Socket Plugin (UDP/TCP) + +This plugin adds UDP and TCP communication protocols to UTCP 1.0. + +## Running Tests + +Prerequisites: +- Python 3.10+ +- `pip` +- (Optional) a virtual environment + +1) Install core and the socket plugin in editable mode with dev extras: + +```bash +pip install -e "core[dev]" +pip install -e plugins/communication_protocols/socket[dev] +``` + +2) Run the socket plugin tests: + +```bash +python -m pytest plugins/communication_protocols/socket/tests -v +``` + +3) Run a single test or filter by keyword: + +```bash +# One file +python -m pytest plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py -v + +# Filter by keyword (e.g., delimiter framing) +python -m pytest plugins/communication_protocols/socket/tests -k delimiter -q +``` + +4) Optional end-to-end sanity check (mock UDP/TCP servers): + +```bash +python scripts/socket_sanity.py +``` + +Notes: +- On Windows, your firewall may prompt the first time tests open UDP/TCP sockets; allow access or run as admin if needed. +- Tests use `pytest-asyncio`. The dev extras installed above provide required dependencies. +- Streaming is single-chunk by design, consistent with HTTP/Text transports. Multi-chunk streaming can be added later behind provider configuration. \ No newline at end of file diff --git a/plugins/communication_protocols/socket/pyproject.toml b/plugins/communication_protocols/socket/pyproject.toml index 06f845e..2f232ad 100644 --- a/plugins/communication_protocols/socket/pyproject.toml +++ b/plugins/communication_protocols/socket/pyproject.toml @@ -36,4 +36,7 @@ dev = [ [project.urls] Homepage = "https://utcp.io" Source = "https://github.com/universal-tool-calling-protocol/python-utcp" -Issues = "https://github.com/universal-tool-calling-protocol/python-utcp/issues" \ No newline at end of file +Issues = "https://github.com/universal-tool-calling-protocol/python-utcp/issues" + +[project.entry-points."utcp.plugins"] +socket = "utcp_socket:register" \ No newline at end of file diff --git a/plugins/communication_protocols/socket/src/utcp_socket/__init__.py b/plugins/communication_protocols/socket/src/utcp_socket/__init__.py index e69de29..a0b7f3b 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/__init__.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/__init__.py @@ -0,0 +1,18 @@ +from utcp.plugins.discovery import register_communication_protocol, register_call_template +from utcp_socket.tcp_communication_protocol import TCPTransport +from utcp_socket.udp_communication_protocol import UDPTransport +from utcp_socket.tcp_call_template import TCPProviderSerializer +from utcp_socket.udp_call_template import UDPProviderSerializer + + +def register() -> None: + # Register communication protocols + register_communication_protocol("tcp", TCPTransport()) + register_communication_protocol("udp", UDPTransport()) + + # Register call templates and their serializers + register_call_template("tcp", TCPProviderSerializer()) + register_call_template("udp", UDPProviderSerializer()) + + +__all__ = ["register"] \ No newline at end of file diff --git a/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py b/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py index 157e43c..10fc1d6 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py @@ -1,6 +1,9 @@ from utcp.data.call_template import CallTemplate from typing import Optional, Literal from pydantic import Field +from utcp.interfaces.serializer import Serializer +from utcp.exceptions import UtcpSerializerValidationError +import traceback class TCPProvider(CallTemplate): """Provider configuration for raw TCP socket tools. @@ -63,7 +66,7 @@ class TCPProvider(CallTemplate): # Delimiter-based framing options message_delimiter: str = Field( default='\x00', - description="Delimiter to detect end of TCP response (e.g., '\\n', '\\r\\n', '\\x00'). Used with 'delimiter' framing." + description="Delimiter to detect end of TCP response (e.g., '\n', '\r\n', '\x00'). Used with 'delimiter' framing." ) # Fixed-length framing options fixed_message_length: Optional[int] = Field( @@ -77,3 +80,16 @@ class TCPProvider(CallTemplate): ) timeout: int = 30000 auth: None = None + + +class TCPProviderSerializer(Serializer[TCPProvider]): + def to_dict(self, obj: TCPProvider) -> dict: + return obj.model_dump() + + def validate_dict(self, data: dict) -> TCPProvider: + try: + return TCPProvider.model_validate(data) + except Exception as e: + raise UtcpSerializerValidationError( + f"Invalid TCPProvider: {e}\n{traceback.format_exc()}" + ) diff --git a/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py b/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py index 1b360a8..d5d64ac 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py @@ -10,9 +10,12 @@ import sys from typing import Dict, Any, List, Optional, Callable, Union -from utcp.client.client_transport_interface import ClientTransportInterface -from utcp.shared.provider import Provider, TCPProvider -from utcp.shared.tool import Tool +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp_socket.tcp_call_template import TCPProvider, TCPProviderSerializer +from utcp.data.tool import Tool +from utcp.data.call_template import CallTemplate, CallTemplateSerializer +from utcp.data.register_manual_response import RegisterManualResult +from utcp.data.utcp_manual import UtcpManual import logging logging.basicConfig( @@ -22,7 +25,7 @@ logger = logging.getLogger(__name__) -class TCPTransport(ClientTransportInterface): +class TCPTransport(CommunicationProtocol): """Transport implementation for TCP-based tool providers. This transport communicates with tools over TCP sockets. It supports: @@ -85,6 +88,35 @@ def _format_tool_call_message( else: # Default to JSON format return json.dumps(tool_args) + + def _ensure_tool_call_template(self, tool_data: Dict[str, Any], manual_call_template: TCPProvider) -> Dict[str, Any]: + """Normalize tool definition to include a valid 'tool_call_template'. + + - If 'tool_call_template' exists, validate it. + - Else if legacy 'tool_provider' exists, convert using TCPProviderSerializer. + - Else default to the provided manual_call_template. + """ + normalized = dict(tool_data) + try: + if "tool_call_template" in normalized and normalized["tool_call_template"] is not None: + try: + ctpl = CallTemplateSerializer().validate_dict(normalized["tool_call_template"]) # type: ignore + normalized["tool_call_template"] = ctpl + except Exception: + normalized["tool_call_template"] = manual_call_template + elif "tool_provider" in normalized and normalized["tool_provider"] is not None: + try: + ctpl = TCPProviderSerializer().validate_dict(normalized["tool_provider"]) # type: ignore + normalized.pop("tool_provider", None) + normalized["tool_call_template"] = ctpl + except Exception: + normalized.pop("tool_provider", None) + normalized["tool_call_template"] = manual_call_template + else: + normalized["tool_call_template"] = manual_call_template + except Exception: + normalized["tool_call_template"] = manual_call_template + return normalized def _encode_message_with_framing(self, message: str, provider: TCPProvider) -> bytes: """Encode message with appropriate TCP framing. @@ -115,7 +147,7 @@ def _encode_message_with_framing(self, message: str, provider: TCPProvider) -> b elif provider.framing_strategy == "delimiter": # Add delimiter after the message - delimiter = provider.message_delimiter or "\\x00" + delimiter = provider.message_delimiter or "\x00" # Handle escape sequences delimiter = delimiter.encode('utf-8').decode('unicode_escape') return message_bytes + delimiter.encode('utf-8') @@ -170,7 +202,7 @@ def _decode_response_with_framing(self, sock: socket.socket, provider: TCPProvid elif provider.framing_strategy == "delimiter": # Read until delimiter is found - delimiter = provider.message_delimiter or "\\x00" + delimiter = provider.message_delimiter or "\x00" delimiter = delimiter.encode('utf-8').decode('unicode_escape').encode('utf-8') response_data = b"" @@ -215,9 +247,6 @@ def _decode_response_with_framing(self, sock: socket.socket, provider: TCPProvid return response_data - else: - raise ValueError(f"Unknown framing strategy: {provider.framing_strategy}") - async def _send_tcp_message( self, host: str, @@ -289,122 +318,91 @@ def _send_and_receive(): self._log_error(f"Error in TCP communication: {e}") raise - async def register_tool_provider(self, manual_provider: Provider) -> List[Tool]: - """Register a TCP provider and discover its tools. - - Sends a discovery message to the TCP provider and parses the response. - - Args: - manual_provider: The TCPProvider to register - - Returns: - List of tools discovered from the TCP provider - - Raises: - ValueError: If provider is not a TCPProvider - """ - if not isinstance(manual_provider, TCPProvider): + async def register_manual(self, caller, manual_call_template: CallTemplate) -> RegisterManualResult: + """Register a TCP manual and discover its tools.""" + if not isinstance(manual_call_template, TCPProvider): raise ValueError("TCPTransport can only be used with TCPProvider") - self._log_info(f"Registering TCP provider '{manual_provider.name}'") + self._log_info(f"Registering TCP provider '{manual_call_template.name}'") try: - # Send discovery message - discovery_message = json.dumps({ - "type": "utcp" - }) - + discovery_message = json.dumps({"type": "utcp"}) response = await self._send_tcp_message( - manual_provider.host, - manual_provider.port, + manual_call_template.host, + manual_call_template.port, discovery_message, - manual_provider, - manual_provider.timeout / 1000.0, # Convert ms to seconds - manual_provider.response_byte_format + manual_call_template, + manual_call_template.timeout / 1000.0, + manual_call_template.response_byte_format ) - - # Parse response try: - # Handle bytes response by trying to decode as UTF-8 for JSON parsing - if isinstance(response, bytes): - response_str = response.decode('utf-8') - else: - response_str = response - + response_str = response.decode('utf-8') if isinstance(response, bytes) else response response_data = json.loads(response_str) - - # Check if response contains tools + tools: List[Tool] = [] if isinstance(response_data, dict) and 'tools' in response_data: tools_data = response_data['tools'] - - # Parse tools - tools = [] for tool_data in tools_data: try: - tool = Tool(**tool_data) - tools.append(tool) + normalized = self._ensure_tool_call_template(tool_data, manual_call_template) + tools.append(Tool(**normalized)) except Exception as e: - self._log_error(f"Invalid tool definition in TCP provider '{manual_provider.name}': {e}") + self._log_error(f"Invalid tool definition in TCP provider '{manual_call_template.name}': {e}") continue - - self._log_info(f"Discovered {len(tools)} tools from TCP provider '{manual_provider.name}'") - return tools + self._log_info(f"Discovered {len(tools)} tools from TCP provider '{manual_call_template.name}'") else: - self._log_info(f"No tools found in TCP provider '{manual_provider.name}' response") - return [] - + self._log_info(f"No tools found in TCP provider '{manual_call_template.name}' response") + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=tools) + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=manual, + success=True, + errors=[] + ) except json.JSONDecodeError as e: - self._log_error(f"Invalid JSON response from TCP provider '{manual_provider.name}': {e}") - return [] - + self._log_error(f"Invalid JSON response from TCP provider '{manual_call_template.name}': {e}") + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[]), + success=False, + errors=[str(e)] + ) except Exception as e: - self._log_error(f"Error registering TCP provider '{manual_provider.name}': {e}") - return [] + self._log_error(f"Error registering TCP provider '{manual_call_template.name}': {e}") + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[]), + success=False, + errors=[str(e)] + ) - async def deregister_tool_provider(self, manual_provider: Provider) -> None: - """Deregister a TCP provider. - - This is a no-op for TCP providers since connections are created per request. - - Args: - manual_provider: The provider to deregister - """ - if not isinstance(manual_provider, TCPProvider): + async def deregister_manual(self, caller, manual_call_template: CallTemplate) -> None: + """Deregister a TCP provider (no-op).""" + if not isinstance(manual_call_template, TCPProvider): raise ValueError("TCPTransport can only be used with TCPProvider") - - self._log_info(f"Deregistering TCP provider '{manual_provider.name}' (no-op)") + self._log_info(f"Deregistering TCP provider '{manual_call_template.name}' (no-op)") - async def call_tool(self, tool_name: str, tool_args: Dict[str, Any], tool_provider: Provider) -> Any: - """Call a TCP tool. - - Sends a tool call message to the TCP provider and returns the response. - - Args: - tool_name: Name of the tool to call - tool_args: Arguments for the tool call - tool_provider: The TCPProvider containing the tool - - Returns: - The response from the TCP tool - - Raises: - ValueError: If provider is not a TCPProvider - """ - if not isinstance(tool_provider, TCPProvider): + async def call_tool_streaming(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate): + async def _generator(): + yield await self.call_tool(caller, tool_name, tool_args, tool_call_template) + return _generator() + + async def call_tool(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate) -> Any: + """Call a TCP tool.""" + if not isinstance(tool_call_template, TCPProvider): raise ValueError("TCPTransport can only be used with TCPProvider") - self._log_info(f"Calling TCP tool '{tool_name}' on provider '{tool_provider.name}'") + self._log_info(f"Calling TCP tool '{tool_name}' on provider '{tool_call_template.name}'") try: - tool_call_message = self._format_tool_call_message(tool_args, tool_provider) + tool_call_message = self._format_tool_call_message(tool_args, tool_call_template) response = await self._send_tcp_message( - tool_provider.host, - tool_provider.port, + tool_call_template.host, + tool_call_template.port, tool_call_message, - tool_provider, - tool_provider.timeout / 1000.0, # Convert ms to seconds - tool_provider.response_byte_format + tool_call_template, + tool_call_template.timeout / 1000.0, + tool_call_template.response_byte_format ) return response diff --git a/plugins/communication_protocols/socket/src/utcp_socket/udp_call_template.py b/plugins/communication_protocols/socket/src/utcp_socket/udp_call_template.py index 4c704da..8c30c86 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/udp_call_template.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/udp_call_template.py @@ -1,6 +1,9 @@ from utcp.data.call_template import CallTemplate from typing import Optional, Literal from pydantic import Field +from utcp.interfaces.serializer import Serializer +from utcp.exceptions import UtcpSerializerValidationError +import traceback class UDPProvider(CallTemplate): """Provider configuration for UDP (User Datagram Protocol) socket tools. @@ -38,3 +41,16 @@ class UDPProvider(CallTemplate): response_byte_format: Optional[str] = Field(default="utf-8", description="Encoding to decode response bytes. If None, returns raw bytes.") timeout: int = 30000 auth: None = None + + +class UDPProviderSerializer(Serializer[UDPProvider]): + def to_dict(self, obj: UDPProvider) -> dict: + return obj.model_dump() + + def validate_dict(self, data: dict) -> UDPProvider: + try: + return UDPProvider.model_validate(data) + except Exception as e: + raise UtcpSerializerValidationError( + f"Invalid UDPProvider: {e}\n{traceback.format_exc()}" + ) diff --git a/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py b/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py index 8d4d404..b59ef37 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py @@ -9,14 +9,17 @@ import traceback from typing import Dict, Any, List, Optional, Callable, Union -from utcp.client.client_transport_interface import ClientTransportInterface -from utcp.shared.provider import Provider, UDPProvider -from utcp.shared.tool import Tool +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp_socket.udp_call_template import UDPProvider, UDPProviderSerializer +from utcp.data.tool import Tool +from utcp.data.call_template import CallTemplate, CallTemplateSerializer +from utcp.data.register_manual_response import RegisterManualResult +from utcp.data.utcp_manual import UtcpManual import logging logger = logging.getLogger(__name__) -class UDPTransport(ClientTransportInterface): +class UDPTransport(CommunicationProtocol): """Transport implementation for UDP-based tool providers. This transport communicates with tools over UDP sockets. It supports: @@ -80,6 +83,38 @@ def _format_tool_call_message( else: # Default to JSON format return json.dumps(tool_args) + + def _ensure_tool_call_template(self, tool_data: Dict[str, Any], manual_call_template: UDPProvider) -> Dict[str, Any]: + """Normalize tool definition to include a valid 'tool_call_template'. + + - If 'tool_call_template' exists, validate it. + - Else if legacy 'tool_provider' exists, convert using UDPProviderSerializer. + - Else default to the provided manual_call_template. + """ + normalized = dict(tool_data) + try: + if "tool_call_template" in normalized and normalized["tool_call_template"] is not None: + # Validate via generic CallTemplate serializer (type-dispatched) + try: + ctpl = CallTemplateSerializer().validate_dict(normalized["tool_call_template"]) # type: ignore + normalized["tool_call_template"] = ctpl + except Exception: + # Fallback to manual template if validation fails + normalized["tool_call_template"] = manual_call_template + elif "tool_provider" in normalized and normalized["tool_provider"] is not None: + # Convert legacy provider -> call template + try: + ctpl = UDPProviderSerializer().validate_dict(normalized["tool_provider"]) # type: ignore + normalized.pop("tool_provider", None) + normalized["tool_call_template"] = ctpl + except Exception: + normalized.pop("tool_provider", None) + normalized["tool_call_template"] = manual_call_template + else: + normalized["tool_call_template"] = manual_call_template + except Exception: + normalized["tool_call_template"] = manual_call_template + return normalized async def _send_udp_message( self, @@ -202,125 +237,89 @@ def _send_only(): self._log_error(f"Error sending UDP message (no response): {traceback.format_exc()}") raise - async def register_tool_provider(self, manual_provider: Provider) -> List[Tool]: - """Register a UDP provider and discover its tools. - - Sends a discovery message to the UDP provider and parses the response. - - Args: - manual_provider: The UDPProvider to register - - Returns: - List of tools discovered from the UDP provider - - Raises: - ValueError: If provider is not a UDPProvider - """ - if not isinstance(manual_provider, UDPProvider): + async def register_manual(self, caller, manual_call_template: CallTemplate) -> RegisterManualResult: + """Register a UDP manual and discover its tools.""" + if not isinstance(manual_call_template, UDPProvider): raise ValueError("UDPTransport can only be used with UDPProvider") - self._log_info(f"Registering UDP provider '{manual_provider.name}' at {manual_provider.host}:{manual_provider.port}") + self._log_info(f"Registering UDP provider '{manual_call_template.name}' at {manual_call_template.host}:{manual_call_template.port}") try: - # Send discovery message - discovery_message = json.dumps({ - "type": "utcp" - }) - + discovery_message = json.dumps({"type": "utcp"}) response = await self._send_udp_message( - manual_provider.host, - manual_provider.port, + manual_call_template.host, + manual_call_template.port, discovery_message, - manual_provider.timeout / 1000.0, # Convert ms to seconds - manual_provider.number_of_response_datagrams, - manual_provider.response_byte_format + manual_call_template.timeout / 1000.0, + manual_call_template.number_of_response_datagrams, + manual_call_template.response_byte_format ) - - # Parse response try: - # Handle bytes response by trying to decode as UTF-8 for JSON parsing - if isinstance(response, bytes): - response_str = response.decode('utf-8') - else: - response_str = response - + response_str = response.decode('utf-8') if isinstance(response, bytes) else response response_data = json.loads(response_str) - - # Check if response contains tools + tools: List[Tool] = [] if isinstance(response_data, dict) and 'tools' in response_data: tools_data = response_data['tools'] - - # Parse tools - tools = [] for tool_data in tools_data: try: - tool = Tool(**tool_data) + normalized = self._ensure_tool_call_template(tool_data, manual_call_template) + tool = Tool(**normalized) tools.append(tool) - except Exception as e: - self._log_error(f"Invalid tool definition in UDP provider '{manual_provider.name}': {traceback.format_exc()}") + except Exception: + self._log_error(f"Invalid tool definition in UDP provider '{manual_call_template.name}': {traceback.format_exc()}") continue - - self._log_info(f"Discovered {len(tools)} tools from UDP provider '{manual_provider.name}'") - return tools + self._log_info(f"Discovered {len(tools)} tools from UDP provider '{manual_call_template.name}'") else: - self._log_info(f"No tools found in UDP provider '{manual_provider.name}' response") - return [] - + self._log_info(f"No tools found in UDP provider '{manual_call_template.name}' response") + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=tools) + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=manual, + success=True, + errors=[] + ) except json.JSONDecodeError as e: - self._log_error(f"Invalid JSON response from UDP provider '{manual_provider.name}': {traceback.format_exc()}") - return [] - + self._log_error(f"Invalid JSON response from UDP provider '{manual_call_template.name}': {traceback.format_exc()}") + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[]) + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=manual, + success=False, + errors=[str(e)] + ) except Exception as e: - self._log_error(f"Error registering UDP provider '{manual_provider.name}': {traceback.format_exc()}") - return [] + self._log_error(f"Error registering UDP provider '{manual_call_template.name}': {traceback.format_exc()}") + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[]) + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=manual, + success=False, + errors=[str(e)] + ) - async def deregister_tool_provider(self, manual_provider: Provider) -> None: - """Deregister a UDP provider. - - This is a no-op for UDP providers since they are stateless. - - Args: - manual_provider: The provider to deregister - """ - if not isinstance(manual_provider, UDPProvider): + async def deregister_manual(self, caller, manual_call_template: CallTemplate) -> None: + if not isinstance(manual_call_template, UDPProvider): raise ValueError("UDPTransport can only be used with UDPProvider") - - self._log_info(f"Deregistering UDP provider '{manual_provider.name}' (no-op)") + self._log_info(f"Deregistering UDP provider '{manual_call_template.name}' (no-op)") - async def call_tool(self, tool_name: str, tool_args: Dict[str, Any], tool_provider: Provider) -> Any: - """Call a UDP tool. - - Sends a tool call message to the UDP provider and returns the response. - - Args: - tool_name: Name of the tool to call - arguments: Arguments for the tool call - tool_provider: The UDPProvider containing the tool - - Returns: - The response from the UDP tool - - Raises: - ValueError: If provider is not a UDPProvider - """ - if not isinstance(tool_provider, UDPProvider): + async def call_tool(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate) -> Any: + if not isinstance(tool_call_template, UDPProvider): raise ValueError("UDPTransport can only be used with UDPProvider") - - self._log_info(f"Calling UDP tool '{tool_name}' on provider '{tool_provider.name}'") - + self._log_info(f"Calling UDP tool '{tool_name}' on provider '{tool_call_template.name}'") try: - tool_call_message = self._format_tool_call_message(tool_args, tool_provider) - + tool_call_message = self._format_tool_call_message(tool_args, tool_call_template) response = await self._send_udp_message( - tool_provider.host, - tool_provider.port, + tool_call_template.host, + tool_call_template.port, tool_call_message, - tool_provider.timeout / 1000.0, # Convert ms to seconds - tool_provider.number_of_response_datagrams, - tool_provider.response_byte_format + tool_call_template.timeout / 1000.0, + tool_call_template.number_of_response_datagrams, + tool_call_template.response_byte_format ) return response - except Exception as e: self._log_error(f"Error calling UDP tool '{tool_name}': {traceback.format_exc()}") raise + + async def call_tool_streaming(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate): + yield await self.call_tool(caller, tool_name, tool_args, tool_call_template) diff --git a/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py b/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py new file mode 100644 index 0000000..1b6ffb2 --- /dev/null +++ b/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py @@ -0,0 +1,178 @@ +import asyncio +import json +import pytest + +from utcp_socket.tcp_communication_protocol import TCPTransport +from utcp_socket.tcp_call_template import TCPProvider + + +async def start_tcp_server(): + """Start a simple TCP server that sends a mutable JSON object then closes.""" + response_container = {"bytes": b""} + + async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + try: + # Read any incoming data to simulate request handling + await reader.read(1024) + except Exception: + pass + # Send response and close connection + writer.write(response_container["bytes"]) + await writer.drain() + try: + writer.close() + await writer.wait_closed() + except Exception: + pass + + server = await asyncio.start_server(handle, host="127.0.0.1", port=0) + port = server.sockets[0].getsockname()[1] + + def set_response(obj): + response_container["bytes"] = json.dumps(obj).encode("utf-8") + + return server, port, set_response + + +@pytest.mark.asyncio +async def test_register_manual_converts_legacy_tool_provider_tcp(): + """When manual returns legacy tool_provider, it is converted to tool_call_template.""" + # Start server and configure response after obtaining port + server, port, set_response = await start_tcp_server() + set_response({ + "tools": [ + { + "name": "tcp_tool", + "description": "Echo over TCP", + "inputs": {}, + "outputs": {}, + "tool_provider": { + "call_template_type": "tcp", + "name": "tcp-executor", + "host": "127.0.0.1", + "port": port, + "request_data_format": "json", + "response_byte_format": "utf-8", + "framing_strategy": "stream", + "timeout": 2000 + } + } + ] + }) + + try: + provider = TCPProvider( + name="tcp-provider", + host="127.0.0.1", + port=port, + request_data_format="json", + response_byte_format="utf-8", + framing_strategy="stream", + timeout=2000 + ) + transport_client = TCPTransport() + result = await transport_client.register_manual(None, provider) + + assert result.success + assert result.manual is not None + assert len(result.manual.tools) == 1 + tool = result.manual.tools[0] + assert tool.tool_call_template.call_template_type == "tcp" + assert isinstance(tool.tool_call_template, TCPProvider) + assert tool.tool_call_template.host == "127.0.0.1" + assert tool.tool_call_template.port == port + finally: + server.close() + await server.wait_closed() + + +@pytest.mark.asyncio +async def test_register_manual_validates_provided_tool_call_template_tcp(): + """When manual provides tool_call_template, it is validated and preserved.""" + server, port, set_response = await start_tcp_server() + set_response({ + "tools": [ + { + "name": "tcp_tool", + "description": "Echo over TCP", + "inputs": {}, + "outputs": {}, + "tool_call_template": { + "call_template_type": "tcp", + "name": "tcp-executor", + "host": "127.0.0.1", + "port": port, + "request_data_format": "json", + "response_byte_format": "utf-8", + "framing_strategy": "stream", + "timeout": 2000 + } + } + ] + }) + + try: + provider = TCPProvider( + name="tcp-provider", + host="127.0.0.1", + port=port, + request_data_format="json", + response_byte_format="utf-8", + framing_strategy="stream", + timeout=2000 + ) + transport_client = TCPTransport() + result = await transport_client.register_manual(None, provider) + + assert result.success + assert len(result.manual.tools) == 1 + tool = result.manual.tools[0] + assert tool.tool_call_template.call_template_type == "tcp" + assert isinstance(tool.tool_call_template, TCPProvider) + assert tool.tool_call_template.host == "127.0.0.1" + assert tool.tool_call_template.port == port + finally: + server.close() + await server.wait_closed() + + +@pytest.mark.asyncio +async def test_register_manual_fallbacks_to_manual_template_tcp(): + """When neither tool_provider nor tool_call_template is provided, fall back to manual template.""" + server, port, set_response = await start_tcp_server() + set_response({ + "tools": [ + { + "name": "tcp_tool", + "description": "Echo over TCP", + "inputs": {}, + "outputs": {} + } + ] + }) + + try: + provider = TCPProvider( + name="tcp-provider", + host="127.0.0.1", + port=port, + request_data_format="json", + response_byte_format="utf-8", + framing_strategy="stream", + timeout=2000 + ) + transport_client = TCPTransport() + result = await transport_client.register_manual(None, provider) + + assert result.success + assert len(result.manual.tools) == 1 + tool = result.manual.tools[0] + assert tool.tool_call_template.call_template_type == "tcp" + assert isinstance(tool.tool_call_template, TCPProvider) + # Should match manual (discovery) provider values + assert tool.tool_call_template.host == provider.host + assert tool.tool_call_template.port == provider.port + assert tool.tool_call_template.name == provider.name + finally: + server.close() + await server.wait_closed() \ No newline at end of file diff --git a/plugins/communication_protocols/socket/tests/test_udp_communication_protocol.py b/plugins/communication_protocols/socket/tests/test_udp_communication_protocol.py new file mode 100644 index 0000000..d6a770c --- /dev/null +++ b/plugins/communication_protocols/socket/tests/test_udp_communication_protocol.py @@ -0,0 +1,176 @@ +import asyncio +import json +import pytest + +from utcp_socket.udp_communication_protocol import UDPTransport +from utcp_socket.udp_call_template import UDPProvider + + +async def start_udp_server(): + """Start a simple UDP server that replies with a mutable JSON payload.""" + loop = asyncio.get_running_loop() + response_container = {"bytes": b""} + + class _Protocol(asyncio.DatagramProtocol): + def __init__(self, container): + self.container = container + self.transport = None + + def connection_made(self, transport): + self.transport = transport + + def datagram_received(self, data, addr): + # Always respond with the prepared payload + if self.transport: + self.transport.sendto(self.container["bytes"], addr) + + transport, protocol = await loop.create_datagram_endpoint( + lambda: _Protocol(response_container), local_addr=("127.0.0.1", 0) + ) + port = transport.get_extra_info("socket").getsockname()[1] + + def set_response(obj): + response_container["bytes"] = json.dumps(obj).encode("utf-8") + + return transport, port, set_response + + +@pytest.mark.asyncio +async def test_register_manual_converts_legacy_tool_provider_udp(): + """When manual returns legacy tool_provider, it is converted to tool_call_template.""" + # Start server and configure response after obtaining port + transport, port, set_response = await start_udp_server() + set_response({ + "tools": [ + { + "name": "udp_tool", + "description": "Echo over UDP", + "inputs": {}, + "outputs": {}, + "tool_provider": { + "call_template_type": "udp", + "name": "udp-executor", + "host": "127.0.0.1", + "port": port, + "number_of_response_datagrams": 1, + "request_data_format": "json", + "response_byte_format": "utf-8", + "timeout": 2000 + } + } + ] + }) + + try: + provider = UDPProvider( + name="udp-provider", + host="127.0.0.1", + port=port, + number_of_response_datagrams=1, + request_data_format="json", + response_byte_format="utf-8", + timeout=2000 + ) + transport_client = UDPTransport() + result = await transport_client.register_manual(None, provider) + + assert result.success + assert result.manual is not None + assert len(result.manual.tools) == 1 + tool = result.manual.tools[0] + assert tool.tool_call_template.call_template_type == "udp" + assert isinstance(tool.tool_call_template, UDPProvider) + assert tool.tool_call_template.host == "127.0.0.1" + assert tool.tool_call_template.port == port + finally: + transport.close() + + +@pytest.mark.asyncio +async def test_register_manual_validates_provided_tool_call_template_udp(): + """When manual provides tool_call_template, it is validated and preserved.""" + transport, port, set_response = await start_udp_server() + set_response({ + "tools": [ + { + "name": "udp_tool", + "description": "Echo over UDP", + "inputs": {}, + "outputs": {}, + "tool_call_template": { + "call_template_type": "udp", + "name": "udp-executor", + "host": "127.0.0.1", + "port": port, + "number_of_response_datagrams": 1, + "request_data_format": "json", + "response_byte_format": "utf-8", + "timeout": 2000 + } + } + ] + }) + + try: + provider = UDPProvider( + name="udp-provider", + host="127.0.0.1", + port=port, + number_of_response_datagrams=1, + request_data_format="json", + response_byte_format="utf-8", + timeout=2000 + ) + transport_client = UDPTransport() + result = await transport_client.register_manual(None, provider) + + assert result.success + assert len(result.manual.tools) == 1 + tool = result.manual.tools[0] + assert tool.tool_call_template.call_template_type == "udp" + assert isinstance(tool.tool_call_template, UDPProvider) + assert tool.tool_call_template.host == "127.0.0.1" + assert tool.tool_call_template.port == port + finally: + transport.close() + + +@pytest.mark.asyncio +async def test_register_manual_fallbacks_to_manual_template_udp(): + """When neither tool_provider nor tool_call_template is provided, fall back to manual template.""" + transport, port, set_response = await start_udp_server() + set_response({ + "tools": [ + { + "name": "udp_tool", + "description": "Echo over UDP", + "inputs": {}, + "outputs": {} + } + ] + }) + + try: + provider = UDPProvider( + name="udp-provider", + host="127.0.0.1", + port=port, + number_of_response_datagrams=1, + request_data_format="json", + response_byte_format="utf-8", + timeout=2000 + ) + transport_client = UDPTransport() + result = await transport_client.register_manual(None, provider) + + assert result.success + assert len(result.manual.tools) == 1 + tool = result.manual.tools[0] + assert tool.tool_call_template.call_template_type == "udp" + assert isinstance(tool.tool_call_template, UDPProvider) + # Should match manual (discovery) provider values + assert tool.tool_call_template.host == provider.host + assert tool.tool_call_template.port == provider.port + assert tool.tool_call_template.name == provider.name + finally: + transport.close() \ No newline at end of file diff --git a/scripts/socket_sanity.py b/scripts/socket_sanity.py new file mode 100644 index 0000000..40b2c16 --- /dev/null +++ b/scripts/socket_sanity.py @@ -0,0 +1,265 @@ +import sys +import os +import json +import time +import socket +import threading +import asyncio +from pathlib import Path + +# Ensure core and socket plugin sources are on sys.path +ROOT = Path(__file__).resolve().parent.parent +CORE_SRC = ROOT / "core" / "src" +SOCKET_SRC = ROOT / "plugins" / "communication_protocols" / "socket" / "src" +for p in [str(CORE_SRC), str(SOCKET_SRC)]: + if p not in sys.path: + sys.path.insert(0, p) + +from utcp_socket.udp_communication_protocol import UDPTransport +from utcp_socket.tcp_communication_protocol import TCPTransport +from utcp_socket.udp_call_template import UDPProvider +from utcp_socket.tcp_call_template import TCPProvider + +# ------------------------------- +# Mock UDP Server +# ------------------------------- + +def start_udp_server(host: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind((host, port)) + + def run(): + while True: + data, addr = sock.recvfrom(65535) + try: + msg = data.decode("utf-8") + except Exception: + msg = "" + # Handle discovery + try: + parsed = json.loads(msg) + except Exception: + parsed = None + if isinstance(parsed, dict) and parsed.get("type") == "utcp": + manual = { + "utcp_version": "1.0", + "manual_version": "1.0", + "tools": [ + { + "name": "udp.echo", + "description": "Echo UDP args as JSON", + "inputs": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "extra": {"type": "number"} + }, + "required": ["text"] + }, + "outputs": { + "type": "object", + "properties": { + "ok": {"type": "boolean"}, + "echo": {"type": "string"}, + "args": {"type": "object"} + } + }, + "tags": ["socket", "udp"], + "average_response_size": 64, + # Return legacy provider to exercise conversion path + "tool_provider": { + "call_template_type": "udp", + "name": "udp", + "host": host, + "port": port, + "request_data_format": "json", + "response_byte_format": "utf-8", + "number_of_response_datagrams": 1, + "timeout": 3000 + } + } + ] + } + payload = json.dumps(manual).encode("utf-8") + sock.sendto(payload, addr) + else: + # Tool call: echo JSON payload + try: + args = json.loads(msg) + except Exception: + args = {"raw": msg} + resp = { + "ok": True, + "echo": args.get("text", ""), + "args": args + } + sock.sendto(json.dumps(resp).encode("utf-8"), addr) + t = threading.Thread(target=run, daemon=True) + t.start() + return t + +# ------------------------------- +# Mock TCP Server (delimiter-based) +# ------------------------------- + +def start_tcp_server(host: str, port: int, delimiter: str = "\n"): + delim_bytes = delimiter.encode("utf-8") + + def handle_client(conn: socket.socket, addr): + try: + # Read until delimiter + buf = b"" + while True: + chunk = conn.recv(1) + if not chunk: + break + buf += chunk + if buf.endswith(delim_bytes): + break + msg = buf[:-len(delim_bytes)].decode("utf-8") if buf.endswith(delim_bytes) else buf.decode("utf-8") + # Discovery + parsed = None + try: + parsed = json.loads(msg) + except Exception: + pass + if isinstance(parsed, dict) and parsed.get("type") == "utcp": + manual = { + "utcp_version": "1.0", + "manual_version": "1.0", + "tools": [ + { + "name": "tcp.echo", + "description": "Echo TCP args as JSON", + "inputs": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "extra": {"type": "number"} + }, + "required": ["text"] + }, + "outputs": { + "type": "object", + "properties": { + "ok": {"type": "boolean"}, + "echo": {"type": "string"}, + "args": {"type": "object"} + } + }, + "tags": ["socket", "tcp"], + "average_response_size": 64, + # Legacy provider to exercise conversion + "tool_provider": { + "call_template_type": "tcp", + "name": "tcp", + "host": host, + "port": port, + "request_data_format": "json", + "response_byte_format": "utf-8", + "framing_strategy": "delimiter", + "message_delimiter": "\\n", + "timeout": 3000 + } + } + ] + } + payload = json.dumps(manual).encode("utf-8") + delim_bytes + conn.sendall(payload) + else: + # Tool call: echo JSON payload + try: + args = json.loads(msg) + except Exception: + args = {"raw": msg} + resp = { + "ok": True, + "echo": args.get("text", ""), + "args": args + } + conn.sendall(json.dumps(resp).encode("utf-8") + delim_bytes) + finally: + try: + conn.shutdown(socket.SHUT_RDWR) + except Exception: + pass + conn.close() + + def run(): + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind((host, port)) + srv.listen(5) + while True: + conn, addr = srv.accept() + threading.Thread(target=handle_client, args=(conn, addr), daemon=True).start() + + t = threading.Thread(target=run, daemon=True) + t.start() + return t + +# ------------------------------- +# Sanity test runner +# ------------------------------- + +async def run_sanity(): + udp_host, udp_port = "127.0.0.1", 23456 + tcp_host, tcp_port = "127.0.0.1", 23457 + + # Start servers + start_udp_server(udp_host, udp_port) + start_tcp_server(tcp_host, tcp_port, delimiter="\n") + await asyncio.sleep(0.2) # small delay to ensure servers are listening + + # Transports + udp_transport = UDPTransport() + tcp_transport = TCPTransport() + + # Register manuals + udp_manual_template = UDPProvider(name="udp", host=udp_host, port=udp_port, request_data_format="json", response_byte_format="utf-8", number_of_response_datagrams=1, timeout=3000) + tcp_manual_template = TCPProvider(name="tcp", host=tcp_host, port=tcp_port, request_data_format="json", response_byte_format="utf-8", framing_strategy="delimiter", message_delimiter="\n", timeout=3000) + + udp_reg = await udp_transport.register_manual(None, udp_manual_template) + tcp_reg = await tcp_transport.register_manual(None, tcp_manual_template) + + print("UDP register success:", udp_reg.success, "tools:", len(udp_reg.manual.tools)) + print("TCP register success:", tcp_reg.success, "tools:", len(tcp_reg.manual.tools)) + + assert udp_reg.success and len(udp_reg.manual.tools) == 1 + assert tcp_reg.success and len(tcp_reg.manual.tools) == 1 + + # Verify tool_call_template present + assert udp_reg.manual.tools[0].tool_call_template.call_template_type == "udp" + assert tcp_reg.manual.tools[0].tool_call_template.call_template_type == "tcp" + + # Call tools + udp_result = await udp_transport.call_tool(None, "udp.echo", {"text": "hello", "extra": 42}, udp_reg.manual.tools[0].tool_call_template) + tcp_result = await tcp_transport.call_tool(None, "tcp.echo", {"text": "world", "extra": 99}, tcp_reg.manual.tools[0].tool_call_template) + + print("UDP call result:", udp_result) + print("TCP call result:", tcp_result) + + # Basic assertions on response shape + def ensure_dict(s): + if isinstance(s, (bytes, bytearray)): + try: + s = s.decode("utf-8") + except Exception: + return {} + if isinstance(s, str): + try: + return json.loads(s) + except Exception: + return {"raw": s} + return s if isinstance(s, dict) else {} + + udp_resp = ensure_dict(udp_result) + tcp_resp = ensure_dict(tcp_result) + + assert udp_resp.get("ok") is True and udp_resp.get("echo") == "hello" + assert tcp_resp.get("ok") is True and tcp_resp.get("echo") == "world" + + print("Sanity passed: UDP/TCP discovery and calls work with tool_call_template normalization.") + +if __name__ == "__main__": + asyncio.run(run_sanity()) \ No newline at end of file diff --git a/socket_plugin_test.py b/socket_plugin_test.py new file mode 100644 index 0000000..c03d2a9 --- /dev/null +++ b/socket_plugin_test.py @@ -0,0 +1,40 @@ +import asyncio +import sys +from pathlib import Path + +# Add core and plugin src paths so imports work without installing packages +core_src = Path(__file__).parent / "core" / "src" +socket_src = Path(__file__).parent / "plugins" / "communication_protocols" / "socket" / "src" +sys.path.insert(0, str(core_src.resolve())) +sys.path.insert(0, str(socket_src.resolve())) + +from utcp.plugins.plugin_loader import ensure_plugins_initialized +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp.data.call_template import CallTemplateSerializer +from utcp_socket import register as register_socket + +async def main(): + # Manually register the socket plugin + register_socket() + + # Load core plugins (auth, repo, search, post-processors) + ensure_plugins_initialized() + + # 1. Check if communication protocols are registered + registered_protocols = CommunicationProtocol.communication_protocols + print(f"Registered communication protocols: {list(registered_protocols.keys())}") + assert "tcp" in registered_protocols, "TCP communication protocol not registered" + assert "udp" in registered_protocols, "UDP communication protocol not registered" + print("āœ… TCP and UDP communication protocols are registered.") + + # 2. Check if call templates are registered + registered_serializers = CallTemplateSerializer.call_template_serializers + print(f"Registered call template serializers: {list(registered_serializers.keys())}") + assert "tcp" in registered_serializers, "TCP call template serializer not registered" + assert "udp" in registered_serializers, "UDP call template serializer not registered" + print("āœ… TCP and UDP call template serializers are registered.") + + print("\nšŸŽ‰ Socket plugin sanity check passed! šŸŽ‰") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From dd2eb3c1085f4f1ad885f94583305e5988d0bd15 Mon Sep 17 00:00:00 2001 From: Thuraabtech <97426541+Thuraabtech@users.noreply.github.com> Date: Sat, 29 Nov 2025 08:25:43 -0600 Subject: [PATCH 02/10] GraphQL Plugin: UTCP 1.0 Migration (#75) * socket protocol updated to be compatible with 1.0v utcp * cubic fixes done * pinned mcp-use to use langchain 0.3.27 * removed mcp denpendency on langchain * adding the langchain dependency for testing (temporary) * remove langchain-core pin to resolve dependency conflict * feat: Updated Graphql implementation to be compatible with UTCP 1.0v * Added gql 'how to use' guide in the README.md * updated cubic comments for GraphQl * Update comment on delimeter handling Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Co-authored-by: Salman Mohammed Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- plugins/communication_protocols/gql/README.md | 48 +++- .../gql/src/utcp_gql/__init__.py | 9 + .../gql/src/utcp_gql/gql_call_template.py | 35 ++- .../utcp_gql/gql_communication_protocol.py | 257 ++++++++++++------ .../gql/tests/test_graphql_protocol.py | 110 ++++++++ .../mcp/pyproject.toml | 2 +- .../src/utcp_socket/tcp_call_template.py | 4 + .../utcp_socket/tcp_communication_protocol.py | 35 ++- .../utcp_socket/udp_communication_protocol.py | 18 +- .../tests/test_tcp_communication_protocol.py | 2 + scripts/socket_sanity.py | 6 +- 11 files changed, 421 insertions(+), 105 deletions(-) create mode 100644 plugins/communication_protocols/gql/tests/test_graphql_protocol.py diff --git a/plugins/communication_protocols/gql/README.md b/plugins/communication_protocols/gql/README.md index 8febb5a..34a2518 100644 --- a/plugins/communication_protocols/gql/README.md +++ b/plugins/communication_protocols/gql/README.md @@ -1 +1,47 @@ -Find the UTCP readme at https://github.com/universal-tool-calling-protocol/python-utcp. \ No newline at end of file + +# UTCP GraphQL Communication Protocol Plugin + +This plugin integrates GraphQL as a UTCP 1.0 communication protocol and call template. It supports discovery via schema introspection, authenticated calls, and header handling. + +## Getting Started + +### Installation + +```bash +pip install gql +``` + +### Registration + +```python +import utcp_gql +utcp_gql.register() +``` + +## How To Use + +- Ensure the plugin is imported and registered: `import utcp_gql; utcp_gql.register()`. +- Add a manual in your client config: + ```json + { + "name": "my_graph", + "call_template_type": "graphql", + "url": "https://your.graphql/endpoint", + "operation_type": "query", + "headers": { "x-client": "utcp" }, + "header_fields": ["x-session-id"] + } + ``` +- Call a tool: + ```python + await client.call_tool("my_graph.someQuery", {"id": "123", "x-session-id": "abc"}) + ``` + +## Notes + +- Tool names are prefixed by the manual name (e.g., `my_graph.someQuery`). +- Headers merge static `headers` plus whitelisted dynamic fields from `header_fields`. +- Supported auth: API key, Basic auth, OAuth2 (client-credentials). +- Security: only `https://` or `http://localhost`/`http://127.0.0.1` endpoints. + +For UTCP core docs, see https://github.com/universal-tool-calling-protocol/python-utcp. \ No newline at end of file diff --git a/plugins/communication_protocols/gql/src/utcp_gql/__init__.py b/plugins/communication_protocols/gql/src/utcp_gql/__init__.py index e69de29..7362502 100644 --- a/plugins/communication_protocols/gql/src/utcp_gql/__init__.py +++ b/plugins/communication_protocols/gql/src/utcp_gql/__init__.py @@ -0,0 +1,9 @@ +from utcp.plugins.discovery import register_communication_protocol, register_call_template + +from .gql_communication_protocol import GraphQLCommunicationProtocol +from .gql_call_template import GraphQLProvider, GraphQLProviderSerializer + + +def register(): + register_communication_protocol("graphql", GraphQLCommunicationProtocol()) + register_call_template("graphql", GraphQLProviderSerializer()) \ No newline at end of file diff --git a/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py b/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py index dfe5b07..3848d29 100644 --- a/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py +++ b/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py @@ -1,7 +1,10 @@ from utcp.data.call_template import CallTemplate -from utcp.data.auth import Auth +from utcp.data.auth import Auth, AuthSerializer +from utcp.interfaces.serializer import Serializer +from utcp.exceptions import UtcpSerializerValidationError +import traceback from typing import Dict, List, Optional, Literal -from pydantic import Field +from pydantic import Field, field_serializer, field_validator class GraphQLProvider(CallTemplate): """Provider configuration for GraphQL-based tools. @@ -27,3 +30,31 @@ class GraphQLProvider(CallTemplate): auth: Optional[Auth] = None headers: Optional[Dict[str, str]] = None header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers for the initial connection.") + + @field_serializer("auth") + def serialize_auth(self, auth: Optional[Auth]): + if auth is None: + return None + return AuthSerializer().to_dict(auth) + + @field_validator("auth", mode="before") + @classmethod + def validate_auth(cls, v: Optional[Auth | dict]): + if v is None: + return None + if isinstance(v, Auth): + return v + return AuthSerializer().validate_dict(v) + + +class GraphQLProviderSerializer(Serializer[GraphQLProvider]): + def to_dict(self, obj: GraphQLProvider) -> dict: + return obj.model_dump() + + def validate_dict(self, data: dict) -> GraphQLProvider: + try: + return GraphQLProvider.model_validate(data) + except Exception as e: + raise UtcpSerializerValidationError( + f"Invalid GraphQLProvider: {e}\n{traceback.format_exc()}" + ) diff --git a/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py b/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py index f27f803..9d26cab 100644 --- a/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py +++ b/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py @@ -1,36 +1,55 @@ -import sys -from typing import Dict, Any, List, Optional, Callable +import logging +from typing import Dict, Any, List, Optional, AsyncGenerator, TYPE_CHECKING + import aiohttp -import asyncio -import ssl from gql import Client as GqlClient, gql as gql_query from gql.transport.aiohttp import AIOHTTPTransport -from utcp.client.client_transport_interface import ClientTransportInterface -from utcp.shared.provider import Provider, GraphQLProvider -from utcp.shared.tool import Tool, ToolInputOutputSchema -from utcp.shared.auth import ApiKeyAuth, BasicAuth, OAuth2Auth -import logging + +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp.data.call_template import CallTemplate +from utcp.data.tool import Tool, JsonSchema +from utcp.data.utcp_manual import UtcpManual +from utcp.data.register_manual_response import RegisterManualResult +from utcp.data.auth_implementations.api_key_auth import ApiKeyAuth +from utcp.data.auth_implementations.basic_auth import BasicAuth +from utcp.data.auth_implementations.oauth2_auth import OAuth2Auth + +from utcp_gql.gql_call_template import GraphQLProvider + +if TYPE_CHECKING: + from utcp.utcp_client import UtcpClient + logging.basicConfig( level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d - %(message)s" + format="%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d - %(message)s", ) logger = logging.getLogger(__name__) -class GraphQLClientTransport(ClientTransportInterface): - """ - Simple, robust, production-ready GraphQL transport using gql. - Stateless, per-operation. Supports all GraphQL features. + +class GraphQLCommunicationProtocol(CommunicationProtocol): + """GraphQL protocol implementation for UTCP 1.0. + + - Discovers tools via GraphQL schema introspection. + - Executes per-call sessions using `gql` over HTTP(S). + - Supports `ApiKeyAuth`, `BasicAuth`, and `OAuth2Auth`. + - Enforces HTTPS or localhost for security. """ - def __init__(self): + + def __init__(self) -> None: self._oauth_tokens: Dict[str, Dict[str, Any]] = {} - def _enforce_https_or_localhost(self, url: str): - if not (url.startswith("https://") or url.startswith("http://localhost") or url.startswith("http://127.0.0.1")): + def _enforce_https_or_localhost(self, url: str) -> None: + if not ( + url.startswith("https://") + or url.startswith("http://localhost") + or url.startswith("http://127.0.0.1") + ): raise ValueError( - f"Security error: URL must use HTTPS or start with 'http://localhost' or 'http://127.0.0.1'. Got: {url}. " - "Non-secure URLs are vulnerable to man-in-the-middle attacks." + "Security error: URL must use HTTPS or start with 'http://localhost' or 'http://127.0.0.1'. " + "Non-secure URLs are vulnerable to man-in-the-middle attacks. " + f"Got: {url}." ) async def _handle_oauth2(self, auth: OAuth2Auth) -> str: @@ -39,10 +58,10 @@ async def _handle_oauth2(self, auth: OAuth2Auth) -> str: return self._oauth_tokens[client_id]["access_token"] async with aiohttp.ClientSession() as session: data = { - 'grant_type': 'client_credentials', - 'client_id': client_id, - 'client_secret': auth.client_secret, - 'scope': auth.scope + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": auth.client_secret, + "scope": auth.scope, } async with session.post(auth.token_url, data=data) as resp: resp.raise_for_status() @@ -50,87 +69,147 @@ async def _handle_oauth2(self, auth: OAuth2Auth) -> str: self._oauth_tokens[client_id] = token_response return token_response["access_token"] - async def _prepare_headers(self, provider: GraphQLProvider) -> Dict[str, str]: - headers = provider.headers.copy() if provider.headers else {} + async def _prepare_headers( + self, provider: GraphQLProvider, tool_args: Optional[Dict[str, Any]] = None + ) -> Dict[str, str]: + headers: Dict[str, str] = provider.headers.copy() if provider.headers else {} if provider.auth: if isinstance(provider.auth, ApiKeyAuth): - if provider.auth.api_key: - if provider.auth.location == "header": - headers[provider.auth.var_name] = provider.auth.api_key - # (query/cookie not supported for GraphQL by default) + if provider.auth.api_key and provider.auth.location == "header": + headers[provider.auth.var_name] = provider.auth.api_key elif isinstance(provider.auth, BasicAuth): import base64 + userpass = f"{provider.auth.username}:{provider.auth.password}" headers["Authorization"] = "Basic " + base64.b64encode(userpass.encode()).decode() elif isinstance(provider.auth, OAuth2Auth): token = await self._handle_oauth2(provider.auth) headers["Authorization"] = f"Bearer {token}" + + # Map selected tool_args into headers if requested + if tool_args and provider.header_fields: + for field in provider.header_fields: + if field in tool_args and isinstance(tool_args[field], str): + headers[field] = tool_args[field] + return headers - async def register_tool_provider(self, manual_provider: Provider) -> List[Tool]: - if not isinstance(manual_provider, GraphQLProvider): - raise ValueError("GraphQLClientTransport can only be used with GraphQLProvider") - self._enforce_https_or_localhost(manual_provider.url) - headers = await self._prepare_headers(manual_provider) - transport = AIOHTTPTransport(url=manual_provider.url, headers=headers) - async with GqlClient(transport=transport, fetch_schema_from_transport=True) as session: - schema = session.client.schema - tools = [] - # Queries - if hasattr(schema, 'query_type') and schema.query_type: - for name, field in schema.query_type.fields.items(): - tools.append(Tool( - name=name, - description=getattr(field, 'description', '') or '', - inputs=ToolInputOutputSchema(required=None), - tool_provider=manual_provider - )) - # Mutations - if hasattr(schema, 'mutation_type') and schema.mutation_type: - for name, field in schema.mutation_type.fields.items(): - tools.append(Tool( - name=name, - description=getattr(field, 'description', '') or '', - inputs=ToolInputOutputSchema(required=None), - tool_provider=manual_provider - )) - # Subscriptions (listed, but not called here) - if hasattr(schema, 'subscription_type') and schema.subscription_type: - for name, field in schema.subscription_type.fields.items(): - tools.append(Tool( - name=name, - description=getattr(field, 'description', '') or '', - inputs=ToolInputOutputSchema(required=None), - tool_provider=manual_provider - )) - return tools - - async def deregister_tool_provider(self, manual_provider: Provider) -> None: - # Stateless: nothing to do - pass - - async def call_tool(self, tool_name: str, tool_args: Dict[str, Any], tool_provider: Provider, query: Optional[str] = None) -> Any: - if not isinstance(tool_provider, GraphQLProvider): - raise ValueError("GraphQLClientTransport can only be used with GraphQLProvider") - self._enforce_https_or_localhost(tool_provider.url) - headers = await self._prepare_headers(tool_provider) - transport = AIOHTTPTransport(url=tool_provider.url, headers=headers) + async def register_manual( + self, caller: "UtcpClient", manual_call_template: CallTemplate + ) -> RegisterManualResult: + if not isinstance(manual_call_template, GraphQLProvider): + raise ValueError("GraphQLCommunicationProtocol requires a GraphQLProvider call template") + self._enforce_https_or_localhost(manual_call_template.url) + + try: + headers = await self._prepare_headers(manual_call_template) + transport = AIOHTTPTransport(url=manual_call_template.url, headers=headers) + async with GqlClient(transport=transport, fetch_schema_from_transport=True) as session: + schema = session.client.schema + tools: List[Tool] = [] + + # Queries + if hasattr(schema, "query_type") and schema.query_type: + for name, field in schema.query_type.fields.items(): + tools.append( + Tool( + name=name, + description=getattr(field, "description", "") or "", + inputs=JsonSchema(type="object"), + outputs=JsonSchema(type="object"), + tool_call_template=manual_call_template, + ) + ) + + # Mutations + if hasattr(schema, "mutation_type") and schema.mutation_type: + for name, field in schema.mutation_type.fields.items(): + tools.append( + Tool( + name=name, + description=getattr(field, "description", "") or "", + inputs=JsonSchema(type="object"), + outputs=JsonSchema(type="object"), + tool_call_template=manual_call_template, + ) + ) + + # Subscriptions (listed for completeness) + if hasattr(schema, "subscription_type") and schema.subscription_type: + for name, field in schema.subscription_type.fields.items(): + tools.append( + Tool( + name=name, + description=getattr(field, "description", "") or "", + inputs=JsonSchema(type="object"), + outputs=JsonSchema(type="object"), + tool_call_template=manual_call_template, + ) + ) + + manual = UtcpManual(tools=tools) + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=manual, + success=True, + errors=[], + ) + except Exception as e: + logger.error(f"GraphQL manual registration failed for '{manual_call_template.name}': {e}") + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=UtcpManual(manual_version="0.0.0", tools=[]), + success=False, + errors=[str(e)], + ) + + async def deregister_manual( + self, caller: "UtcpClient", manual_call_template: CallTemplate + ) -> None: + # Stateless: nothing to clean up + return None + + async def call_tool( + self, + caller: "UtcpClient", + tool_name: str, + tool_args: Dict[str, Any], + tool_call_template: CallTemplate, + ) -> Any: + if not isinstance(tool_call_template, GraphQLProvider): + raise ValueError("GraphQLCommunicationProtocol requires a GraphQLProvider call template") + self._enforce_https_or_localhost(tool_call_template.url) + + headers = await self._prepare_headers(tool_call_template, tool_args) + transport = AIOHTTPTransport(url=tool_call_template.url, headers=headers) async with GqlClient(transport=transport, fetch_schema_from_transport=True) as session: - if query is not None: - document = gql_query(query) - result = await session.execute(document, variable_values=tool_args) - return result - # If no query provided, build a simple query - # Default to query operation - op_type = getattr(tool_provider, 'operation_type', 'query') - arg_str = ', '.join(f"${k}: String" for k in tool_args.keys()) + op_type = getattr(tool_call_template, "operation_type", "query") + # Strip manual prefix if present (client prefixes at save time) + base_tool_name = tool_name.split(".", 1)[-1] if "." in tool_name else tool_name + # Filter out header fields from GraphQL variables; these are sent via HTTP headers + header_fields = tool_call_template.header_fields or [] + filtered_args = {k: v for k, v in tool_args.items() if k not in header_fields} + + arg_str = ", ".join(f"${k}: String" for k in filtered_args.keys()) var_defs = f"({arg_str})" if arg_str else "" - arg_pass = ', '.join(f"{k}: ${k}" for k in tool_args.keys()) + arg_pass = ", ".join(f"{k}: ${k}" for k in filtered_args.keys()) arg_pass = f"({arg_pass})" if arg_pass else "" - gql_str = f"{op_type} {var_defs} {{ {tool_name}{arg_pass} }}" + + gql_str = f"{op_type} {var_defs} {{ {base_tool_name}{arg_pass} }}" document = gql_query(gql_str) - result = await session.execute(document, variable_values=tool_args) + result = await session.execute(document, variable_values=filtered_args) return result + async def call_tool_streaming( + self, + caller: "UtcpClient", + tool_name: str, + tool_args: Dict[str, Any], + tool_call_template: CallTemplate, + ) -> AsyncGenerator[Any, None]: + # Basic implementation: execute non-streaming and yield once + result = await self.call_tool(caller, tool_name, tool_args, tool_call_template) + yield result + async def close(self) -> None: - self._oauth_tokens.clear() + self._oauth_tokens.clear() \ No newline at end of file diff --git a/plugins/communication_protocols/gql/tests/test_graphql_protocol.py b/plugins/communication_protocols/gql/tests/test_graphql_protocol.py new file mode 100644 index 0000000..1b1bb74 --- /dev/null +++ b/plugins/communication_protocols/gql/tests/test_graphql_protocol.py @@ -0,0 +1,110 @@ +import os +import sys +import types +import pytest + + +# Ensure plugin src is importable +PLUGIN_SRC = os.path.join(os.path.dirname(__file__), "..", "src") +PLUGIN_SRC = os.path.abspath(PLUGIN_SRC) +if PLUGIN_SRC not in sys.path: + sys.path.append(PLUGIN_SRC) + +import utcp_gql +# Simplify imports: use the main module and assign local aliases +GraphQLProvider = utcp_gql.gql_call_template.GraphQLProvider +gql_module = utcp_gql.gql_communication_protocol + +from utcp.data.utcp_manual import UtcpManual +from utcp.utcp_client import UtcpClient +from utcp.implementations.utcp_client_implementation import UtcpClientImplementation + + +class FakeSchema: + def __init__(self): + # Minimal field objects with descriptions + self.query_type = types.SimpleNamespace( + fields={ + "hello": types.SimpleNamespace(description="Returns greeting"), + } + ) + self.mutation_type = types.SimpleNamespace( + fields={ + "add": types.SimpleNamespace(description="Adds numbers"), + } + ) + self.subscription_type = None + + +class FakeClientObj: + def __init__(self): + self.client = types.SimpleNamespace(schema=FakeSchema()) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def execute(self, document, variable_values=None): + # document is a gql query; we can base behavior on variable_values + variable_values = variable_values or {} + # Determine operation by presence of variables used + if "hello" in str(document): + name = variable_values.get("name", "") + return {"hello": f"Hello {name}"} + if "add" in str(document): + a = int(variable_values.get("a", 0)) + b = int(variable_values.get("b", 0)) + return {"add": a + b} + return {"ok": True} + + +class FakeTransport: + def __init__(self, url: str, headers: dict | None = None): + self.url = url + self.headers = headers or {} + + +@pytest.mark.asyncio +async def test_graphql_register_and_call(monkeypatch): + # Patch gql client/transport used by protocol to avoid needing a real server + monkeypatch.setattr(gql_module, "GqlClient", lambda *args, **kwargs: FakeClientObj()) + monkeypatch.setattr(gql_module, "AIOHTTPTransport", FakeTransport) + # Avoid real GraphQL parsing; pass-through document string to fake execute + monkeypatch.setattr(gql_module, "gql_query", lambda s: s) + + # Register plugin (call_template serializer + protocol) + utcp_gql.register() + + # Create protocol and manual call template + protocol = gql_module.GraphQLCommunicationProtocol() + provider = GraphQLProvider( + name="mock_graph", + call_template_type="graphql", + url="http://localhost/graphql", + operation_type="query", + headers={"x-client": "utcp"}, + header_fields=["x-session-id"], + ) + + # Minimal UTCP client implementation for caller context + client: UtcpClient = await UtcpClientImplementation.create() + client.config.variables = {} + + # Register and discover tools + reg = await protocol.register_manual(client, provider) + assert reg.success is True + assert isinstance(reg.manual, UtcpManual) + tool_names = sorted(t.name for t in reg.manual.tools) + assert "hello" in tool_names + assert "add" in tool_names + + # Call hello + res = await protocol.call_tool(client, "mock_graph.hello", {"name": "UTCP", "x-session-id": "abc"}, provider) + assert res == {"hello": "Hello UTCP"} + + # Call add (mutation) + provider.operation_type = "mutation" + res2 = await protocol.call_tool(client, "mock_graph.add", {"a": 2, "b": 3}, provider) + assert res2 == {"add": 5} \ No newline at end of file diff --git a/plugins/communication_protocols/mcp/pyproject.toml b/plugins/communication_protocols/mcp/pyproject.toml index 9232cd5..2efd4c3 100644 --- a/plugins/communication_protocols/mcp/pyproject.toml +++ b/plugins/communication_protocols/mcp/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "mcp>=1.12", "utcp>=1.0", "mcp-use>=1.3", - "langchain==0.3.27", + "langchain>=0.3.27,<0.4.0", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py b/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py index 10fc1d6..8b27d1c 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/tcp_call_template.py @@ -68,6 +68,10 @@ class TCPProvider(CallTemplate): default='\x00', description="Delimiter to detect end of TCP response (e.g., '\n', '\r\n', '\x00'). Used with 'delimiter' framing." ) + interpret_escape_sequences: bool = Field( + default=True, + description="If True, interpret Python-style escape sequences in message_delimiter (e.g., '\\n', '\\r\\n', '\\x00'). If False, use the delimiter literally as provided." + ) # Fixed-length framing options fixed_message_length: Optional[int] = Field( default=None, diff --git a/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py b/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py index d5d64ac..b2f08c3 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/tcp_communication_protocol.py @@ -148,9 +148,14 @@ def _encode_message_with_framing(self, message: str, provider: TCPProvider) -> b elif provider.framing_strategy == "delimiter": # Add delimiter after the message delimiter = provider.message_delimiter or "\x00" - # Handle escape sequences - delimiter = delimiter.encode('utf-8').decode('unicode_escape') - return message_bytes + delimiter.encode('utf-8') + if provider.interpret_escape_sequences: + # Handle escape sequences (e.g., "\n", "\r\n", "\x00") + delimiter = delimiter.encode('utf-8').decode('unicode_escape') + delimiter_bytes = delimiter.encode('utf-8') + else: + # Use delimiter literally as provided + delimiter_bytes = delimiter.encode('utf-8') + return message_bytes + delimiter_bytes elif provider.framing_strategy in ("fixed_length", "stream"): # No additional framing needed @@ -202,8 +207,19 @@ def _decode_response_with_framing(self, sock: socket.socket, provider: TCPProvid elif provider.framing_strategy == "delimiter": # Read until delimiter is found + # Delimiter handling: + # The code supports both literal delimiters (e.g., "\\x00") and escape-sequence interpreted delimiters (e.g., "\x00") + # via the `interpret_escape_sequences` flag in TCPProvider. This ensures compatibility with both legacy and updated + # wire protocols. The delimiter is interpreted according to the flag, so no breaking change occurs unless the flag + # is set differently than expected by the server/client. + # Example: + # If interpret_escape_sequences is True, "\\x00" becomes a null byte; if False, it remains four literal bytes. + # delimiter = delimiter.encode('utf-8') delimiter = provider.message_delimiter or "\x00" - delimiter = delimiter.encode('utf-8').decode('unicode_escape').encode('utf-8') + if provider.interpret_escape_sequences: + delimiter_bytes = delimiter.encode('utf-8').decode('unicode_escape').encode('utf-8') + else: + delimiter_bytes = delimiter.encode('utf-8') response_data = b"" while True: @@ -213,9 +229,9 @@ def _decode_response_with_framing(self, sock: socket.socket, provider: TCPProvid response_data += chunk # Check if we've received the delimiter - if response_data.endswith(delimiter): + if response_data.endswith(delimiter_bytes): # Remove delimiter from response - return response_data[:-len(delimiter)] + return response_data[:-len(delimiter_bytes)] elif provider.framing_strategy == "fixed_length": # Read exactly fixed_message_length bytes @@ -246,6 +262,13 @@ def _decode_response_with_framing(self, sock: socket.socket, provider: TCPProvid break return response_data + + else: + # Copilot AI (5 days ago): + # The else branch for unknown framing strategies was previously removed, + # which could cause silent fallthrough and confusing behavior. Add explicit + # validation to raise a descriptive error when an unsupported strategy is provided. + raise ValueError(f"Unknown framing strategy: {provider.framing_strategy!r}") async def _send_tcp_message( self, diff --git a/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py b/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py index b59ef37..89ae3e3 100644 --- a/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py +++ b/plugins/communication_protocols/socket/src/utcp_socket/udp_communication_protocol.py @@ -15,6 +15,7 @@ from utcp.data.call_template import CallTemplate, CallTemplateSerializer from utcp.data.register_manual_response import RegisterManualResult from utcp.data.utcp_manual import UtcpManual +from utcp.exceptions import UtcpSerializerValidationError import logging logger = logging.getLogger(__name__) @@ -98,8 +99,9 @@ def _ensure_tool_call_template(self, tool_data: Dict[str, Any], manual_call_temp try: ctpl = CallTemplateSerializer().validate_dict(normalized["tool_call_template"]) # type: ignore normalized["tool_call_template"] = ctpl - except Exception: - # Fallback to manual template if validation fails + except (UtcpSerializerValidationError, ValueError) as e: + # Fallback to manual template if validation fails, but log details + logger.exception("Failed to validate existing tool_call_template; falling back to manual template") normalized["tool_call_template"] = manual_call_template elif "tool_provider" in normalized and normalized["tool_provider"] is not None: # Convert legacy provider -> call template @@ -107,12 +109,15 @@ def _ensure_tool_call_template(self, tool_data: Dict[str, Any], manual_call_temp ctpl = UDPProviderSerializer().validate_dict(normalized["tool_provider"]) # type: ignore normalized.pop("tool_provider", None) normalized["tool_call_template"] = ctpl - except Exception: + except UtcpSerializerValidationError as e: + logger.exception("Failed to convert legacy tool_provider to call template; falling back to manual template") normalized.pop("tool_provider", None) normalized["tool_call_template"] = manual_call_template else: normalized["tool_call_template"] = manual_call_template except Exception: + # Any unexpected error during normalization should be logged + logger.exception("Unexpected error normalizing tool definition; falling back to manual template") normalized["tool_call_template"] = manual_call_template return normalized @@ -321,5 +326,12 @@ async def call_tool(self, caller, tool_name: str, tool_args: Dict[str, Any], too self._log_error(f"Error calling UDP tool '{tool_name}': {traceback.format_exc()}") raise + # Copilot AI (5 days ago): + # The call_tool_streaming method wraps a generator function but doesn't use the async def syntax for the method itself. + # While this works, it's inconsistent with the other implementation in tcp_communication_protocol.py (lines 384-387) which properly uses async def with an inner generator. + # For consistency and clarity, this should also use async def directly: + # + # async def call_tool_streaming(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate): + # yield await self.call_tool(caller, tool_name, tool_args, tool_call_template) async def call_tool_streaming(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate): yield await self.call_tool(caller, tool_name, tool_args, tool_call_template) diff --git a/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py b/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py index 1b6ffb2..d359fd9 100644 --- a/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py +++ b/plugins/communication_protocols/socket/tests/test_tcp_communication_protocol.py @@ -15,6 +15,7 @@ async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): # Read any incoming data to simulate request handling await reader.read(1024) except Exception: + # Ignore exceptions during read (e.g., client disconnects), as this is a test server. pass # Send response and close connection writer.write(response_container["bytes"]) @@ -23,6 +24,7 @@ async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): writer.close() await writer.wait_closed() except Exception: + # Ignore exceptions during writer close; connection may already be closed or in error state. pass server = await asyncio.start_server(handle, host="127.0.0.1", port=0) diff --git a/scripts/socket_sanity.py b/scripts/socket_sanity.py index 40b2c16..5ac6028 100644 --- a/scripts/socket_sanity.py +++ b/scripts/socket_sanity.py @@ -1,7 +1,5 @@ import sys -import os import json -import time import socket import threading import asyncio @@ -39,6 +37,7 @@ def run(): try: parsed = json.loads(msg) except Exception: + # Ignore JSON parsing errors; non-JSON input will be handled below parsed = None if isinstance(parsed, dict) and parsed.get("type") == "utcp": manual = { @@ -182,6 +181,7 @@ def handle_client(conn: socket.socket, addr): try: conn.shutdown(socket.SHUT_RDWR) except Exception: + # Ignore errors if socket is already closed or shutdown fails pass conn.close() @@ -259,7 +259,7 @@ def ensure_dict(s): assert udp_resp.get("ok") is True and udp_resp.get("echo") == "hello" assert tcp_resp.get("ok") is True and tcp_resp.get("echo") == "world" - print("Sanity passed: UDP/TCP discovery and calls work with tool_call_template normalization.") + print("Sanity check passed: UDP/TCP discovery and calls work with tool_call_template normalization.") if __name__ == "__main__": asyncio.run(run_sanity()) \ No newline at end of file From 7e1d47da1fa42b8b424ab66ceeec5d162b1d0602 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:09:22 +0100 Subject: [PATCH 03/10] update --- .../gql/old_tests/test_graphql_transport.py | 129 -------- .../gql/src/utcp_gql/__init__.py | 4 +- .../gql/src/utcp_gql/gql_call_template.py | 47 ++- .../utcp_gql/gql_communication_protocol.py | 64 ++-- .../gql/tests/test_graphql_integration.py | 275 ++++++++++++++++++ .../gql/tests/test_graphql_protocol.py | 110 ------- .../communication_protocols/socket/README.md | 4 +- .../websocket/tests/__init__.py | 1 - socket_plugin_test.py | 40 --- test_websocket_manual.py | 201 ------------- 10 files changed, 358 insertions(+), 517 deletions(-) delete mode 100644 plugins/communication_protocols/gql/old_tests/test_graphql_transport.py create mode 100644 plugins/communication_protocols/gql/tests/test_graphql_integration.py delete mode 100644 plugins/communication_protocols/gql/tests/test_graphql_protocol.py delete mode 100644 plugins/communication_protocols/websocket/tests/__init__.py delete mode 100644 socket_plugin_test.py delete mode 100644 test_websocket_manual.py diff --git a/plugins/communication_protocols/gql/old_tests/test_graphql_transport.py b/plugins/communication_protocols/gql/old_tests/test_graphql_transport.py deleted file mode 100644 index d33c323..0000000 --- a/plugins/communication_protocols/gql/old_tests/test_graphql_transport.py +++ /dev/null @@ -1,129 +0,0 @@ -# import pytest -# import pytest_asyncio -# import json -# from aiohttp import web -# from utcp.client.transport_interfaces.graphql_transport import GraphQLClientTransport -# from utcp.shared.provider import GraphQLProvider -# from utcp.shared.auth import ApiKeyAuth, BasicAuth, OAuth2Auth - - -# @pytest_asyncio.fixture -# async def graphql_app(): -# async def graphql_handler(request): -# body = await request.json() -# query = body.get("query", "") -# variables = body.get("variables", {}) -# # Introspection query (minimal response) -# if "__schema" in query: -# return web.json_response({ -# "data": { -# "__schema": { -# "queryType": {"name": "Query"}, -# "mutationType": {"name": "Mutation"}, -# "subscriptionType": None, -# "types": [ -# {"kind": "OBJECT", "name": "Query", "fields": [ -# {"name": "hello", "args": [{"name": "name", "type": {"kind": "SCALAR", "name": "String"}, "defaultValue": None}], "type": {"kind": "SCALAR", "name": "String"}, "isDeprecated": False, "deprecationReason": None} -# ], "interfaces": []}, -# {"kind": "OBJECT", "name": "Mutation", "fields": [ -# {"name": "add", "args": [ -# {"name": "a", "type": {"kind": "SCALAR", "name": "Int"}, "defaultValue": None}, -# {"name": "b", "type": {"kind": "SCALAR", "name": "Int"}, "defaultValue": None} -# ], "type": {"kind": "SCALAR", "name": "Int"}, "isDeprecated": False, "deprecationReason": None} -# ], "interfaces": []}, -# {"kind": "SCALAR", "name": "String"}, -# {"kind": "SCALAR", "name": "Int"}, -# {"kind": "SCALAR", "name": "Boolean"} -# ], -# "directives": [] -# } -# } -# }) -# # hello query -# if "hello" in query: -# name = variables.get("name", "world") -# return web.json_response({"data": {"hello": f"Hello, {name}!"}}) -# # add mutation -# if "add" in query: -# a = variables.get("a", 0) -# b = variables.get("b", 0) -# return web.json_response({"data": {"add": a + b}}) -# # fallback -# return web.json_response({"data": {}}, status=200) - -# app = web.Application() -# app.router.add_post("/graphql", graphql_handler) -# return app - -# @pytest_asyncio.fixture -# async def aiohttp_graphql_client(aiohttp_client, graphql_app): -# return await aiohttp_client(graphql_app) - -# @pytest_asyncio.fixture -# def transport(): -# return GraphQLClientTransport() - -# @pytest_asyncio.fixture -# def provider(aiohttp_graphql_client): -# return GraphQLProvider( -# name="test-graphql-provider", -# url=str(aiohttp_graphql_client.make_url("/graphql")), -# headers={}, -# ) - -# @pytest.mark.asyncio -# async def test_register_tool_provider_discovers_tools(transport, provider): -# tools = await transport.register_tool_provider(provider) -# tool_names = [tool.name for tool in tools] -# assert "hello" in tool_names -# assert "add" in tool_names - -# @pytest.mark.asyncio -# async def test_call_tool_query(transport, provider): -# result = await transport.call_tool("hello", {"name": "Alice"}, provider) -# assert result["hello"] == "Hello, Alice!" - -# @pytest.mark.asyncio -# async def test_call_tool_mutation(transport, provider): -# provider.operation_type = "mutation" -# mutation = ''' -# mutation ($a: Int, $b: Int) { -# add(a: $a, b: $b) -# } -# ''' -# result = await transport.call_tool("add", {"a": 2, "b": 3}, provider, query=mutation) -# assert result["add"] == 5 - -# @pytest.mark.asyncio -# async def test_call_tool_api_key(transport, provider): -# provider.headers = {} -# provider.auth = ApiKeyAuth(var_name="X-API-Key", api_key="test-key") -# result = await transport.call_tool("hello", {"name": "Bob"}, provider) -# assert result["hello"] == "Hello, Bob!" - -# @pytest.mark.asyncio -# async def test_call_tool_basic_auth(transport, provider): -# provider.headers = {} -# provider.auth = BasicAuth(username="user", password="pass") -# result = await transport.call_tool("hello", {"name": "Eve"}, provider) -# assert result["hello"] == "Hello, Eve!" - -# @pytest.mark.asyncio -# async def test_call_tool_oauth2(monkeypatch, transport, provider): -# async def fake_oauth2(auth): -# return "fake-token" -# transport._handle_oauth2 = fake_oauth2 -# provider.headers = {} -# provider.auth = OAuth2Auth(token_url="http://fake/token", client_id="id", client_secret="secret", scope="scope") -# result = await transport.call_tool("hello", {"name": "Zoe"}, provider) -# assert result["hello"] == "Hello, Zoe!" - -# @pytest.mark.asyncio -# async def test_enforce_https_or_localhost_raises(transport, provider): -# provider.url = "http://evil.com/graphql" -# with pytest.raises(ValueError): -# await transport.call_tool("hello", {"name": "Mallory"}, provider) - -# @pytest.mark.asyncio -# async def test_deregister_tool_provider_noop(transport, provider): -# await transport.deregister_tool_provider(provider) \ No newline at end of file diff --git a/plugins/communication_protocols/gql/src/utcp_gql/__init__.py b/plugins/communication_protocols/gql/src/utcp_gql/__init__.py index 7362502..6dd0fda 100644 --- a/plugins/communication_protocols/gql/src/utcp_gql/__init__.py +++ b/plugins/communication_protocols/gql/src/utcp_gql/__init__.py @@ -1,9 +1,9 @@ from utcp.plugins.discovery import register_communication_protocol, register_call_template from .gql_communication_protocol import GraphQLCommunicationProtocol -from .gql_call_template import GraphQLProvider, GraphQLProviderSerializer +from .gql_call_template import GraphQLCallTemplate, GraphQLCallTemplateSerializer def register(): register_communication_protocol("graphql", GraphQLCommunicationProtocol()) - register_call_template("graphql", GraphQLProviderSerializer()) \ No newline at end of file + register_call_template("graphql", GraphQLCallTemplateSerializer()) \ No newline at end of file diff --git a/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py b/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py index 3848d29..579d691 100644 --- a/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py +++ b/plugins/communication_protocols/gql/src/utcp_gql/gql_call_template.py @@ -6,13 +6,17 @@ from typing import Dict, List, Optional, Literal from pydantic import Field, field_serializer, field_validator -class GraphQLProvider(CallTemplate): +class GraphQLCallTemplate(CallTemplate): """Provider configuration for GraphQL-based tools. Enables communication with GraphQL endpoints supporting queries, mutations, and subscriptions. Provides flexible query execution with custom headers and authentication. + For maximum flexibility, use the `query` field to provide a complete GraphQL + query string with proper selection sets and variable types. This allows agents + to call any existing GraphQL endpoint without limitations. + Attributes: call_template_type: Always "graphql" for GraphQL providers. url: The GraphQL endpoint URL. @@ -21,6 +25,23 @@ class GraphQLProvider(CallTemplate): auth: Optional authentication configuration. headers: Optional static headers to include in requests. header_fields: List of tool argument names to map to HTTP request headers. + query: Custom GraphQL query string with full control over selection sets + and variable types. Example: 'query GetUser($id: ID!) { user(id: $id) { id name } }' + variable_types: Map of variable names to GraphQL types for auto-generated queries. + Example: {'id': 'ID!', 'limit': 'Int'}. Defaults to 'String' if not specified. + + Example: + # Full flexibility with custom query + template = GraphQLCallTemplate( + url="https://api.example.com/graphql", + query="query GetUser($id: ID!) { user(id: $id) { id name email } }", + ) + + # Auto-generation with proper types + template = GraphQLCallTemplate( + url="https://api.example.com/graphql", + variable_types={"limit": "Int", "active": "Boolean"}, + ) """ call_template_type: Literal["graphql"] = "graphql" @@ -30,6 +51,18 @@ class GraphQLProvider(CallTemplate): auth: Optional[Auth] = None headers: Optional[Dict[str, str]] = None header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers for the initial connection.") + query: Optional[str] = Field( + default=None, + description="Custom GraphQL query/mutation string. Use $varName syntax for variables. " + "If provided, this takes precedence over auto-generation. " + "Example: 'query GetUser($id: ID!) { user(id: $id) { id name email } }'" + ) + variable_types: Optional[Dict[str, str]] = Field( + default=None, + description="Map of variable names to GraphQL types for auto-generated queries. " + "Example: {'id': 'ID!', 'limit': 'Int', 'active': 'Boolean'}. " + "Defaults to 'String' if not specified." + ) @field_serializer("auth") def serialize_auth(self, auth: Optional[Auth]): @@ -47,14 +80,14 @@ def validate_auth(cls, v: Optional[Auth | dict]): return AuthSerializer().validate_dict(v) -class GraphQLProviderSerializer(Serializer[GraphQLProvider]): - def to_dict(self, obj: GraphQLProvider) -> dict: +class GraphQLCallTemplateSerializer(Serializer[GraphQLCallTemplate]): + def to_dict(self, obj: GraphQLCallTemplate) -> dict: return obj.model_dump() - def validate_dict(self, data: dict) -> GraphQLProvider: + def validate_dict(self, data: dict) -> GraphQLCallTemplate: try: - return GraphQLProvider.model_validate(data) + return GraphQLCallTemplate.model_validate(data) except Exception as e: raise UtcpSerializerValidationError( - f"Invalid GraphQLProvider: {e}\n{traceback.format_exc()}" - ) + f"Invalid GraphQLCallTemplate: {e}\n{traceback.format_exc()}" + ) \ No newline at end of file diff --git a/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py b/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py index 9d26cab..16b945c 100644 --- a/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py +++ b/plugins/communication_protocols/gql/src/utcp_gql/gql_communication_protocol.py @@ -14,7 +14,7 @@ from utcp.data.auth_implementations.basic_auth import BasicAuth from utcp.data.auth_implementations.oauth2_auth import OAuth2Auth -from utcp_gql.gql_call_template import GraphQLProvider +from utcp_gql.gql_call_template import GraphQLCallTemplate if TYPE_CHECKING: from utcp.utcp_client import UtcpClient @@ -70,25 +70,25 @@ async def _handle_oauth2(self, auth: OAuth2Auth) -> str: return token_response["access_token"] async def _prepare_headers( - self, provider: GraphQLProvider, tool_args: Optional[Dict[str, Any]] = None + self, call_template: GraphQLCallTemplate, tool_args: Optional[Dict[str, Any]] = None ) -> Dict[str, str]: - headers: Dict[str, str] = provider.headers.copy() if provider.headers else {} - if provider.auth: - if isinstance(provider.auth, ApiKeyAuth): - if provider.auth.api_key and provider.auth.location == "header": - headers[provider.auth.var_name] = provider.auth.api_key - elif isinstance(provider.auth, BasicAuth): + headers: Dict[str, str] = call_template.headers.copy() if call_template.headers else {} + if call_template.auth: + if isinstance(call_template.auth, ApiKeyAuth): + if call_template.auth.api_key and call_template.auth.location == "header": + headers[call_template.auth.var_name] = call_template.auth.api_key + elif isinstance(call_template.auth, BasicAuth): import base64 - userpass = f"{provider.auth.username}:{provider.auth.password}" + userpass = f"{call_template.auth.username}:{call_template.auth.password}" headers["Authorization"] = "Basic " + base64.b64encode(userpass.encode()).decode() - elif isinstance(provider.auth, OAuth2Auth): - token = await self._handle_oauth2(provider.auth) + elif isinstance(call_template.auth, OAuth2Auth): + token = await self._handle_oauth2(call_template.auth) headers["Authorization"] = f"Bearer {token}" # Map selected tool_args into headers if requested - if tool_args and provider.header_fields: - for field in provider.header_fields: + if tool_args and call_template.header_fields: + for field in call_template.header_fields: if field in tool_args and isinstance(tool_args[field], str): headers[field] = tool_args[field] @@ -97,8 +97,8 @@ async def _prepare_headers( async def register_manual( self, caller: "UtcpClient", manual_call_template: CallTemplate ) -> RegisterManualResult: - if not isinstance(manual_call_template, GraphQLProvider): - raise ValueError("GraphQLCommunicationProtocol requires a GraphQLProvider call template") + if not isinstance(manual_call_template, GraphQLCallTemplate): + raise ValueError("GraphQLCommunicationProtocol requires a GraphQLCallTemplate call template") self._enforce_https_or_localhost(manual_call_template.url) try: @@ -176,26 +176,40 @@ async def call_tool( tool_args: Dict[str, Any], tool_call_template: CallTemplate, ) -> Any: - if not isinstance(tool_call_template, GraphQLProvider): - raise ValueError("GraphQLCommunicationProtocol requires a GraphQLProvider call template") + if not isinstance(tool_call_template, GraphQLCallTemplate): + raise ValueError("GraphQLCommunicationProtocol requires a GraphQLCallTemplate call template") self._enforce_https_or_localhost(tool_call_template.url) headers = await self._prepare_headers(tool_call_template, tool_args) transport = AIOHTTPTransport(url=tool_call_template.url, headers=headers) async with GqlClient(transport=transport, fetch_schema_from_transport=True) as session: - op_type = getattr(tool_call_template, "operation_type", "query") - # Strip manual prefix if present (client prefixes at save time) - base_tool_name = tool_name.split(".", 1)[-1] if "." in tool_name else tool_name # Filter out header fields from GraphQL variables; these are sent via HTTP headers header_fields = tool_call_template.header_fields or [] filtered_args = {k: v for k, v in tool_args.items() if k not in header_fields} - arg_str = ", ".join(f"${k}: String" for k in filtered_args.keys()) - var_defs = f"({arg_str})" if arg_str else "" - arg_pass = ", ".join(f"{k}: ${k}" for k in filtered_args.keys()) - arg_pass = f"({arg_pass})" if arg_pass else "" + # Use custom query if provided (highest flexibility for agents) + if tool_call_template.query: + gql_str = tool_call_template.query + else: + # Auto-generate query - use variable_types for proper typing + op_type = getattr(tool_call_template, "operation_type", "query") + base_tool_name = tool_name.split(".", 1)[-1] if "." in tool_name else tool_name + variable_types = tool_call_template.variable_types or {} + + # Build variable definitions with proper types (default to String) + arg_str = ", ".join( + f"${k}: {variable_types.get(k, 'String')}" + for k in filtered_args.keys() + ) + var_defs = f"({arg_str})" if arg_str else "" + arg_pass = ", ".join(f"{k}: ${k}" for k in filtered_args.keys()) + arg_pass = f"({arg_pass})" if arg_pass else "" + + # Note: Auto-generated queries for object-returning fields will still fail + # without a selection set. Use the `query` field for full control. + gql_str = f"{op_type} {var_defs} {{ {base_tool_name}{arg_pass} }}" + logger.debug(f"Auto-generated GraphQL: {gql_str}") - gql_str = f"{op_type} {var_defs} {{ {base_tool_name}{arg_pass} }}" document = gql_query(gql_str) result = await session.execute(document, variable_values=filtered_args) return result diff --git a/plugins/communication_protocols/gql/tests/test_graphql_integration.py b/plugins/communication_protocols/gql/tests/test_graphql_integration.py new file mode 100644 index 0000000..fdc4fcb --- /dev/null +++ b/plugins/communication_protocols/gql/tests/test_graphql_integration.py @@ -0,0 +1,275 @@ +"""Integration tests for GraphQL communication protocol using real GraphQL servers. + +Uses the public Countries API (https://countries.trevorblades.com/graphql) which +requires no authentication and has a stable schema. +""" +import os +import sys +import warnings +import pytest +import pytest_asyncio + +# Ensure plugin src is importable +PLUGIN_SRC = os.path.join(os.path.dirname(__file__), "..", "src") +PLUGIN_SRC = os.path.abspath(PLUGIN_SRC) +if PLUGIN_SRC not in sys.path: + sys.path.append(PLUGIN_SRC) + +import utcp_gql +from utcp_gql.gql_call_template import GraphQLCallTemplate +from utcp_gql.gql_communication_protocol import GraphQLCommunicationProtocol + +from utcp.implementations.utcp_client_implementation import UtcpClientImplementation + +# Public GraphQL API for testing (no auth required) +COUNTRIES_API_URL = "https://countries.trevorblades.com/graphql" + +# Suppress gql SSL warning (we're using HTTPS which is secure) +warnings.filterwarnings("ignore", message=".*AIOHTTPTransport does not verify ssl.*") + + +@pytest.fixture +def protocol(): + """Create a fresh GraphQL protocol instance.""" + utcp_gql.register() + return GraphQLCommunicationProtocol() + + +@pytest_asyncio.fixture +async def client(): + """Create a minimal UTCP client.""" + return await UtcpClientImplementation.create() + + +@pytest.mark.asyncio +async def test_register_manual_discovers_tools(protocol, client): + """Test that register_manual discovers tools from a real GraphQL schema.""" + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + ) + + result = await protocol.register_manual(client, template) + + assert result.success is True + assert len(result.manual.tools) > 0 + + # The Countries API should have these common queries + tool_names = [t.name for t in result.manual.tools] + assert "countries" in tool_names or "country" in tool_names + + +@pytest.mark.asyncio +async def test_call_tool_with_custom_query(protocol, client): + """Test calling a tool with a custom query string (fixes selection set issue).""" + # Custom query with proper selection set - this is the UTCP-flexible approach + custom_query = """ + query GetCountry($code: ID!) { + country(code: $code) { + name + capital + currency + } + } + """ + + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + query=custom_query, + ) + + result = await protocol.call_tool( + client, + "country", + {"code": "US"}, + template, + ) + + assert result is not None + assert "country" in result + assert result["country"]["name"] == "United States" + assert result["country"]["capital"] == "Washington D.C." + + +@pytest.mark.asyncio +async def test_call_tool_with_variable_types(protocol, client): + """Test that variable_types properly maps GraphQL types (fixes String-only issue).""" + # The country query expects code: ID!, not String + # Using variable_types to specify the correct type + custom_query = """ + query GetCountry($code: ID!) { + country(code: $code) { + name + emoji + } + } + """ + + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + query=custom_query, + variable_types={"code": "ID!"}, + ) + + result = await protocol.call_tool( + client, + "country", + {"code": "FR"}, + template, + ) + + assert result is not None + assert result["country"]["name"] == "France" + assert result["country"]["emoji"] == "šŸ‡«šŸ‡·" + + +@pytest.mark.asyncio +async def test_call_tool_list_query(protocol, client): + """Test querying a list of items with proper selection set.""" + custom_query = """ + query GetContinents { + continents { + code + name + } + } + """ + + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + query=custom_query, + ) + + result = await protocol.call_tool( + client, + "continents", + {}, + template, + ) + + assert result is not None + assert "continents" in result + assert len(result["continents"]) == 7 # 7 continents + + continent_names = [c["name"] for c in result["continents"]] + assert "Europe" in continent_names + assert "Asia" in continent_names + + +@pytest.mark.asyncio +async def test_call_tool_nested_query(protocol, client): + """Test querying nested objects with proper selection sets.""" + custom_query = """ + query GetCountryWithLanguages($code: ID!) { + country(code: $code) { + name + languages { + code + name + } + } + } + """ + + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + query=custom_query, + ) + + result = await protocol.call_tool( + client, + "country", + {"code": "CH"}, # Switzerland - has multiple languages + template, + ) + + assert result is not None + assert result["country"]["name"] == "Switzerland" + assert len(result["country"]["languages"]) >= 3 # German, French, Italian, Romansh + + +@pytest.mark.asyncio +async def test_call_tool_with_filter_arguments(protocol, client): + """Test queries with filter arguments using proper types.""" + custom_query = """ + query GetCountriesByContinent($filter: CountryFilterInput) { + countries(filter: $filter) { + code + name + } + } + """ + + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + query=custom_query, + variable_types={"filter": "CountryFilterInput"}, + ) + + result = await protocol.call_tool( + client, + "countries", + {"filter": {"continent": {"eq": "EU"}}}, + template, + ) + + assert result is not None + assert "countries" in result + # Should return European countries + country_codes = [c["code"] for c in result["countries"]] + assert "DE" in country_codes # Germany + assert "FR" in country_codes # France + + +@pytest.mark.asyncio +async def test_error_handling_invalid_query(protocol, client): + """Test that invalid queries return proper errors.""" + # Invalid query syntax + invalid_query = "this is not valid graphql" + + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + query=invalid_query, + ) + + with pytest.raises(Exception): + await protocol.call_tool( + client, + "invalid", + {}, + template, + ) + + +@pytest.mark.asyncio +async def test_error_handling_missing_selection_set_auto_generated(protocol, client): + """ + Demonstrate that auto-generated queries fail for object-returning fields. + + This test documents the limitation: without a custom query, object fields fail. + The fix is to always use the `query` field for object-returning operations. + """ + # No custom query - will auto-generate without selection set + template = GraphQLCallTemplate( + name="countries_api", + url=COUNTRIES_API_URL, + operation_type="query", + variable_types={"code": "ID!"}, + ) + + # This should fail because auto-generated query lacks selection set + # The query becomes: query ($code: ID!) { country(code: $code) } + # But country returns an object that needs: { name capital ... } + with pytest.raises(Exception): + await protocol.call_tool( + client, + "country", + {"code": "US"}, + template, + ) diff --git a/plugins/communication_protocols/gql/tests/test_graphql_protocol.py b/plugins/communication_protocols/gql/tests/test_graphql_protocol.py deleted file mode 100644 index 1b1bb74..0000000 --- a/plugins/communication_protocols/gql/tests/test_graphql_protocol.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import sys -import types -import pytest - - -# Ensure plugin src is importable -PLUGIN_SRC = os.path.join(os.path.dirname(__file__), "..", "src") -PLUGIN_SRC = os.path.abspath(PLUGIN_SRC) -if PLUGIN_SRC not in sys.path: - sys.path.append(PLUGIN_SRC) - -import utcp_gql -# Simplify imports: use the main module and assign local aliases -GraphQLProvider = utcp_gql.gql_call_template.GraphQLProvider -gql_module = utcp_gql.gql_communication_protocol - -from utcp.data.utcp_manual import UtcpManual -from utcp.utcp_client import UtcpClient -from utcp.implementations.utcp_client_implementation import UtcpClientImplementation - - -class FakeSchema: - def __init__(self): - # Minimal field objects with descriptions - self.query_type = types.SimpleNamespace( - fields={ - "hello": types.SimpleNamespace(description="Returns greeting"), - } - ) - self.mutation_type = types.SimpleNamespace( - fields={ - "add": types.SimpleNamespace(description="Adds numbers"), - } - ) - self.subscription_type = None - - -class FakeClientObj: - def __init__(self): - self.client = types.SimpleNamespace(schema=FakeSchema()) - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - return False - - async def execute(self, document, variable_values=None): - # document is a gql query; we can base behavior on variable_values - variable_values = variable_values or {} - # Determine operation by presence of variables used - if "hello" in str(document): - name = variable_values.get("name", "") - return {"hello": f"Hello {name}"} - if "add" in str(document): - a = int(variable_values.get("a", 0)) - b = int(variable_values.get("b", 0)) - return {"add": a + b} - return {"ok": True} - - -class FakeTransport: - def __init__(self, url: str, headers: dict | None = None): - self.url = url - self.headers = headers or {} - - -@pytest.mark.asyncio -async def test_graphql_register_and_call(monkeypatch): - # Patch gql client/transport used by protocol to avoid needing a real server - monkeypatch.setattr(gql_module, "GqlClient", lambda *args, **kwargs: FakeClientObj()) - monkeypatch.setattr(gql_module, "AIOHTTPTransport", FakeTransport) - # Avoid real GraphQL parsing; pass-through document string to fake execute - monkeypatch.setattr(gql_module, "gql_query", lambda s: s) - - # Register plugin (call_template serializer + protocol) - utcp_gql.register() - - # Create protocol and manual call template - protocol = gql_module.GraphQLCommunicationProtocol() - provider = GraphQLProvider( - name="mock_graph", - call_template_type="graphql", - url="http://localhost/graphql", - operation_type="query", - headers={"x-client": "utcp"}, - header_fields=["x-session-id"], - ) - - # Minimal UTCP client implementation for caller context - client: UtcpClient = await UtcpClientImplementation.create() - client.config.variables = {} - - # Register and discover tools - reg = await protocol.register_manual(client, provider) - assert reg.success is True - assert isinstance(reg.manual, UtcpManual) - tool_names = sorted(t.name for t in reg.manual.tools) - assert "hello" in tool_names - assert "add" in tool_names - - # Call hello - res = await protocol.call_tool(client, "mock_graph.hello", {"name": "UTCP", "x-session-id": "abc"}, provider) - assert res == {"hello": "Hello UTCP"} - - # Call add (mutation) - provider.operation_type = "mutation" - res2 = await protocol.call_tool(client, "mock_graph.add", {"a": 2, "b": 3}, provider) - assert res2 == {"add": 5} \ No newline at end of file diff --git a/plugins/communication_protocols/socket/README.md b/plugins/communication_protocols/socket/README.md index 3e695c9..04c1737 100644 --- a/plugins/communication_protocols/socket/README.md +++ b/plugins/communication_protocols/socket/README.md @@ -12,8 +12,8 @@ Prerequisites: 1) Install core and the socket plugin in editable mode with dev extras: ```bash -pip install -e "core[dev]" -pip install -e plugins/communication_protocols/socket[dev] +pip install -e "./core[dev]" +pip install -e ./plugins/communication_protocols/socket[dev] ``` 2) Run the socket plugin tests: diff --git a/plugins/communication_protocols/websocket/tests/__init__.py b/plugins/communication_protocols/websocket/tests/__init__.py deleted file mode 100644 index 614ce9a..0000000 --- a/plugins/communication_protocols/websocket/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for the WebSocket communication protocol plugin.""" diff --git a/socket_plugin_test.py b/socket_plugin_test.py deleted file mode 100644 index c03d2a9..0000000 --- a/socket_plugin_test.py +++ /dev/null @@ -1,40 +0,0 @@ -import asyncio -import sys -from pathlib import Path - -# Add core and plugin src paths so imports work without installing packages -core_src = Path(__file__).parent / "core" / "src" -socket_src = Path(__file__).parent / "plugins" / "communication_protocols" / "socket" / "src" -sys.path.insert(0, str(core_src.resolve())) -sys.path.insert(0, str(socket_src.resolve())) - -from utcp.plugins.plugin_loader import ensure_plugins_initialized -from utcp.interfaces.communication_protocol import CommunicationProtocol -from utcp.data.call_template import CallTemplateSerializer -from utcp_socket import register as register_socket - -async def main(): - # Manually register the socket plugin - register_socket() - - # Load core plugins (auth, repo, search, post-processors) - ensure_plugins_initialized() - - # 1. Check if communication protocols are registered - registered_protocols = CommunicationProtocol.communication_protocols - print(f"Registered communication protocols: {list(registered_protocols.keys())}") - assert "tcp" in registered_protocols, "TCP communication protocol not registered" - assert "udp" in registered_protocols, "UDP communication protocol not registered" - print("āœ… TCP and UDP communication protocols are registered.") - - # 2. Check if call templates are registered - registered_serializers = CallTemplateSerializer.call_template_serializers - print(f"Registered call template serializers: {list(registered_serializers.keys())}") - assert "tcp" in registered_serializers, "TCP call template serializer not registered" - assert "udp" in registered_serializers, "UDP call template serializer not registered" - print("āœ… TCP and UDP call template serializers are registered.") - - print("\nšŸŽ‰ Socket plugin sanity check passed! šŸŽ‰") - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/test_websocket_manual.py b/test_websocket_manual.py deleted file mode 100644 index a1457c4..0000000 --- a/test_websocket_manual.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -""" -Manual test script for WebSocket transport implementation. -This tests the core functionality without requiring pytest setup. -""" - -import asyncio -import sys -import os - -# Add src to path for imports -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from utcp.client.transport_interfaces.websocket_transport import WebSocketClientTransport -from utcp.shared.provider import WebSocketProvider -from utcp.shared.auth import ApiKeyAuth, BasicAuth - - -async def test_basic_functionality(): - """Test basic WebSocket transport functionality""" - print("Testing WebSocket Transport Implementation...") - - transport = WebSocketClientTransport() - - # Test 1: Security enforcement - print("\n1. Testing security enforcement...") - try: - insecure_provider = WebSocketProvider( - name="insecure", - url="ws://example.com/ws" # Should be rejected - ) - await transport.register_tool_provider(insecure_provider) - print("āŒ FAILED: Insecure URL was accepted") - except ValueError as e: - if "Security error" in str(e): - print("āœ… PASSED: Insecure URL properly rejected") - else: - print(f"āŒ FAILED: Wrong error: {e}") - except Exception as e: - print(f"āŒ FAILED: Unexpected error: {e}") - - # Test 2: Provider type validation - print("\n2. Testing provider type validation...") - try: - from utcp.shared.provider import HttpProvider - wrong_provider = HttpProvider(name="wrong", url="https://example.com") - await transport.register_tool_provider(wrong_provider) - print("āŒ FAILED: Wrong provider type was accepted") - except ValueError as e: - if "WebSocketClientTransport can only be used with WebSocketProvider" in str(e): - print("āœ… PASSED: Provider type validation works") - else: - print(f"āŒ FAILED: Wrong error: {e}") - except Exception as e: - print(f"āŒ FAILED: Unexpected error: {e}") - - # Test 3: Authentication header preparation - print("\n3. Testing authentication...") - try: - # Test API Key auth - api_provider = WebSocketProvider( - name="api_test", - url="wss://example.com/ws", - auth=ApiKeyAuth( - var_name="X-API-Key", - api_key="test-key-123", - location="header" - ) - ) - headers = await transport._prepare_headers(api_provider) - if headers.get("X-API-Key") == "test-key-123": - print("āœ… PASSED: API Key authentication headers prepared correctly") - else: - print(f"āŒ FAILED: API Key headers incorrect: {headers}") - - # Test Basic auth - basic_provider = WebSocketProvider( - name="basic_test", - url="wss://example.com/ws", - auth=BasicAuth(username="user", password="pass") - ) - headers = await transport._prepare_headers(basic_provider) - if "Authorization" in headers and headers["Authorization"].startswith("Basic "): - print("āœ… PASSED: Basic authentication headers prepared correctly") - else: - print(f"āŒ FAILED: Basic auth headers incorrect: {headers}") - - except Exception as e: - print(f"āŒ FAILED: Authentication test error: {e}") - - # Test 4: Connection management - print("\n4. Testing connection management...") - try: - localhost_provider = WebSocketProvider( - name="test_provider", - url="ws://localhost:8765/ws" - ) - - # This should fail to connect but not due to security - try: - await transport.register_tool_provider(localhost_provider) - print("āŒ FAILED: Connection should have failed (no server)") - except ValueError as e: - if "Security error" in str(e): - print("āŒ FAILED: Security error on localhost") - else: - print("ā“ UNEXPECTED: Different error occurred") - except Exception as e: - # Expected - connection refused or similar - print("āœ… PASSED: Connection management works (failed to connect as expected)") - - except Exception as e: - print(f"āŒ FAILED: Connection test error: {e}") - - # Test 5: Cleanup - print("\n5. Testing cleanup...") - try: - await transport.close() - if len(transport._connections) == 0 and len(transport._oauth_tokens) == 0: - print("āœ… PASSED: Cleanup successful") - else: - print("āŒ FAILED: Cleanup incomplete") - except Exception as e: - print(f"āŒ FAILED: Cleanup error: {e}") - - print("\nāœ… WebSocket transport basic functionality tests completed!") - - -async def test_with_mock_server(): - """Test with a real WebSocket connection to our mock server""" - print("\n" + "="*50) - print("Testing with Mock WebSocket Server") - print("="*50) - - # Import and start mock server - sys.path.append('tests/client/transport_interfaces') - try: - from mock_websocket_server import create_app - from aiohttp import web - - print("Starting mock WebSocket server...") - app = await create_app() - runner = web.AppRunner(app) - await runner.setup() - site = web.TCPSite(runner, 'localhost', 8765) - await site.start() - - print("Mock server started on ws://localhost:8765/ws") - - # Test with our transport - transport = WebSocketClientTransport() - provider = WebSocketProvider( - name="test_provider", - url="ws://localhost:8765/ws" - ) - - try: - # Test tool discovery - print("\nTesting tool discovery...") - tools = await transport.register_tool_provider(provider) - print(f"āœ… Discovered {len(tools)} tools:") - for tool in tools: - print(f" - {tool.name}: {tool.description}") - - # Test tool execution - print("\nTesting tool execution...") - result = await transport.call_tool("echo", {"message": "Hello WebSocket!"}, provider) - print(f"āœ… Echo result: {result}") - - result = await transport.call_tool("add_numbers", {"a": 5, "b": 3}, provider) - print(f"āœ… Add result: {result}") - - # Test error handling - print("\nTesting error handling...") - try: - await transport.call_tool("simulate_error", {"error_message": "Test error"}, provider) - print("āŒ FAILED: Error tool should have failed") - except RuntimeError as e: - print(f"āœ… Error properly handled: {e}") - - except Exception as e: - print(f"āŒ Transport test failed: {e}") - finally: - await transport.close() - await runner.cleanup() - print("Mock server stopped") - - except ImportError as e: - print(f"āš ļø Mock server test skipped (missing dependencies): {e}") - except Exception as e: - print(f"āŒ Mock server test failed: {e}") - - -async def main(): - """Run all manual tests""" - await test_basic_functionality() - # await test_with_mock_server() # Uncomment if you want to test with real server - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file From d4d8ede41a5bded516313a70b6e72137e69feede Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:13:11 +0100 Subject: [PATCH 04/10] remove WIP for finished plugins --- plugins/communication_protocols/gql/pyproject.toml | 2 +- plugins/communication_protocols/socket/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/communication_protocols/gql/pyproject.toml b/plugins/communication_protocols/gql/pyproject.toml index 7c752c3..d5b558d 100644 --- a/plugins/communication_protocols/gql/pyproject.toml +++ b/plugins/communication_protocols/gql/pyproject.toml @@ -8,7 +8,7 @@ version = "1.0.2" authors = [ { name = "UTCP Contributors" }, ] -description = "UTCP communication protocol plugin for GraphQL. (Work in progress)" +description = "UTCP communication protocol plugin for GraphQL." readme = "README.md" requires-python = ">=3.10" dependencies = [ diff --git a/plugins/communication_protocols/socket/pyproject.toml b/plugins/communication_protocols/socket/pyproject.toml index 2f232ad..a544648 100644 --- a/plugins/communication_protocols/socket/pyproject.toml +++ b/plugins/communication_protocols/socket/pyproject.toml @@ -8,7 +8,7 @@ version = "1.0.2" authors = [ { name = "UTCP Contributors" }, ] -description = "UTCP communication protocol plugin for TCP and UDP protocols. (Work in progress)" +description = "UTCP communication protocol plugin for TCP and UDP protocols." readme = "README.md" requires-python = ">=3.10" dependencies = [ From 2dc9c02df72cad3770c934959325ec344b441444 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:40:20 +0100 Subject: [PATCH 05/10] Update UTCP to 1.1 --- README.md | 76 ++++++ core/README.md | 186 ++++++++++++-- core/pyproject.toml | 2 +- core/src/utcp/data/call_template.py | 6 + .../utcp_client_implementation.py | 105 +++++++- core/tests/client/test_utcp_client.py | 230 ++++++++++++++++++ 6 files changed, 580 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 6b520f5..400b0da 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,7 @@ Configuration examples for each protocol. Remember to replace `provider_type` wi "url": "https://api.example.com/users/{user_id}", // Required "http_method": "POST", // Required, default: "GET" "content_type": "application/json", // Optional, default: "application/json" + "allowed_communication_protocols": ["http"], // Optional, defaults to [call_template_type]. Restricts which protocols tools can use. "auth": { // Optional, authentication for the HTTP request (example using ApiKeyAuth for Bearer token) "auth_type": "api_key", "api_key": "Bearer $API_KEY", // Required @@ -514,6 +515,81 @@ Note the name change from `http_stream` to `streamable_http`. } ``` +## Security: Protocol Restrictions + +UTCP provides fine-grained control over which communication protocols each manual can use through the `allowed_communication_protocols` field. This prevents potentially dangerous protocol escalation (e.g., an HTTP-based manual accidentally calling CLI tools). + +### Default Behavior (Secure by Default) + +When `allowed_communication_protocols` is not set or is empty, a manual can only register and call tools that use the **same protocol type** as the manual itself: + +```python +from utcp_http.http_call_template import HttpCallTemplate + +# This manual can ONLY register/call HTTP tools (default restriction) +http_manual = HttpCallTemplate( + name="my_api", + call_template_type="http", + url="https://api.example.com/utcp" + # allowed_communication_protocols not set ā†’ defaults to ["http"] +) +``` + +### Allowing Multiple Protocols + +To allow a manual to work with tools from multiple protocols, explicitly set `allowed_communication_protocols`: + +```python +from utcp_http.http_call_template import HttpCallTemplate + +# This manual can register/call both HTTP and CLI tools +multi_protocol_manual = HttpCallTemplate( + name="flexible_manual", + call_template_type="http", + url="https://api.example.com/utcp", + allowed_communication_protocols=["http", "cli"] # Explicitly allow both +) +``` + +### JSON Configuration + +```json +{ + "name": "my_api", + "call_template_type": "http", + "url": "https://api.example.com/utcp", + "allowed_communication_protocols": ["http", "cli", "mcp"] +} +``` + +### Behavior Summary + +| `allowed_communication_protocols` | Manual Type | Allowed Tool Protocols | +|----------------------------------|-------------|------------------------| +| Not set / `null` | `"http"` | Only `"http"` | +| `[]` (empty) | `"http"` | Only `"http"` | +| `["http", "cli"]` | `"http"` | `"http"` and `"cli"` | +| `["http", "cli", "mcp"]` | `"cli"` | `"http"`, `"cli"`, and `"mcp"` | + +### Registration Filtering + +During `register_manual()`, tools that don't match the allowed protocols are automatically filtered out with a warning: + +``` +WARNING - Tool 'dangerous_tool' uses communication protocol 'cli' which is not in +allowed protocols ['http'] for manual 'my_api'. Tool will not be registered. +``` + +### Call-Time Validation + +Even if a tool somehow exists in the repository, calling it will fail if its protocol is not allowed: + +```python +# Raises ValueError: Tool 'my_api.some_cli_tool' uses communication protocol 'cli' +# which is not allowed by manual 'my_api'. Allowed protocols: ['http'] +await client.call_tool("my_api.some_cli_tool", {"arg": "value"}) +``` + ## Testing The testing structure has been updated to reflect the new core/plugin split. diff --git a/core/README.md b/core/README.md index 35ff09e..400b0da 100644 --- a/core/README.md +++ b/core/README.md @@ -86,6 +86,7 @@ UTCP supports multiple communication protocols through dedicated plugins: | [`utcp-cli`](plugins/communication_protocols/cli/) | Command-line tools | āœ… Stable | [CLI Plugin README](plugins/communication_protocols/cli/README.md) | | [`utcp-mcp`](plugins/communication_protocols/mcp/) | Model Context Protocol | āœ… Stable | [MCP Plugin README](plugins/communication_protocols/mcp/README.md) | | [`utcp-text`](plugins/communication_protocols/text/) | Local file-based tools | āœ… Stable | [Text Plugin README](plugins/communication_protocols/text/README.md) | +| [`utcp-websocket`](plugins/communication_protocols/websocket/) | WebSocket real-time bidirectional communication | āœ… Stable | [WebSocket Plugin README](plugins/communication_protocols/websocket/README.md) | | [`utcp-socket`](plugins/communication_protocols/socket/) | TCP/UDP protocols | šŸš§ In Progress | [Socket Plugin README](plugins/communication_protocols/socket/README.md) | | [`utcp-gql`](plugins/communication_protocols/gql/) | GraphQL APIs | šŸš§ In Progress | [GraphQL Plugin README](plugins/communication_protocols/gql/README.md) | @@ -376,12 +377,19 @@ Configuration examples for each protocol. Remember to replace `provider_type` wi "url": "https://api.example.com/users/{user_id}", // Required "http_method": "POST", // Required, default: "GET" "content_type": "application/json", // Optional, default: "application/json" - "auth": { // Optional, example using ApiKeyAuth for a Bearer token. The client must prepend "Bearer " to the token. + "allowed_communication_protocols": ["http"], // Optional, defaults to [call_template_type]. Restricts which protocols tools can use. + "auth": { // Optional, authentication for the HTTP request (example using ApiKeyAuth for Bearer token) "auth_type": "api_key", "api_key": "Bearer $API_KEY", // Required "var_name": "Authorization", // Optional, default: "X-Api-Key" "location": "header" // Optional, default: "header" }, + "auth_tools": { // Optional, authentication for converted tools, if this call template points to an openapi spec that should be automatically converted to a utcp manual (applied only to endpoints requiring auth per OpenAPI spec) + "auth_type": "api_key", + "api_key": "Bearer $TOOL_API_KEY", // Required + "var_name": "Authorization", // Optional, default: "X-Api-Key" + "location": "header" // Optional, default: "header" + }, "headers": { // Optional "X-Custom-Header": "value" }, @@ -437,31 +445,34 @@ Note the name change from `http_stream` to `streamable_http`. ```json { - "name": "my_cli_tool", + "name": "multi_step_cli_tool", "call_template_type": "cli", // Required - "commands": [ // Required - array of commands to execute in sequence + "commands": [ // Required - sequential command execution { - "command": "cd UTCP_ARG_target_dir_UTCP_END", - "append_to_final_output": false // Optional, default is false if not last command + "command": "git clone UTCP_ARG_repo_url_UTCP_END temp_repo", + "append_to_final_output": false }, { - "command": "my-command --input UTCP_ARG_input_file_UTCP_END" - // append_to_final_output defaults to true for last command + "command": "cd temp_repo && find . -name '*.py' | wc -l" + // Last command output returned by default } ], "env_vars": { // Optional - "MY_VAR": "my_value" + "GIT_AUTHOR_NAME": "UTCP Bot", + "API_KEY": "${MY_API_KEY}" }, - "working_dir": "/path/to/working/directory", // Optional + "working_dir": "/tmp", // Optional "auth": null // Optional (always null for CLI) } ``` -**Notes:** -- Commands execute in a single subprocess (PowerShell on Windows, Bash on Unix) -- Use `UTCP_ARG_argname_UTCP_END` placeholders for arguments -- Reference previous command output with `$CMD_0_OUTPUT`, `$CMD_1_OUTPUT`, etc. -- Only the last command's output is returned by default +**CLI Protocol Features:** +- **Multi-command execution**: Commands run sequentially in single subprocess +- **Cross-platform**: PowerShell on Windows, Bash on Unix/Linux/macOS +- **State preservation**: Directory changes (`cd`) persist between commands +- **Argument placeholders**: `UTCP_ARG_argname_UTCP_END` format +- **Output referencing**: Access previous outputs with `$CMD_0_OUTPUT`, `$CMD_1_OUTPUT` +- **Flexible output control**: Choose which command outputs to include in final result ### Text Call Template @@ -470,7 +481,13 @@ Note the name change from `http_stream` to `streamable_http`. "name": "my_text_manual", "call_template_type": "text", // Required "file_path": "./manuals/my_manual.json", // Required - "auth": null // Optional (always null for Text) + "auth": null, // Optional (always null for Text) + "auth_tools": { // Optional, authentication for generated tools from OpenAPI specs + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + } } ``` @@ -498,6 +515,81 @@ Note the name change from `http_stream` to `streamable_http`. } ``` +## Security: Protocol Restrictions + +UTCP provides fine-grained control over which communication protocols each manual can use through the `allowed_communication_protocols` field. This prevents potentially dangerous protocol escalation (e.g., an HTTP-based manual accidentally calling CLI tools). + +### Default Behavior (Secure by Default) + +When `allowed_communication_protocols` is not set or is empty, a manual can only register and call tools that use the **same protocol type** as the manual itself: + +```python +from utcp_http.http_call_template import HttpCallTemplate + +# This manual can ONLY register/call HTTP tools (default restriction) +http_manual = HttpCallTemplate( + name="my_api", + call_template_type="http", + url="https://api.example.com/utcp" + # allowed_communication_protocols not set ā†’ defaults to ["http"] +) +``` + +### Allowing Multiple Protocols + +To allow a manual to work with tools from multiple protocols, explicitly set `allowed_communication_protocols`: + +```python +from utcp_http.http_call_template import HttpCallTemplate + +# This manual can register/call both HTTP and CLI tools +multi_protocol_manual = HttpCallTemplate( + name="flexible_manual", + call_template_type="http", + url="https://api.example.com/utcp", + allowed_communication_protocols=["http", "cli"] # Explicitly allow both +) +``` + +### JSON Configuration + +```json +{ + "name": "my_api", + "call_template_type": "http", + "url": "https://api.example.com/utcp", + "allowed_communication_protocols": ["http", "cli", "mcp"] +} +``` + +### Behavior Summary + +| `allowed_communication_protocols` | Manual Type | Allowed Tool Protocols | +|----------------------------------|-------------|------------------------| +| Not set / `null` | `"http"` | Only `"http"` | +| `[]` (empty) | `"http"` | Only `"http"` | +| `["http", "cli"]` | `"http"` | `"http"` and `"cli"` | +| `["http", "cli", "mcp"]` | `"cli"` | `"http"`, `"cli"`, and `"mcp"` | + +### Registration Filtering + +During `register_manual()`, tools that don't match the allowed protocols are automatically filtered out with a warning: + +``` +WARNING - Tool 'dangerous_tool' uses communication protocol 'cli' which is not in +allowed protocols ['http'] for manual 'my_api'. Tool will not be registered. +``` + +### Call-Time Validation + +Even if a tool somehow exists in the repository, calling it will fail if its protocol is not allowed: + +```python +# Raises ValueError: Tool 'my_api.some_cli_tool' uses communication protocol 'cli' +# which is not allowed by manual 'my_api'. Allowed protocols: ['http'] +await client.call_tool("my_api.some_cli_tool", {"arg": "value"}) +``` + ## Testing The testing structure has been updated to reflect the new core/plugin split. @@ -535,4 +627,68 @@ The build process now involves building each package (`core` and `plugins`) sepa 4. Run the build: `python -m build`. 5. The distributable files (`.whl` and `.tar.gz`) will be in the `dist/` directory. +## OpenAPI Ingestion - Zero Infrastructure Tool Integration + +šŸš€ **Transform any existing REST API into UTCP tools without server modifications!** + +UTCP's OpenAPI ingestion feature automatically converts OpenAPI 2.0/3.0 specifications into UTCP tools, enabling AI agents to interact with existing APIs directly - no wrapper servers, no API changes, no additional infrastructure required. + +### Quick Start with OpenAPI + +```python +from utcp_http.openapi_converter import OpenApiConverter +import aiohttp + +# Convert any OpenAPI spec to UTCP tools +async def convert_api(): + async with aiohttp.ClientSession() as session: + async with session.get("https://api.github.com/openapi.json") as response: + openapi_spec = await response.json() + + converter = OpenApiConverter(openapi_spec) + manual = converter.convert() + + print(f"Generated {len(manual.tools)} tools from GitHub API!") + return manual + +# Or use UTCP Client configuration for automatic detection +from utcp.utcp_client import UtcpClient + +client = await UtcpClient.create(config={ + "manual_call_templates": [{ + "name": "github", + "call_template_type": "http", + "url": "https://api.github.com/openapi.json", + "auth_tools": { # Authentication for generated tools requiring auth + "auth_type": "api_key", + "api_key": "Bearer ${GITHUB_TOKEN}", + "var_name": "Authorization", + "location": "header" + } + }] +}) +``` + +### Key Benefits + +- āœ… **Zero Infrastructure**: No servers to deploy or maintain +- āœ… **Direct API Calls**: Native performance, no proxy overhead +- āœ… **Automatic Conversion**: OpenAPI schemas ā†’ UTCP tools +- āœ… **Selective Authentication**: Only protected endpoints get auth, public endpoints remain accessible +- āœ… **Authentication Preserved**: API keys, OAuth2, Basic auth supported +- āœ… **Multi-format Support**: JSON, YAML, OpenAPI 2.0/3.0 +- āœ… **Batch Processing**: Convert multiple APIs simultaneously + +### Multiple Ingestion Methods + +1. **Direct Converter**: `OpenApiConverter` class for full control +2. **Remote URLs**: Fetch and convert specs from any URL +3. **Client Configuration**: Include specs directly in UTCP config +4. **Batch Processing**: Process multiple specs programmatically +5. **File-based**: Convert local JSON/YAML specifications + +šŸ“– **[Complete OpenAPI Ingestion Guide](docs/openapi-ingestion.md)** - Detailed examples and advanced usage + +--- + ## [Contributors](https://www.utcp.io/about) diff --git a/core/pyproject.toml b/core/pyproject.toml index 482b147..2844db0 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp" -version = "1.0.4" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] diff --git a/core/src/utcp/data/call_template.py b/core/src/utcp/data/call_template.py index eff0f1b..718f560 100644 --- a/core/src/utcp/data/call_template.py +++ b/core/src/utcp/data/call_template.py @@ -40,11 +40,17 @@ class CallTemplate(BaseModel): Should be unique across all providers and recommended to be set to a human-readable name. Can only contain letters, numbers and underscores. All special characters must be replaced with underscores. call_template_type: The transport protocol type used by this provider. + allowed_communication_protocols: Optional list of communication protocol types that tools + registered under this manual are allowed to use. If None or empty, defaults to only allowing + the same protocol type as the manual's call_template_type. This provides fine-grained security + control - e.g., set to ["http", "cli"] to allow both HTTP and CLI tools, or leave unset to + restrict tools to the manual's own protocol type. """ name: str = Field(default_factory=lambda: uuid.uuid4().hex) call_template_type: str auth: Optional[Auth] = None + allowed_communication_protocols: Optional[List[str]] = None @field_serializer("auth") def serialize_auth(self, auth: Optional[Auth]): diff --git a/core/src/utcp/implementations/utcp_client_implementation.py b/core/src/utcp/implementations/utcp_client_implementation.py index 01c13a9..b88bead 100644 --- a/core/src/utcp/implementations/utcp_client_implementation.py +++ b/core/src/utcp/implementations/utcp_client_implementation.py @@ -95,11 +95,26 @@ async def register_manual(self, manual_call_template: CallTemplate) -> RegisterM """REQUIRED Register a manual in the client. + Registers a manual and its tools with the client. During registration, tools are + filtered based on the manual's `allowed_communication_protocols` setting: + + - If `allowed_communication_protocols` is set to a non-empty list, only tools using + protocols in that list are registered. + - If `allowed_communication_protocols` is None or empty, it defaults to only allowing + the manual's own `call_template_type`. This provides secure-by-default behavior. + + Tools that don't match the allowed protocols are excluded from registration and a + warning is logged for each excluded tool. + Args: manual_call_template: The `CallTemplate` instance representing the manual to register. Returns: - A `RegisterManualResult` instance representing the result of the registration. + A `RegisterManualResult` instance containing the registered tools (filtered by + allowed protocols) and any errors encountered. + + Raises: + ValueError: If manual name is already registered or communication protocol is not found. """ # Replace all non-word characters with underscore manual_call_template.name = re.sub(r'[^\w]', '_', manual_call_template.name) @@ -112,9 +127,27 @@ async def register_manual(self, manual_call_template: CallTemplate) -> RegisterM result = await CommunicationProtocol.communication_protocols[manual_call_template.call_template_type].register_manual(self, manual_call_template) if result.success: + # Determine allowed protocols: use explicit list or default to manual's own protocol + allowed_protocols = manual_call_template.allowed_communication_protocols + if not allowed_protocols: + allowed_protocols = [manual_call_template.call_template_type] + + # Filter tools based on allowed communication protocols + filtered_tools = [] for tool in result.manual.tools: - if not tool.name.startswith(manual_call_template.name + "."): - tool.name = manual_call_template.name + "." + tool.name + tool_protocol = tool.tool_call_template.call_template_type if tool.tool_call_template else manual_call_template.call_template_type + if tool_protocol in allowed_protocols: + if not tool.name.startswith(manual_call_template.name + "."): + tool.name = manual_call_template.name + "." + tool.name + filtered_tools.append(tool) + else: + logger.warning( + f"Tool '{tool.name}' uses communication protocol '{tool_protocol}' " + f"which is not in allowed protocols {allowed_protocols} for manual '{manual_call_template.name}'. " + f"Tool will not be registered." + ) + + result.manual.tools = filtered_tools await self.config.tool_repository.save_manual(result.manual_call_template, result.manual) return result @@ -177,12 +210,25 @@ async def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> Any: """REQUIRED Call a tool in the client. + Executes a registered tool with the provided arguments. Before execution, validates + that the tool's communication protocol is allowed by the parent manual's + `allowed_communication_protocols` setting: + + - If `allowed_communication_protocols` is set to a non-empty list, the tool's protocol + must be in that list. + - If `allowed_communication_protocols` is None or empty, only tools using the manual's + own `call_template_type` are allowed. + Args: - tool_name: The name of the tool to call. + tool_name: The fully qualified name of the tool (e.g., "manual_name.tool_name"). tool_args: A dictionary of arguments to pass to the tool. Returns: - The result of the tool call. + The result of the tool call, after any post-processing. + + Raises: + ValueError: If the tool is not found or if the tool's communication protocol + is not in the manual's allowed protocols. """ manual_name = tool_name.split(".")[0] tool = await self.config.tool_repository.get_tool(tool_name) @@ -190,6 +236,20 @@ async def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> Any: raise ValueError(f"Tool not found: {tool_name}") tool_call_template = tool.tool_call_template tool_call_template = self._substitute_call_template_variables(tool_call_template, manual_name) + + # Check if the tool's communication protocol is allowed by the manual + manual_call_template = await self.config.tool_repository.get_manual_call_template(manual_name) + if manual_call_template: + allowed_protocols = manual_call_template.allowed_communication_protocols + if not allowed_protocols: + allowed_protocols = [manual_call_template.call_template_type] + if tool_call_template.call_template_type not in allowed_protocols: + raise ValueError( + f"Tool '{tool_name}' uses communication protocol '{tool_call_template.call_template_type}' " + f"which is not allowed by manual '{manual_name}'. " + f"Allowed protocols: {allowed_protocols}" + ) + result = await CommunicationProtocol.communication_protocols[tool_call_template.call_template_type].call_tool(self, tool_name, tool_args, tool_call_template) for post_processor in self.config.post_processing: @@ -198,14 +258,27 @@ async def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> Any: async def call_tool_streaming(self, tool_name: str, tool_args: Dict[str, Any]) -> AsyncGenerator[Any, None]: """REQUIRED - Call a tool in the client streamingly. + Call a tool in the client with streaming response. + + Executes a registered tool with streaming output. Before execution, validates + that the tool's communication protocol is allowed by the parent manual's + `allowed_communication_protocols` setting: + + - If `allowed_communication_protocols` is set to a non-empty list, the tool's protocol + must be in that list. + - If `allowed_communication_protocols` is None or empty, only tools using the manual's + own `call_template_type` are allowed. Args: - tool_name: The name of the tool to call. + tool_name: The fully qualified name of the tool (e.g., "manual_name.tool_name"). tool_args: A dictionary of arguments to pass to the tool. - Returns: - An async generator yielding the result of the tool call. + Yields: + Chunks of the tool's streaming response, after any post-processing. + + Raises: + ValueError: If the tool is not found or if the tool's communication protocol + is not in the manual's allowed protocols. """ manual_name = tool_name.split(".")[0] tool = await self.config.tool_repository.get_tool(tool_name) @@ -213,6 +286,20 @@ async def call_tool_streaming(self, tool_name: str, tool_args: Dict[str, Any]) - raise ValueError(f"Tool not found: {tool_name}") tool_call_template = tool.tool_call_template tool_call_template = self._substitute_call_template_variables(tool_call_template, manual_name) + + # Check if the tool's communication protocol is allowed by the manual + manual_call_template = await self.config.tool_repository.get_manual_call_template(manual_name) + if manual_call_template: + allowed_protocols = manual_call_template.allowed_communication_protocols + if not allowed_protocols: + allowed_protocols = [manual_call_template.call_template_type] + if tool_call_template.call_template_type not in allowed_protocols: + raise ValueError( + f"Tool '{tool_name}' uses communication protocol '{tool_call_template.call_template_type}' " + f"which is not allowed by manual '{manual_name}'. " + f"Allowed protocols: {allowed_protocols}" + ) + async for item in CommunicationProtocol.communication_protocols[tool_call_template.call_template_type].call_tool_streaming(self, tool_name, tool_args, tool_call_template): for post_processor in self.config.post_processing: item = post_processor.post_process(self, tool, tool_call_template, item) diff --git a/core/tests/client/test_utcp_client.py b/core/tests/client/test_utcp_client.py index 34c408f..934bebf 100644 --- a/core/tests/client/test_utcp_client.py +++ b/core/tests/client/test_utcp_client.py @@ -725,6 +725,236 @@ async def test_load_call_templates_wrong_format(self): os.unlink(temp_file) +class TestAllowedCommunicationProtocols: + """Test allowed_communication_protocols restriction functionality.""" + + @pytest.mark.asyncio + async def test_call_tool_allowed_protocol(self, utcp_client, sample_tools, isolated_communication_protocols): + """Test calling a tool when its protocol is in the allowed list.""" + client = utcp_client + call_template = HttpCallTemplate( + name="test_manual", + url="https://api.example.com/tool", + http_method="POST", + call_template_type="http", + allowed_communication_protocols=["http", "cli"] # Allow both HTTP and CLI + ) + + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=sample_tools[:1]) + mock_protocol = MockCommunicationProtocol(manual, "test_result") + CommunicationProtocol.communication_protocols["http"] = mock_protocol + + await client.register_manual(call_template) + + # Call should succeed since "http" is in allowed_communication_protocols + result = await client.call_tool("test_manual.http_tool", {"param1": "value1"}) + assert result == "test_result" + + @pytest.mark.asyncio + async def test_register_filters_disallowed_protocol_tools(self, utcp_client, sample_tools, isolated_communication_protocols): + """Test that tools with disallowed protocols are filtered during registration.""" + client = utcp_client + + # Register HTTP manual that only allows "http" protocol + http_call_template = HttpCallTemplate( + name="http_manual", + url="https://api.example.com/tool", + http_method="POST", + call_template_type="http", + allowed_communication_protocols=["http"] # Only allow HTTP + ) + + # Create a tool that uses CLI protocol (which is not allowed) + cli_tool = Tool( + name="cli_tool", + description="CLI test tool", + inputs=JsonSchema( + type="object", + properties={"command": {"type": "string", "description": "Command to execute"}}, + required=["command"] + ), + outputs=JsonSchema( + type="object", + properties={"output": {"type": "string", "description": "Command output"}} + ), + tags=["cli", "test"], + tool_call_template=CliCallTemplate( + name="cli_provider", + commands=[{"command": "echo UTCP_ARG_command_UTCP_END"}], + call_template_type="cli" + ) + ) + + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[cli_tool]) + mock_http_protocol = MockCommunicationProtocol(manual) + mock_cli_protocol = MockCommunicationProtocol() + CommunicationProtocol.communication_protocols["http"] = mock_http_protocol + CommunicationProtocol.communication_protocols["cli"] = mock_cli_protocol + + result = await client.register_manual(http_call_template) + + # CLI tool should be filtered out during registration + assert len(result.manual.tools) == 0 + + # Tool should not exist in repository + tool = await client.config.tool_repository.get_tool("http_manual.cli_tool") + assert tool is None + + @pytest.mark.asyncio + async def test_call_tool_default_protocol_restriction(self, utcp_client, sample_tools, isolated_communication_protocols): + """Test that when no allowed_communication_protocols is set, only the manual's protocol is allowed.""" + client = utcp_client + + # Register HTTP manual without explicit protocol restrictions + # Default behavior: only HTTP tools should be allowed + http_call_template = HttpCallTemplate( + name="http_manual", + url="https://api.example.com/tool", + http_method="POST", + call_template_type="http" + # No allowed_communication_protocols set - defaults to ["http"] + ) + + # Create tools: one HTTP (should be registered), one CLI (should be filtered out) + http_tool = Tool( + name="http_tool", + description="HTTP test tool", + inputs=JsonSchema(type="object", properties={}), + outputs=JsonSchema(type="object", properties={}), + tool_call_template=HttpCallTemplate( + name="http_provider", + url="https://api.example.com/call", + http_method="GET", + call_template_type="http" + ) + ) + cli_tool = Tool( + name="cli_tool", + description="CLI test tool", + inputs=JsonSchema(type="object", properties={}), + outputs=JsonSchema(type="object", properties={}), + tool_call_template=CliCallTemplate( + name="cli_provider", + commands=[{"command": "echo test"}], + call_template_type="cli" + ) + ) + + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[http_tool, cli_tool]) + mock_http_protocol = MockCommunicationProtocol(manual, call_result="http_result") + mock_cli_protocol = MockCommunicationProtocol() + CommunicationProtocol.communication_protocols["http"] = mock_http_protocol + CommunicationProtocol.communication_protocols["cli"] = mock_cli_protocol + + result = await client.register_manual(http_call_template) + + # Only HTTP tool should be registered, CLI tool should be filtered out + assert len(result.manual.tools) == 1 + assert result.manual.tools[0].name == "http_manual.http_tool" + + # HTTP tool call should succeed + call_result = await client.call_tool("http_manual.http_tool", {}) + assert call_result == "http_result" + + # CLI tool should not exist in repository + cli_tool_in_repo = await client.config.tool_repository.get_tool("http_manual.cli_tool") + assert cli_tool_in_repo is None + + @pytest.mark.asyncio + async def test_register_with_multiple_allowed_protocols(self, utcp_client, sample_tools, isolated_communication_protocols): + """Test registration with multiple allowed protocols allows all specified types.""" + client = utcp_client + + http_call_template = HttpCallTemplate( + name="multi_protocol_manual", + url="https://api.example.com/tool", + http_method="POST", + call_template_type="http", + allowed_communication_protocols=["http", "cli"] # Allow both + ) + + http_tool = Tool( + name="http_tool", + description="HTTP test tool", + inputs=JsonSchema(type="object", properties={}), + outputs=JsonSchema(type="object", properties={}), + tool_call_template=HttpCallTemplate( + name="http_provider", + url="https://api.example.com/call", + http_method="GET", + call_template_type="http" + ) + ) + cli_tool = Tool( + name="cli_tool", + description="CLI test tool", + inputs=JsonSchema(type="object", properties={}), + outputs=JsonSchema(type="object", properties={}), + tool_call_template=CliCallTemplate( + name="cli_provider", + commands=[{"command": "echo test"}], + call_template_type="cli" + ) + ) + + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[http_tool, cli_tool]) + mock_http_protocol = MockCommunicationProtocol(manual, call_result="http_result") + mock_cli_protocol = MockCommunicationProtocol(call_result="cli_result") + CommunicationProtocol.communication_protocols["http"] = mock_http_protocol + CommunicationProtocol.communication_protocols["cli"] = mock_cli_protocol + + result = await client.register_manual(http_call_template) + + # Both tools should be registered + assert len(result.manual.tools) == 2 + tool_names = [t.name for t in result.manual.tools] + assert "multi_protocol_manual.http_tool" in tool_names + assert "multi_protocol_manual.cli_tool" in tool_names + + # Both tools should be callable + http_result = await client.call_tool("multi_protocol_manual.http_tool", {}) + assert http_result == "http_result" + + cli_result = await client.call_tool("multi_protocol_manual.cli_tool", {}) + assert cli_result == "cli_result" + + @pytest.mark.asyncio + async def test_call_tool_empty_allowed_protocols_defaults_to_manual_type(self, utcp_client, sample_tools, isolated_communication_protocols): + """Test that empty allowed_communication_protocols defaults to manual's protocol type.""" + client = utcp_client + + http_call_template = HttpCallTemplate( + name="http_manual", + url="https://api.example.com/tool", + http_method="POST", + call_template_type="http", + allowed_communication_protocols=[] # Empty list defaults to ["http"] + ) + + cli_tool = Tool( + name="cli_tool", + description="CLI test tool", + inputs=JsonSchema(type="object", properties={}), + outputs=JsonSchema(type="object", properties={}), + tool_call_template=CliCallTemplate( + name="cli_provider", + commands=[{"command": "echo test"}], + call_template_type="cli" + ) + ) + + manual = UtcpManual(utcp_version="1.0", manual_version="1.0", tools=[cli_tool]) + mock_http_protocol = MockCommunicationProtocol(manual) + mock_cli_protocol = MockCommunicationProtocol(call_result="cli_result") + CommunicationProtocol.communication_protocols["http"] = mock_http_protocol + CommunicationProtocol.communication_protocols["cli"] = mock_cli_protocol + + result = await client.register_manual(http_call_template) + + # CLI tool should be filtered out during registration + assert len(result.manual.tools) == 0 + + class TestToolSerialization: """Test Tool and JsonSchema serialization.""" From e72241393233eea4da785e37e3ce27c65cc78a7d Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sun, 30 Nov 2025 23:43:13 +0100 Subject: [PATCH 06/10] Update all plugins to 1.1 --- core/pyproject.toml | 2 +- .../default_variable_substitutor.py | 35 +- .../cli/pyproject.toml | 4 +- .../communication_protocols/file/README.md | 129 +++++++ .../file/pyproject.toml | 45 +++ .../file/src/utcp_file/__init__.py | 17 + .../file/src/utcp_file/file_call_template.py | 66 ++++ .../utcp_file/file_communication_protocol.py | 140 ++++++++ .../tests/test_file_communication_protocol.py | 286 ++++++++++++++++ .../gql/pyproject.toml | 4 +- .../http/pyproject.toml | 4 +- .../mcp/pyproject.toml | 4 +- .../socket/pyproject.toml | 4 +- .../communication_protocols/text/README.md | 148 ++++---- .../text/pyproject.toml | 9 +- .../text/src/utcp_text/text_call_template.py | 27 +- .../utcp_text/text_communication_protocol.py | 79 ++--- .../tests/test_text_communication_protocol.py | 317 ++++++------------ .../websocket/pyproject.toml | 4 +- 19 files changed, 948 insertions(+), 376 deletions(-) create mode 100644 plugins/communication_protocols/file/README.md create mode 100644 plugins/communication_protocols/file/pyproject.toml create mode 100644 plugins/communication_protocols/file/src/utcp_file/__init__.py create mode 100644 plugins/communication_protocols/file/src/utcp_file/file_call_template.py create mode 100644 plugins/communication_protocols/file/src/utcp_file/file_communication_protocol.py create mode 100644 plugins/communication_protocols/file/tests/test_file_communication_protocol.py diff --git a/core/pyproject.toml b/core/pyproject.toml index 2844db0..e9214da 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp" -version = "1.1.0" +version = "1.1.1" authors = [ { name = "UTCP Contributors" }, ] diff --git a/core/src/utcp/implementations/default_variable_substitutor.py b/core/src/utcp/implementations/default_variable_substitutor.py index cb33fe4..ccc6dfb 100644 --- a/core/src/utcp/implementations/default_variable_substitutor.py +++ b/core/src/utcp/implementations/default_variable_substitutor.py @@ -67,6 +67,10 @@ def substitute(self, obj: dict | list | str, config: UtcpClientConfig, variable_ Non-string types are returned unchanged. String values are scanned for variable references using ${VAR} and $VAR syntax. + Note: + Strings containing '$ref' are skipped to support OpenAPI specs + stored as string content, where $ref is a JSON reference keyword. + Args: obj: Object to perform substitution on. Can be any type. config: UTCP client configuration containing variable sources. @@ -95,18 +99,22 @@ def substitute(self, obj: dict | list | str, config: UtcpClientConfig, variable_ if variable_namespace and not all(c.isalnum() or c == '_' for c in variable_namespace): raise ValueError(f"Variable namespace '{variable_namespace}' contains invalid characters. Only alphanumeric characters and underscores are allowed.") - if isinstance(obj, dict): - return {k: self.substitute(v, config, variable_namespace) for k, v in obj.items()} - elif isinstance(obj, list): - return [self.substitute(elem, config, variable_namespace) for elem in obj] - elif isinstance(obj, str): + if isinstance(obj, str): + # Skip substitution for JSON $ref strings + if '$ref' in obj: + return obj + # Use a regular expression to find all variables in the string, supporting ${VAR} and $VAR formats def replacer(match): # The first group that is not None is the one that matched var_name = next((g for g in match.groups() if g is not None), "") return self._get_variable(var_name, config, variable_namespace) - return re.sub(r'\${(\w+)}|\$(\w+)', replacer, obj) + return re.sub(r'\${([a-zA-Z0-9_]+)}|\$([a-zA-Z0-9_]+)', replacer, obj) + elif isinstance(obj, dict): + return {k: self.substitute(v, config, variable_namespace) for k, v in obj.items()} + elif isinstance(obj, list): + return [self.substitute(elem, config, variable_namespace) for elem in obj] else: return obj @@ -118,6 +126,10 @@ def find_required_variables(self, obj: dict | list | str, variable_namespace: Op returning fully-qualified variable names with variable namespacing. Useful for validation and dependency analysis. + Note: + Strings containing '$ref' are skipped to support OpenAPI specs + stored as string content, where $ref is a JSON reference keyword. + Args: obj: Object to scan for variable references. variable_namespace: Variable namespace used for variable namespacing. @@ -127,7 +139,7 @@ def find_required_variables(self, obj: dict | list | str, variable_namespace: Op ValueError: If variable_namespace contains invalid characters. Returns: - List of fully-qualified variable names found in the object. + List of unique fully-qualified variable names found in the object. Example: ```python @@ -156,19 +168,22 @@ def find_required_variables(self, obj: dict | list | str, variable_namespace: Op result.extend(vars) return result elif isinstance(obj, str): + # Skip substitution for JSON $ref strings + if '$ref' in obj: + return [] # Find all variables in the string, supporting ${VAR} and $VAR formats variables = [] - pattern = r'\${(\w+)}|\$(\w+)' + pattern = r'\${([a-zA-Z0-9_]+)}|\$([a-zA-Z0-9_]+)' for match in re.finditer(pattern, obj): # The first group that is not None is the one that matched var_name = next(g for g in match.groups() if g is not None) if variable_namespace: - full_var_name = variable_namespace.replace("_", "!").replace("!", "__") + "_" + var_name + full_var_name = variable_namespace.replace("_", "__") + "_" + var_name else: full_var_name = var_name variables.append(full_var_name) - return variables + return list(set(variables)) else: return [] diff --git a/plugins/communication_protocols/cli/pyproject.toml b/plugins/communication_protocols/cli/pyproject.toml index a13fd96..70d2808 100644 --- a/plugins/communication_protocols/cli/pyproject.toml +++ b/plugins/communication_protocols/cli/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-cli" -version = "1.1.0" +version = "1.1.1" authors = [ { name = "UTCP Contributors" }, ] @@ -14,7 +14,7 @@ requires-python = ">=3.10" dependencies = [ "pydantic>=2.0", "pyyaml>=6.0", - "utcp>=1.0" + "utcp>=1.1" ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/file/README.md b/plugins/communication_protocols/file/README.md new file mode 100644 index 0000000..4e85f5a --- /dev/null +++ b/plugins/communication_protocols/file/README.md @@ -0,0 +1,129 @@ +# UTCP File Plugin + +[![PyPI Downloads](https://static.pepy.tech/badge/utcp-file)](https://pepy.tech/projects/utcp-file) + +A file-based resource plugin for UTCP. This plugin allows you to define tools that return the content of a specified local file. + +## Features + +- **Local File Content**: Define tools that read and return the content of local files. +- **UTCP Manual Discovery**: Load tool definitions from local UTCP manual files in JSON or YAML format. +- **OpenAPI Support**: Automatically converts local OpenAPI specs to UTCP tools with optional authentication. +- **Static & Simple**: Ideal for returning mock data, configuration, or any static text content from a file. +- **Version Control**: Tool definitions and their corresponding content files can be versioned with your code. +- **No File Authentication**: Designed for simple, local file access without authentication for file reading. +- **Tool Authentication**: Supports authentication for generated tools from OpenAPI specs via `auth_tools`. + +## Installation + +```bash +pip install utcp-file +``` + +## How It Works + +The File plugin operates in two main ways: + +1. **Tool Discovery (`register_manual`)**: It can read a standard UTCP manual file (e.g., `my-tools.json`) to learn about available tools. This is how the `UtcpClient` discovers what tools can be called. +2. **Tool Execution (`call_tool`)**: When you call a tool, the plugin looks at the `tool_call_template` associated with that tool. It expects a `file` template, and it will read and return the entire content of the `file_path` specified in that template. + +**Important**: The `call_tool` function **does not** use the arguments you pass to it. It simply returns the full content of the file defined in the tool's template. + +## Quick Start + +Here is a complete example demonstrating how to define and use a tool that returns the content of a file. + +### 1. Create a Content File + +First, create a file with some content that you want your tool to return. + +`./mock_data/user.json`: +```json +{ + "id": 123, + "name": "John Doe", + "email": "john.doe@example.com" +} +``` + +### 2. Create a UTCP Manual + +Next, define a UTCP manual that describes your tool. The `tool_call_template` must be of type `file` and point to the content file you just created. + +`./manuals/local_tools.json`: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.2", + "tools": [ + { + "name": "get_mock_user", + "description": "Returns a mock user profile from a local file.", + "tool_call_template": { + "call_template_type": "file", + "file_path": "./mock_data/user.json" + } + } + ] +} +``` + +### 3. Use the Tool in Python + +Finally, use the `UtcpClient` to load the manual and call the tool. + +```python +import asyncio +from utcp.utcp_client import UtcpClient + +async def main(): + # Create a client, providing the path to the manual. + # The file plugin is used automatically for the "file" call_template_type. + client = await UtcpClient.create(config={ + "manual_call_templates": [{ + "name": "local_file_tools", + "call_template_type": "file", + "file_path": "./manuals/local_tools.json" + }] + }) + + # List the tools to confirm it was loaded + tools = await client.list_tools() + print("Available tools:", [tool.name for tool in tools]) + + # Call the tool. The result will be the content of './mock_data/user.json' + result = await client.call_tool("local_file_tools.get_mock_user", {}) + + print("\nTool Result:") + print(result) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Expected Output: + +``` +Available tools: ['local_file_tools.get_mock_user'] + +Tool Result: +{ + "id": 123, + "name": "John Doe", + "email": "john.doe@example.com" +} +``` + +## Use Cases + +- **Mocking**: Return mock data for tests or local development without needing a live server. +- **Configuration**: Load static configuration files as tool outputs. +- **Templates**: Retrieve text templates (e.g., for emails or reports). + +## Related Documentation + +- [Main UTCP Documentation](../../../README.md) +- [Core Package Documentation](../../../core/README.md) +- [HTTP Plugin](../http/README.md) - For calling real web APIs. +- [Text Plugin](../text/README.md) - For direct text content (browser-compatible). +- [CLI Plugin](../cli/README.md) - For executing command-line tools. diff --git a/plugins/communication_protocols/file/pyproject.toml b/plugins/communication_protocols/file/pyproject.toml new file mode 100644 index 0000000..3551f74 --- /dev/null +++ b/plugins/communication_protocols/file/pyproject.toml @@ -0,0 +1,45 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "utcp-file" +version = "1.1.0" +authors = [ + { name = "UTCP Contributors" }, +] +description = "UTCP communication protocol plugin for reading local files." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "pydantic>=2.0", + "pyyaml>=6.0", + "utcp>=1.1", + "utcp-http>=1.1", + "aiofiles>=23.2.1" +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +license = "MPL-2.0" + +[project.optional-dependencies] +dev = [ + "build", + "pytest", + "pytest-asyncio", + "pytest-cov", + "coverage", + "twine", +] + +[project.urls] +Homepage = "https://utcp.io" +Source = "https://github.com/universal-tool-calling-protocol/python-utcp" +Issues = "https://github.com/universal-tool-calling-protocol/python-utcp/issues" + +[project.entry-points."utcp.plugins"] +file = "utcp_file:register" diff --git a/plugins/communication_protocols/file/src/utcp_file/__init__.py b/plugins/communication_protocols/file/src/utcp_file/__init__.py new file mode 100644 index 0000000..f1ac313 --- /dev/null +++ b/plugins/communication_protocols/file/src/utcp_file/__init__.py @@ -0,0 +1,17 @@ +"""File Communication Protocol plugin for UTCP.""" + +from utcp.plugins.discovery import register_communication_protocol, register_call_template +from utcp_file.file_communication_protocol import FileCommunicationProtocol +from utcp_file.file_call_template import FileCallTemplate, FileCallTemplateSerializer + + +def register(): + register_communication_protocol("file", FileCommunicationProtocol()) + register_call_template("file", FileCallTemplateSerializer()) + + +__all__ = [ + "FileCommunicationProtocol", + "FileCallTemplate", + "FileCallTemplateSerializer", +] diff --git a/plugins/communication_protocols/file/src/utcp_file/file_call_template.py b/plugins/communication_protocols/file/src/utcp_file/file_call_template.py new file mode 100644 index 0000000..e343862 --- /dev/null +++ b/plugins/communication_protocols/file/src/utcp_file/file_call_template.py @@ -0,0 +1,66 @@ +from typing import Literal, Optional, Any +from pydantic import Field, field_serializer, field_validator + +from utcp.data.call_template import CallTemplate +from utcp.data.auth import Auth, AuthSerializer +from utcp.interfaces.serializer import Serializer +from utcp.exceptions import UtcpSerializerValidationError +import traceback + + +class FileCallTemplate(CallTemplate): + """REQUIRED + Call template for file-based manuals and tools. + + Reads UTCP manuals or tool definitions from local JSON/YAML files. Useful for + static tool configurations or environments where manuals are distributed as files. + For direct text content, use the text protocol instead. + + Attributes: + call_template_type: Always "file" for file call templates. + file_path: Path to the file containing the UTCP manual or tool definitions. + auth: Always None - file call templates don't support authentication for file access. + auth_tools: Optional authentication to apply to generated tools from OpenAPI specs. + """ + + call_template_type: Literal["file"] = "file" + file_path: str = Field(..., description="The path to the file containing the UTCP manual or tool definitions.") + auth: None = None + auth_tools: Optional[Auth] = Field(None, description="Authentication to apply to generated tools from OpenAPI specs.") + + @field_serializer('auth_tools') + def serialize_auth_tools(self, auth_tools: Optional[Auth]) -> Optional[dict]: + """Serialize auth_tools to dictionary.""" + if auth_tools is None: + return None + return AuthSerializer().to_dict(auth_tools) + + @field_validator('auth_tools', mode='before') + @classmethod + def validate_auth_tools(cls, v: Any) -> Optional[Auth]: + """Validate and deserialize auth_tools from dictionary.""" + if v is None: + return None + if isinstance(v, Auth): + return v + if isinstance(v, dict): + return AuthSerializer().validate_dict(v) + raise ValueError(f"auth_tools must be None, Auth instance, or dict, got {type(v)}") + + +class FileCallTemplateSerializer(Serializer[FileCallTemplate]): + """REQUIRED + Serializer for FileCallTemplate.""" + + def to_dict(self, obj: FileCallTemplate) -> dict: + """REQUIRED + Convert a FileCallTemplate to a dictionary.""" + return obj.model_dump() + + def validate_dict(self, obj: dict) -> FileCallTemplate: + """REQUIRED + Validate and convert a dictionary to a FileCallTemplate.""" + try: + return FileCallTemplate.model_validate(obj) + except Exception as e: + raise UtcpSerializerValidationError("Invalid FileCallTemplate: " + traceback.format_exc()) diff --git a/plugins/communication_protocols/file/src/utcp_file/file_communication_protocol.py b/plugins/communication_protocols/file/src/utcp_file/file_communication_protocol.py new file mode 100644 index 0000000..f6ae27a --- /dev/null +++ b/plugins/communication_protocols/file/src/utcp_file/file_communication_protocol.py @@ -0,0 +1,140 @@ +""" +File communication protocol for UTCP client. + +This protocol reads UTCP manuals (or OpenAPI specs) from local files to register +tools. It does not maintain any persistent connections. +For direct text content, use the text protocol instead. +""" +import json +import yaml +import aiofiles +from pathlib import Path +from typing import Dict, Any, AsyncGenerator, TYPE_CHECKING + +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp.data.call_template import CallTemplate +from utcp.data.utcp_manual import UtcpManual, UtcpManualSerializer +from utcp.data.register_manual_response import RegisterManualResult +from utcp_http.openapi_converter import OpenApiConverter +from utcp_file.file_call_template import FileCallTemplate +import traceback + +if TYPE_CHECKING: + from utcp.utcp_client import UtcpClient + +import logging + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d - %(message)s" +) + +logger = logging.getLogger(__name__) + + +class FileCommunicationProtocol(CommunicationProtocol): + """REQUIRED + Communication protocol for file-based UTCP manuals and tools.""" + + def _log_info(self, message: str) -> None: + logger.info(f"[FileCommunicationProtocol] {message}") + + def _log_error(self, message: str) -> None: + logger.error(f"[FileCommunicationProtocol Error] {message}") + + async def register_manual(self, caller: 'UtcpClient', manual_call_template: CallTemplate) -> RegisterManualResult: + """REQUIRED + Register a file manual and return its tools as a UtcpManual.""" + if not isinstance(manual_call_template, FileCallTemplate): + raise ValueError("FileCommunicationProtocol requires a FileCallTemplate") + + file_path = Path(manual_call_template.file_path) + if not file_path.is_absolute() and caller.root_dir: + file_path = Path(caller.root_dir) / file_path + + self._log_info(f"Reading manual from '{file_path}'") + + try: + if not file_path.exists(): + raise FileNotFoundError(f"Manual file not found: {file_path}") + + async with aiofiles.open(file_path, "r", encoding="utf-8") as f: + file_content = await f.read() + + # Parse based on extension + data: Any + if file_path.suffix.lower() in [".yaml", ".yml"]: + data = yaml.safe_load(file_content) + else: + data = json.loads(file_content) + + utcp_manual: UtcpManual + if isinstance(data, dict) and ("openapi" in data or "swagger" in data or "paths" in data): + self._log_info("Detected OpenAPI specification. Converting to UTCP manual.") + converter = OpenApiConverter( + data, + spec_url=file_path.as_uri(), + call_template_name=manual_call_template.name, + auth_tools=manual_call_template.auth_tools + ) + utcp_manual = converter.convert() + else: + # Try to validate as UTCP manual directly + utcp_manual = UtcpManualSerializer().validate_dict(data) + + self._log_info(f"Loaded {len(utcp_manual.tools)} tools from '{file_path}'") + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=utcp_manual, + success=True, + errors=[], + ) + + except (json.JSONDecodeError, yaml.YAMLError) as e: + self._log_error(f"Failed to parse manual '{file_path}': {traceback.format_exc()}") + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=UtcpManual(tools=[]), + success=False, + errors=[traceback.format_exc()], + ) + except Exception as e: + self._log_error(f"Unexpected error reading manual '{file_path}': {traceback.format_exc()}") + return RegisterManualResult( + manual_call_template=manual_call_template, + manual=UtcpManual(tools=[]), + success=False, + errors=[traceback.format_exc()], + ) + + async def deregister_manual(self, caller: 'UtcpClient', manual_call_template: CallTemplate) -> None: + """REQUIRED + Deregister a file manual (no-op).""" + if isinstance(manual_call_template, FileCallTemplate): + self._log_info(f"Deregistering file manual '{manual_call_template.name}' (no-op)") + + async def call_tool(self, caller: 'UtcpClient', tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate) -> Any: + """REQUIRED + Call a tool: for file templates, return file content from the configured path.""" + if not isinstance(tool_call_template, FileCallTemplate): + raise ValueError("FileCommunicationProtocol requires a FileCallTemplate for tool calls") + + file_path = Path(tool_call_template.file_path) + if not file_path.is_absolute() and caller.root_dir: + file_path = Path(caller.root_dir) / file_path + + self._log_info(f"Reading content from '{file_path}' for tool '{tool_name}'") + + try: + async with aiofiles.open(file_path, "r", encoding="utf-8") as f: + content = await f.read() + return content + except FileNotFoundError: + self._log_error(f"File not found for tool '{tool_name}': {file_path}") + raise + + async def call_tool_streaming(self, caller: 'UtcpClient', tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate) -> AsyncGenerator[Any, None]: + """REQUIRED + Streaming variant: yields the full content as a single chunk.""" + result = await self.call_tool(caller, tool_name, tool_args, tool_call_template) + yield result diff --git a/plugins/communication_protocols/file/tests/test_file_communication_protocol.py b/plugins/communication_protocols/file/tests/test_file_communication_protocol.py new file mode 100644 index 0000000..54ac213 --- /dev/null +++ b/plugins/communication_protocols/file/tests/test_file_communication_protocol.py @@ -0,0 +1,286 @@ +""" +Tests for the File communication protocol (file-based) implementation. +""" +import json +import tempfile +from pathlib import Path +import pytest +import pytest_asyncio +from unittest.mock import Mock + +from utcp_file.file_communication_protocol import FileCommunicationProtocol +from utcp_file.file_call_template import FileCallTemplate +from utcp.data.call_template import CallTemplate +from utcp.data.register_manual_response import RegisterManualResult +from utcp.data.auth_implementations.api_key_auth import ApiKeyAuth +from utcp.utcp_client import UtcpClient + + +@pytest_asyncio.fixture +async def file_protocol() -> FileCommunicationProtocol: + """Provides a FileCommunicationProtocol instance.""" + yield FileCommunicationProtocol() + + +@pytest_asyncio.fixture +def mock_utcp_client(tmp_path: Path) -> Mock: + """Provides a mock UtcpClient with a root_dir.""" + client = Mock(spec=UtcpClient) + client.root_dir = tmp_path + return client + + +@pytest_asyncio.fixture +def sample_utcp_manual(): + """Sample UTCP manual with multiple tools.""" + return { + "utcp_version": "1.0.0", + "manual_version": "1.0.0", + "tools": [ + { + "name": "calculator", + "description": "Performs basic arithmetic operations", + "inputs": { + "type": "object", + "properties": { + "operation": { + "type": "string", + "enum": ["add", "subtract", "multiply", "divide"] + }, + "a": {"type": "number"}, + "b": {"type": "number"} + }, + "required": ["operation", "a", "b"] + }, + "outputs": { + "type": "object", + "properties": { + "result": {"type": "number"} + } + }, + "tags": ["math", "arithmetic"], + "tool_call_template": { + "call_template_type": "file", + "name": "test-file-call-template", + "file_path": "dummy.json" + } + }, + { + "name": "string_utils", + "description": "String manipulation utilities", + "inputs": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "operation": { + "type": "string", + "enum": ["uppercase", "lowercase", "reverse"] + } + }, + "required": ["text", "operation"] + }, + "outputs": { + "type": "object", + "properties": { + "result": {"type": "string"} + } + }, + "tags": ["text", "utilities"], + "tool_call_template": { + "call_template_type": "file", + "name": "test-file-call-template", + "file_path": "dummy.json" + } + } + ] + } + + +@pytest.mark.asyncio +async def test_register_manual_with_utcp_manual( + file_protocol: FileCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock +): + """Register a manual from a local file and validate returned tools.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(sample_utcp_manual, f) + temp_file = f.name + + try: + manual_template = FileCallTemplate(name="test_manual", file_path=temp_file) + result = await file_protocol.register_manual(mock_utcp_client, manual_template) + + assert isinstance(result, RegisterManualResult) + assert result.success is True + assert result.errors == [] + assert result.manual is not None + assert len(result.manual.tools) == 2 + + tool0 = result.manual.tools[0] + assert tool0.name == "calculator" + assert tool0.description == "Performs basic arithmetic operations" + assert tool0.tags == ["math", "arithmetic"] + assert tool0.tool_call_template.call_template_type == "file" + + tool1 = result.manual.tools[1] + assert tool1.name == "string_utils" + assert tool1.description == "String manipulation utilities" + assert tool1.tags == ["text", "utilities"] + assert tool1.tool_call_template.call_template_type == "file" + finally: + Path(temp_file).unlink() + + +@pytest.mark.asyncio +async def test_register_manual_file_not_found( + file_protocol: FileCommunicationProtocol, mock_utcp_client: Mock +): + """Registering a manual with a non-existent file should return errors.""" + manual_template = FileCallTemplate(name="missing", file_path="/path/that/does/not/exist.json") + result = await file_protocol.register_manual(mock_utcp_client, manual_template) + assert isinstance(result, RegisterManualResult) + assert result.success is False + assert result.errors + + +@pytest.mark.asyncio +async def test_register_manual_invalid_json( + file_protocol: FileCommunicationProtocol, mock_utcp_client: Mock +): + """Registering a manual with invalid JSON should return errors (no exception).""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write("{ invalid json content }") + temp_file = f.name + + try: + manual_template = FileCallTemplate(name="invalid_json", file_path=temp_file) + result = await file_protocol.register_manual(mock_utcp_client, manual_template) + assert isinstance(result, RegisterManualResult) + assert result.success is False + assert result.errors + finally: + Path(temp_file).unlink() + + +@pytest.mark.asyncio +async def test_register_manual_wrong_call_template_type(file_protocol: FileCommunicationProtocol, mock_utcp_client: Mock): + """Registering with a non-File call template should raise ValueError.""" + wrong_template = CallTemplate(call_template_type="invalid", name="wrong") + with pytest.raises(ValueError, match="requires a FileCallTemplate"): + await file_protocol.register_manual(mock_utcp_client, wrong_template) + + +@pytest.mark.asyncio +async def test_call_tool_returns_file_content( + file_protocol: FileCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock +): + """Calling a tool returns the file content from the call template path.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(sample_utcp_manual, f) + temp_file = f.name + + try: + tool_template = FileCallTemplate(name="tool_call", file_path=temp_file) + + # Call a tool should return the file content + content = await file_protocol.call_tool( + mock_utcp_client, "calculator", {"operation": "add", "a": 1, "b": 2}, tool_template + ) + + # Verify we get the JSON content back as a string + assert isinstance(content, str) + # Parse it back to verify it's the same content + parsed_content = json.loads(content) + assert parsed_content == sample_utcp_manual + finally: + Path(temp_file).unlink() + + +@pytest.mark.asyncio +async def test_call_tool_wrong_call_template_type(file_protocol: FileCommunicationProtocol, mock_utcp_client: Mock): + """Calling a tool with wrong call template type should raise ValueError.""" + wrong_template = CallTemplate(call_template_type="invalid", name="wrong") + with pytest.raises(ValueError, match="requires a FileCallTemplate"): + await file_protocol.call_tool(mock_utcp_client, "some_tool", {}, wrong_template) + + +@pytest.mark.asyncio +async def test_call_tool_file_not_found(file_protocol: FileCommunicationProtocol, mock_utcp_client: Mock): + """Calling a tool when the file doesn't exist should raise FileNotFoundError.""" + tool_template = FileCallTemplate(name="missing", file_path="/path/that/does/not/exist.json") + with pytest.raises(FileNotFoundError): + await file_protocol.call_tool(mock_utcp_client, "some_tool", {}, tool_template) + + +@pytest.mark.asyncio +async def test_deregister_manual(file_protocol: FileCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock): + """Deregistering a manual should be a no-op (no errors).""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(sample_utcp_manual, f) + temp_file = f.name + + try: + manual_template = FileCallTemplate(name="test_manual", file_path=temp_file) + await file_protocol.deregister_manual(mock_utcp_client, manual_template) + finally: + Path(temp_file).unlink() + + +@pytest.mark.asyncio +async def test_call_tool_streaming(file_protocol: FileCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock): + """Streaming call should yield a single chunk equal to non-streaming content.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(sample_utcp_manual, f) + temp_file = f.name + + try: + tool_template = FileCallTemplate(name="tool_call", file_path=temp_file) + # Non-streaming + content = await file_protocol.call_tool(mock_utcp_client, "calculator", {}, tool_template) + # Streaming + stream = file_protocol.call_tool_streaming(mock_utcp_client, "calculator", {}, tool_template) + chunks = [c async for c in stream] + assert chunks == [content] + finally: + Path(temp_file).unlink() + + +@pytest.mark.asyncio +async def test_file_call_template_with_auth_tools(): + """Test that FileCallTemplate can be created with auth_tools.""" + auth_tools = ApiKeyAuth(api_key="test-key", var_name="Authorization", location="header") + + template = FileCallTemplate( + name="test-template", + file_path="test.json", + auth_tools=auth_tools + ) + + assert template.auth_tools == auth_tools + assert template.auth is None # auth should still be None for file access + + +@pytest.mark.asyncio +async def test_file_call_template_auth_tools_serialization(): + """Test that auth_tools field properly serializes and validates from dict.""" + # Test creation from dict + template_dict = { + "name": "test-template", + "call_template_type": "file", + "file_path": "test.json", + "auth_tools": { + "auth_type": "api_key", + "api_key": "test-key", + "var_name": "Authorization", + "location": "header" + } + } + + template = FileCallTemplate(**template_dict) + assert template.auth_tools is not None + assert template.auth_tools.api_key == "test-key" + assert template.auth_tools.var_name == "Authorization" + + # Test serialization to dict + serialized = template.model_dump() + assert serialized["auth_tools"]["auth_type"] == "api_key" + assert serialized["auth_tools"]["api_key"] == "test-key" diff --git a/plugins/communication_protocols/gql/pyproject.toml b/plugins/communication_protocols/gql/pyproject.toml index d5b558d..4377268 100644 --- a/plugins/communication_protocols/gql/pyproject.toml +++ b/plugins/communication_protocols/gql/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-gql" -version = "1.0.2" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] @@ -14,7 +14,7 @@ requires-python = ">=3.10" dependencies = [ "pydantic>=2.0", "gql>=3.0", - "utcp>=1.0" + "utcp>=1.1" ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/http/pyproject.toml b/plugins/communication_protocols/http/pyproject.toml index 42f0951..ae759ed 100644 --- a/plugins/communication_protocols/http/pyproject.toml +++ b/plugins/communication_protocols/http/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-http" -version = "1.0.5" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] @@ -16,7 +16,7 @@ dependencies = [ "authlib>=1.0", "aiohttp>=3.8", "pyyaml>=6.0", - "utcp>=1.0" + "utcp>=1.1" ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/mcp/pyproject.toml b/plugins/communication_protocols/mcp/pyproject.toml index 2efd4c3..2cfd9a2 100644 --- a/plugins/communication_protocols/mcp/pyproject.toml +++ b/plugins/communication_protocols/mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-mcp" -version = "1.0.2" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] @@ -14,7 +14,7 @@ requires-python = ">=3.11" dependencies = [ "pydantic>=2.0", "mcp>=1.12", - "utcp>=1.0", + "utcp>=1.1", "mcp-use>=1.3", "langchain>=0.3.27,<0.4.0", ] diff --git a/plugins/communication_protocols/socket/pyproject.toml b/plugins/communication_protocols/socket/pyproject.toml index a544648..dbbc1b0 100644 --- a/plugins/communication_protocols/socket/pyproject.toml +++ b/plugins/communication_protocols/socket/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-socket" -version = "1.0.2" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] @@ -13,7 +13,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "pydantic>=2.0", - "utcp>=1.0" + "utcp>=1.1" ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/text/README.md b/plugins/communication_protocols/text/README.md index 27f8525..ea875d0 100644 --- a/plugins/communication_protocols/text/README.md +++ b/plugins/communication_protocols/text/README.md @@ -2,16 +2,15 @@ [![PyPI Downloads](https://static.pepy.tech/badge/utcp-text)](https://pepy.tech/projects/utcp-text) -A simple, file-based resource plugin for UTCP. This plugin allows you to define tools that return the content of a specified local file. +A text content plugin for UTCP. This plugin allows you to pass UTCP manuals or tool definitions directly as text content, without requiring file system access. It's browser-compatible and ideal for embedded configurations. ## Features -- **Local File Content**: Define tools that read and return the content of local files. -- **UTCP Manual Discovery**: Load tool definitions from local UTCP manual files in JSON or YAML format. -- **OpenAPI Support**: Automatically converts local OpenAPI specs to UTCP tools with optional authentication. -- **Static & Simple**: Ideal for returning mock data, configuration, or any static text content from a file. -- **Version Control**: Tool definitions and their corresponding content files can be versioned with your code. -- **No File Authentication**: Designed for simple, local file access without authentication for file reading. +- **Direct Text Content**: Pass UTCP manuals or tool definitions directly as strings. +- **Browser Compatible**: No file system access required, works in browser environments. +- **JSON & YAML Support**: Parses both JSON and YAML formatted content. +- **OpenAPI Support**: Automatically converts OpenAPI specs to UTCP tools with optional authentication. +- **Base URL Override**: Override API base URLs when converting OpenAPI specs. - **Tool Authentication**: Supports authentication for generated tools from OpenAPI specs via `auth_tools`. ## Installation @@ -24,66 +23,49 @@ pip install utcp-text The Text plugin operates in two main ways: -1. **Tool Discovery (`register_manual`)**: It can read a standard UTCP manual file (e.g., `my-tools.json`) to learn about available tools. This is how the `UtcpClient` discovers what tools can be called. -2. **Tool Execution (`call_tool`)**: When you call a tool, the plugin looks at the `tool_call_template` associated with that tool. It expects a `text` template, and it will read and return the entire content of the `file_path` specified in that template. +1. **Tool Discovery (`register_manual`)**: It parses the `content` field directly as a UTCP manual or OpenAPI spec. This is how the `UtcpClient` discovers what tools can be called. +2. **Tool Execution (`call_tool`)**: When you call a tool, the plugin returns the `content` field directly. -**Important**: The `call_tool` function **does not** use the arguments you pass to it. It simply returns the full content of the file defined in the tool's template. +**Note**: For file-based tool definitions, use the `utcp-file` plugin instead. ## Quick Start -Here is a complete example demonstrating how to define and use a tool that returns the content of a file. +Here is a complete example demonstrating how to define and use tools with direct text content. -### 1. Create a Content File - -First, create a file with some content that you want your tool to return. - -`./mock_data/user.json`: -```json -{ - "id": 123, - "name": "John Doe", - "email": "john.doe@example.com" -} -``` - -### 2. Create a UTCP Manual - -Next, define a UTCP manual that describes your tool. The `tool_call_template` must be of type `text` and point to the content file you just created. - -`./manuals/local_tools.json`: -```json -{ - "manual_version": "1.0.0", - "utcp_version": "1.0.2", - "tools": [ - { - "name": "get_mock_user", - "description": "Returns a mock user profile from a local file.", - "tool_call_template": { - "call_template_type": "text", - "file_path": "./mock_data/user.json" - } - } - ] -} -``` - -### 3. Use the Tool in Python - -Finally, use the `UtcpClient` to load the manual and call the tool. +### 1. Define Tools with Inline Content ```python import asyncio +import json from utcp.utcp_client import UtcpClient +# Define a UTCP manual as a Python dict, then convert to JSON string +manual_content = json.dumps({ + "manual_version": "1.0.0", + "utcp_version": "1.0.2", + "tools": [ + { + "name": "get_mock_user", + "description": "Returns a mock user profile.", + "tool_call_template": { + "call_template_type": "text", + "content": json.dumps({ + "id": 123, + "name": "John Doe", + "email": "john.doe@example.com" + }) + } + } + ] +}) + async def main(): - # Create a client, providing the path to the manual. - # The text plugin is used automatically for the "text" call_template_type. + # Create a client with direct text content client = await UtcpClient.create(config={ "manual_call_templates": [{ - "name": "local_file_tools", + "name": "inline_tools", "call_template_type": "text", - "file_path": "./manuals/local_tools.json" + "content": manual_content }] }) @@ -91,8 +73,8 @@ async def main(): tools = await client.list_tools() print("Available tools:", [tool.name for tool in tools]) - # Call the tool. The result will be the content of './mock_data/user.json' - result = await client.call_tool("local_file_tools.get_mock_user", {}) + # Call the tool + result = await client.call_tool("inline_tools.get_mock_user", {}) print("\nTool Result:") print(result) @@ -101,28 +83,58 @@ if __name__ == "__main__": asyncio.run(main()) ``` -### Expected Output: +### 2. Using with OpenAPI Specs -``` -Available tools: ['local_file_tools.get_mock_user'] - -Tool Result: -{ - "id": 123, - "name": "John Doe", - "email": "john.doe@example.com" -} +You can also pass OpenAPI specs directly as text content: + +```python +import asyncio +import json +from utcp.utcp_client import UtcpClient + +openapi_spec = json.dumps({ + "openapi": "3.0.0", + "info": {"title": "Pet Store", "version": "1.0.0"}, + "servers": [{"url": "https://api.example.com"}], + "paths": { + "/pets": { + "get": { + "operationId": "listPets", + "summary": "List all pets", + "responses": {"200": {"description": "Success"}} + } + } + } +}) + +async def main(): + client = await UtcpClient.create(config={ + "manual_call_templates": [{ + "name": "pet_api", + "call_template_type": "text", + "content": openapi_spec, + "base_url": "https://api.petstore.io/v1" # Optional: override base URL + }] + }) + + tools = await client.list_tools() + print("Available tools:", [tool.name for tool in tools]) + +if __name__ == "__main__": + asyncio.run(main()) ``` ## Use Cases -- **Mocking**: Return mock data for tests or local development without needing a live server. -- **Configuration**: Load static configuration files as tool outputs. -- **Templates**: Retrieve text templates (e.g., for emails or reports). +- **Embedded Configurations**: Embed tool definitions directly in your application code. +- **Browser Applications**: Use UTCP in browser environments without file system access. +- **Dynamic Tool Generation**: Generate tool definitions programmatically at runtime. +- **Testing**: Define mock tools inline for unit tests. ## Related Documentation - [Main UTCP Documentation](../../../README.md) - [Core Package Documentation](../../../core/README.md) +- [File Plugin](../file/README.md) - For file-based tool definitions. - [HTTP Plugin](../http/README.md) - For calling real web APIs. - [CLI Plugin](../cli/README.md) - For executing command-line tools. diff --git a/plugins/communication_protocols/text/pyproject.toml b/plugins/communication_protocols/text/pyproject.toml index c624e8c..b6bcfce 100644 --- a/plugins/communication_protocols/text/pyproject.toml +++ b/plugins/communication_protocols/text/pyproject.toml @@ -4,19 +4,18 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-text" -version = "1.0.3" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] -description = "UTCP communication protocol plugin for reading text files." +description = "UTCP communication protocol plugin for direct text content (browser-compatible)." readme = "README.md" requires-python = ">=3.10" dependencies = [ "pydantic>=2.0", "pyyaml>=6.0", - "utcp>=1.0", - "utcp-http>=1.0", - "aiofiles>=23.2.1" + "utcp>=1.1", + "utcp-http>=1.1" ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/plugins/communication_protocols/text/src/utcp_text/text_call_template.py b/plugins/communication_protocols/text/src/utcp_text/text_call_template.py index a090817..f5ca2c4 100644 --- a/plugins/communication_protocols/text/src/utcp_text/text_call_template.py +++ b/plugins/communication_protocols/text/src/utcp_text/text_call_template.py @@ -7,22 +7,27 @@ from utcp.exceptions import UtcpSerializerValidationError import traceback + class TextCallTemplate(CallTemplate): """REQUIRED - Call template for text file-based manuals and tools. + Text call template for UTCP client. - Reads UTCP manuals or tool definitions from local JSON/YAML files. Useful for - static tool configurations or environments where manuals are distributed as files. + This template allows passing UTCP manuals or tool definitions directly as text content. + It supports both JSON and YAML formats and can convert OpenAPI specifications to UTCP manuals. + It's browser-compatible and requires no file system access. + For file-based manuals, use the file protocol instead. Attributes: - call_template_type: Always "text" for text file call templates. - file_path: Path to the file containing the UTCP manual or tool definitions. - auth: Always None - text call templates don't support authentication for file access. + call_template_type: Always "text" for text call templates. + content: Direct text content of the UTCP manual or tool definitions (required). + base_url: Optional base URL for API endpoints when converting OpenAPI specs. + auth: Always None - text call templates don't support authentication. auth_tools: Optional authentication to apply to generated tools from OpenAPI specs. """ call_template_type: Literal["text"] = "text" - file_path: str = Field(..., description="The path to the file containing the UTCP manual or tool definitions.") + content: str = Field(..., description="Direct text content of the UTCP manual or tool definitions.") + base_url: Optional[str] = Field(None, description="Optional base URL for API endpoints when converting OpenAPI specs.") auth: None = None auth_tools: Optional[Auth] = Field(None, description="Authentication to apply to generated tools from OpenAPI specs.") @@ -58,6 +63,14 @@ def to_dict(self, obj: TextCallTemplate) -> dict: def validate_dict(self, obj: dict) -> TextCallTemplate: """REQUIRED Validate and convert a dictionary to a TextCallTemplate.""" + # Check for old file_path field and provide helpful migration message + if "file_path" in obj: + raise UtcpSerializerValidationError( + "TextCallTemplate no longer supports 'file_path'. " + "The text protocol now accepts direct content via the 'content' field. " + "For file-based manuals, use the 'file' protocol instead (call_template_type: 'file'). " + "Install with: pip install utcp-file" + ) try: return TextCallTemplate.model_validate(obj) except Exception as e: diff --git a/plugins/communication_protocols/text/src/utcp_text/text_communication_protocol.py b/plugins/communication_protocols/text/src/utcp_text/text_communication_protocol.py index cdd49ae..c979191 100644 --- a/plugins/communication_protocols/text/src/utcp_text/text_communication_protocol.py +++ b/plugins/communication_protocols/text/src/utcp_text/text_communication_protocol.py @@ -1,15 +1,13 @@ """ Text communication protocol for UTCP client. -This protocol reads UTCP manuals (or OpenAPI specs) from local files to register -tools. It does not maintain any persistent connections. +This protocol parses UTCP manuals (or OpenAPI specs) from direct text content. +It's browser-compatible and requires no file system access. +For file-based manuals, use the file protocol instead. """ import json -import sys import yaml -import aiofiles -from pathlib import Path -from typing import Dict, Any, Optional, AsyncGenerator, TYPE_CHECKING +from typing import Dict, Any, AsyncGenerator, TYPE_CHECKING from utcp.interfaces.communication_protocol import CommunicationProtocol from utcp.data.call_template import CallTemplate @@ -31,9 +29,10 @@ logger = logging.getLogger(__name__) + class TextCommunicationProtocol(CommunicationProtocol): """REQUIRED - Communication protocol for file-based UTCP manuals and tools.""" + Communication protocol for text-based UTCP manuals and tools.""" def _log_info(self, message: str) -> None: logger.info(f"[TextCommunicationProtocol] {message}") @@ -47,41 +46,37 @@ async def register_manual(self, caller: 'UtcpClient', manual_call_template: Call if not isinstance(manual_call_template, TextCallTemplate): raise ValueError("TextCommunicationProtocol requires a TextCallTemplate") - file_path = Path(manual_call_template.file_path) - if not file_path.is_absolute() and caller.root_dir: - file_path = Path(caller.root_dir) / file_path - - self._log_info(f"Reading manual from '{file_path}'") - try: - if not file_path.exists(): - raise FileNotFoundError(f"Manual file not found: {file_path}") - - async with aiofiles.open(file_path, "r", encoding="utf-8") as f: - file_content = await f.read() + self._log_info("Parsing direct content for manual") + content = manual_call_template.content - # Parse based on extension + # Try JSON first, then YAML data: Any - if file_path.suffix.lower() in [".yaml", ".yml"]: - data = yaml.safe_load(file_content) - else: - data = json.loads(file_content) + try: + data = json.loads(content) + except json.JSONDecodeError as json_error: + try: + data = yaml.safe_load(content) + except yaml.YAMLError: + raise ValueError(f"Failed to parse content as JSON or YAML: {json_error}") utcp_manual: UtcpManual if isinstance(data, dict) and ("openapi" in data or "swagger" in data or "paths" in data): self._log_info("Detected OpenAPI specification. Converting to UTCP manual.") converter = OpenApiConverter( - data, - spec_url=file_path.as_uri(), + data, + spec_url="text://content", call_template_name=manual_call_template.name, - auth_tools=manual_call_template.auth_tools + auth_tools=manual_call_template.auth_tools, + base_url=manual_call_template.base_url ) utcp_manual = converter.convert() else: # Try to validate as UTCP manual directly + self._log_info("Validating content as UTCP manual.") utcp_manual = UtcpManualSerializer().validate_dict(data) - self._log_info(f"Loaded {len(utcp_manual.tools)} tools from '{file_path}'") + self._log_info(f"Successfully registered manual with {len(utcp_manual.tools)} tools.") return RegisterManualResult( manual_call_template=manual_call_template, manual=utcp_manual, @@ -89,21 +84,14 @@ async def register_manual(self, caller: 'UtcpClient', manual_call_template: Call errors=[], ) - except (json.JSONDecodeError, yaml.YAMLError) as e: - self._log_error(f"Failed to parse manual '{file_path}': {traceback.format_exc()}") - return RegisterManualResult( - manual_call_template=manual_call_template, - manual=UtcpManual(tools=[]), - success=False, - errors=[traceback.format_exc()], - ) except Exception as e: - self._log_error(f"Unexpected error reading manual '{file_path}': {traceback.format_exc()}") + err_msg = f"Failed to register text manual: {str(e)}" + self._log_error(err_msg) return RegisterManualResult( manual_call_template=manual_call_template, manual=UtcpManual(tools=[]), success=False, - errors=[traceback.format_exc()], + errors=[err_msg], ) async def deregister_manual(self, caller: 'UtcpClient', manual_call_template: CallTemplate) -> None: @@ -114,23 +102,12 @@ async def deregister_manual(self, caller: 'UtcpClient', manual_call_template: Ca async def call_tool(self, caller: 'UtcpClient', tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate) -> Any: """REQUIRED - Call a tool: for text templates, return file content from the configured path.""" + Execute a tool call. Text protocol returns the content directly.""" if not isinstance(tool_call_template, TextCallTemplate): raise ValueError("TextCommunicationProtocol requires a TextCallTemplate for tool calls") - file_path = Path(tool_call_template.file_path) - if not file_path.is_absolute() and caller.root_dir: - file_path = Path(caller.root_dir) / file_path - - self._log_info(f"Reading content from '{file_path}' for tool '{tool_name}'") - - try: - async with aiofiles.open(file_path, "r", encoding="utf-8") as f: - content = await f.read() - return content - except FileNotFoundError: - self._log_error(f"File not found for tool '{tool_name}': {file_path}") - raise + self._log_info(f"Returning direct content for tool '{tool_name}'") + return tool_call_template.content async def call_tool_streaming(self, caller: 'UtcpClient', tool_name: str, tool_args: Dict[str, Any], tool_call_template: CallTemplate) -> AsyncGenerator[Any, None]: """REQUIRED diff --git a/plugins/communication_protocols/text/tests/test_text_communication_protocol.py b/plugins/communication_protocols/text/tests/test_text_communication_protocol.py index 179b34c..f2829ef 100644 --- a/plugins/communication_protocols/text/tests/test_text_communication_protocol.py +++ b/plugins/communication_protocols/text/tests/test_text_communication_protocol.py @@ -1,9 +1,7 @@ """ -Tests for the Text communication protocol (file-based) implementation. +Tests for the Text communication protocol (direct content) implementation. """ import json -import tempfile -from pathlib import Path import pytest import pytest_asyncio from unittest.mock import Mock @@ -15,6 +13,7 @@ from utcp.data.auth_implementations.api_key_auth import ApiKeyAuth from utcp.utcp_client import UtcpClient + @pytest_asyncio.fixture async def text_protocol() -> TextCommunicationProtocol: """Provides a TextCommunicationProtocol instance.""" @@ -22,16 +21,16 @@ async def text_protocol() -> TextCommunicationProtocol: @pytest_asyncio.fixture -def mock_utcp_client(tmp_path: Path) -> Mock: - """Provides a mock UtcpClient with a root_dir.""" +def mock_utcp_client() -> Mock: + """Provides a mock UtcpClient.""" client = Mock(spec=UtcpClient) - client.root_dir = tmp_path + client.root_dir = None return client @pytest_asyncio.fixture def sample_utcp_manual(): - """Sample UTCP manual with multiple tools (new UTCP format).""" + """Sample UTCP manual with multiple tools.""" return { "utcp_version": "1.0.0", "manual_version": "1.0.0", @@ -61,7 +60,7 @@ def sample_utcp_manual(): "tool_call_template": { "call_template_type": "text", "name": "test-text-call-template", - "file_path": "dummy.json" + "content": "dummy content" } }, { @@ -88,226 +87,107 @@ def sample_utcp_manual(): "tool_call_template": { "call_template_type": "text", "name": "test-text-call-template", - "file_path": "dummy.json" + "content": "dummy content" } } ] } -@pytest_asyncio.fixture -def single_tool_definition(): - """Sample single tool definition (new UTCP format).""" - return { - "name": "echo", - "description": "Echoes back the input text", - "inputs": { - "type": "object", - "properties": { - "message": {"type": "string"} - }, - "required": ["message"] - }, - "outputs": { - "type": "object", - "properties": { - "echo": {"type": "string"} - } - }, - "tags": ["utility"], - "tool_call_template": { - "call_template_type": "text", - "name": "test-text-call-template", - "file_path": "dummy.json" - } - } - - -@pytest_asyncio.fixture -def tool_array(): - """Sample array of tool definitions (new UTCP format).""" - return [ - { - "name": "tool1", - "description": "First tool", - "inputs": {"type": "object", "properties": {}, "required": []}, - "outputs": {"type": "object", "properties": {}, "required": []}, - "tags": [], - "tool_call_template": { - "call_template_type": "text", - "name": "test-text-call-template", - "file_path": "dummy.json" - } - }, - { - "name": "tool2", - "description": "Second tool", - "inputs": {"type": "object", "properties": {}, "required": []}, - "outputs": {"type": "object", "properties": {}, "required": []}, - "tags": [], - "tool_call_template": { - "call_template_type": "text", - "name": "test-text-call-template", - "file_path": "dummy.json" - } - } - ] - - @pytest.mark.asyncio async def test_register_manual_with_utcp_manual( text_protocol: TextCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock ): - """Register a manual from a local file and validate returned tools.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(sample_utcp_manual, f) - temp_file = f.name - - try: - manual_template = TextCallTemplate(name="test_manual", file_path=temp_file) - result = await text_protocol.register_manual(mock_utcp_client, manual_template) - - assert isinstance(result, RegisterManualResult) - assert result.success is True - assert result.errors == [] - assert result.manual is not None - assert len(result.manual.tools) == 2 - - tool0 = result.manual.tools[0] - assert tool0.name == "calculator" - assert tool0.description == "Performs basic arithmetic operations" - assert tool0.tags == ["math", "arithmetic"] - assert tool0.tool_call_template.call_template_type == "text" - - tool1 = result.manual.tools[1] - assert tool1.name == "string_utils" - assert tool1.description == "String manipulation utilities" - assert tool1.tags == ["text", "utilities"] - assert tool1.tool_call_template.call_template_type == "text" - finally: - Path(temp_file).unlink() - + """Register a manual from direct content and validate returned tools.""" + content = json.dumps(sample_utcp_manual) + manual_template = TextCallTemplate(name="test_manual", content=content) + result = await text_protocol.register_manual(mock_utcp_client, manual_template) -@pytest.mark.asyncio -async def test_register_manual_with_single_tool( - text_protocol: TextCommunicationProtocol, single_tool_definition, mock_utcp_client: Mock -): - """Register a manual with a single tool definition.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - manual = { - "utcp_version": "1.0.0", - "manual_version": "1.0.0", - "tools": [single_tool_definition], - } - json.dump(manual, f) - temp_file = f.name + assert isinstance(result, RegisterManualResult) + assert result.success is True + assert result.errors == [] + assert result.manual is not None + assert len(result.manual.tools) == 2 - try: - manual_template = TextCallTemplate(name="single_tool_manual", file_path=temp_file) - result = await text_protocol.register_manual(mock_utcp_client, manual_template) + tool0 = result.manual.tools[0] + assert tool0.name == "calculator" + assert tool0.description == "Performs basic arithmetic operations" + assert tool0.tags == ["math", "arithmetic"] + assert tool0.tool_call_template.call_template_type == "text" - assert result.success is True - assert len(result.manual.tools) == 1 - tool = result.manual.tools[0] - assert tool.name == "echo" - assert tool.description == "Echoes back the input text" - assert tool.tags == ["utility"] - assert tool.tool_call_template.call_template_type == "text" - finally: - Path(temp_file).unlink() + tool1 = result.manual.tools[1] + assert tool1.name == "string_utils" + assert tool1.description == "String manipulation utilities" + assert tool1.tags == ["text", "utilities"] + assert tool1.tool_call_template.call_template_type == "text" @pytest.mark.asyncio -async def test_register_manual_with_tool_array( - text_protocol: TextCommunicationProtocol, tool_array, mock_utcp_client: Mock +async def test_register_manual_with_yaml_content( + text_protocol: TextCommunicationProtocol, mock_utcp_client: Mock ): - """Register a manual with an array of tool definitions.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - manual = { - "utcp_version": "1.0.0", - "manual_version": "1.0.0", - "tools": tool_array, - } - json.dump(manual, f) - temp_file = f.name - - try: - manual_template = TextCallTemplate(name="tool_array_manual", file_path=temp_file) - result = await text_protocol.register_manual(mock_utcp_client, manual_template) + """Register a manual from YAML content.""" + yaml_content = """ +utcp_version: "1.0.0" +manual_version: "1.0.0" +tools: + - name: yaml_tool + description: A tool defined in YAML + inputs: + type: object + properties: {} + outputs: + type: object + properties: {} + tags: [] + tool_call_template: + call_template_type: text + content: "test" +""" + manual_template = TextCallTemplate(name="yaml_manual", content=yaml_content) + result = await text_protocol.register_manual(mock_utcp_client, manual_template) - assert result.success is True - assert len(result.manual.tools) == 2 - assert result.manual.tools[0].name == "tool1" - assert result.manual.tools[1].name == "tool2" - assert result.manual.tools[0].tool_call_template.call_template_type == "text" - assert result.manual.tools[1].tool_call_template.call_template_type == "text" - finally: - Path(temp_file).unlink() + assert result.success is True + assert len(result.manual.tools) == 1 + assert result.manual.tools[0].name == "yaml_tool" @pytest.mark.asyncio -async def test_register_manual_file_not_found( +async def test_register_manual_invalid_json( text_protocol: TextCommunicationProtocol, mock_utcp_client: Mock ): - """Registering a manual with a non-existent file should return errors.""" - manual_template = TextCallTemplate(name="missing", file_path="/path/that/does/not/exist.json") + """Registering a manual with invalid content should return errors.""" + manual_template = TextCallTemplate(name="invalid", content="{ invalid json content }") result = await text_protocol.register_manual(mock_utcp_client, manual_template) assert isinstance(result, RegisterManualResult) assert result.success is False assert result.errors -@pytest.mark.asyncio -async def test_register_manual_invalid_json( - text_protocol: TextCommunicationProtocol, mock_utcp_client: Mock -): - """Registering a manual with invalid JSON should return errors (no exception).""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - f.write("{ invalid json content }") - temp_file = f.name - - try: - manual_template = TextCallTemplate(name="invalid_json", file_path=temp_file) - result = await text_protocol.register_manual(mock_utcp_client, manual_template) - assert isinstance(result, RegisterManualResult) - assert result.success is False - assert result.errors - finally: - Path(temp_file).unlink() - - @pytest.mark.asyncio async def test_register_manual_wrong_call_template_type(text_protocol: TextCommunicationProtocol, mock_utcp_client: Mock): """Registering with a non-Text call template should raise ValueError.""" wrong_template = CallTemplate(call_template_type="invalid", name="wrong") with pytest.raises(ValueError, match="requires a TextCallTemplate"): - await text_protocol.register_manual(mock_utcp_client, wrong_template) # type: ignore[arg-type] + await text_protocol.register_manual(mock_utcp_client, wrong_template) @pytest.mark.asyncio -async def test_call_tool_returns_file_content( +async def test_call_tool_returns_content( text_protocol: TextCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock ): - """Calling a tool returns the file content from the call template path.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(sample_utcp_manual, f) - temp_file = f.name - - try: - tool_template = TextCallTemplate(name="tool_call", file_path=temp_file) + """Calling a tool returns the content directly.""" + content = json.dumps(sample_utcp_manual) + tool_template = TextCallTemplate(name="tool_call", content=content) - # Call a tool should return the file content - content = await text_protocol.call_tool( - mock_utcp_client, "calculator", {"operation": "add", "a": 1, "b": 2}, tool_template - ) + # Call a tool should return the content directly + result = await text_protocol.call_tool( + mock_utcp_client, "calculator", {"operation": "add", "a": 1, "b": 2}, tool_template + ) - # Verify we get the JSON content back as a string - assert isinstance(content, str) - # Parse it back to verify it's the same content - parsed_content = json.loads(content) - assert parsed_content == sample_utcp_manual - finally: - Path(temp_file).unlink() + # Verify we get the content back as-is + assert isinstance(result, str) + assert result == content @pytest.mark.asyncio @@ -315,48 +195,29 @@ async def test_call_tool_wrong_call_template_type(text_protocol: TextCommunicati """Calling a tool with wrong call template type should raise ValueError.""" wrong_template = CallTemplate(call_template_type="invalid", name="wrong") with pytest.raises(ValueError, match="requires a TextCallTemplate"): - await text_protocol.call_tool(mock_utcp_client, "some_tool", {}, wrong_template) # type: ignore[arg-type] - - -@pytest.mark.asyncio -async def test_call_tool_file_not_found(text_protocol: TextCommunicationProtocol, mock_utcp_client: Mock): - """Calling a tool when the file doesn't exist should raise FileNotFoundError.""" - tool_template = TextCallTemplate(name="missing", file_path="/path/that/does/not/exist.json") - with pytest.raises(FileNotFoundError): - await text_protocol.call_tool(mock_utcp_client, "some_tool", {}, tool_template) + await text_protocol.call_tool(mock_utcp_client, "some_tool", {}, wrong_template) @pytest.mark.asyncio async def test_deregister_manual(text_protocol: TextCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock): """Deregistering a manual should be a no-op (no errors).""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(sample_utcp_manual, f) - temp_file = f.name - - try: - manual_template = TextCallTemplate(name="test_manual", file_path=temp_file) - await text_protocol.deregister_manual(mock_utcp_client, manual_template) - finally: - Path(temp_file).unlink() + content = json.dumps(sample_utcp_manual) + manual_template = TextCallTemplate(name="test_manual", content=content) + await text_protocol.deregister_manual(mock_utcp_client, manual_template) @pytest.mark.asyncio async def test_call_tool_streaming(text_protocol: TextCommunicationProtocol, sample_utcp_manual, mock_utcp_client: Mock): """Streaming call should yield a single chunk equal to non-streaming content.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(sample_utcp_manual, f) - temp_file = f.name - - try: - tool_template = TextCallTemplate(name="tool_call", file_path=temp_file) - # Non-streaming - content = await text_protocol.call_tool(mock_utcp_client, "calculator", {}, tool_template) - # Streaming - stream = text_protocol.call_tool_streaming(mock_utcp_client, "calculator", {}, tool_template) - chunks = [c async for c in stream] - assert chunks == [content] - finally: - Path(temp_file).unlink() + content = json.dumps(sample_utcp_manual) + tool_template = TextCallTemplate(name="tool_call", content=content) + + # Non-streaming + result = await text_protocol.call_tool(mock_utcp_client, "calculator", {}, tool_template) + # Streaming + stream = text_protocol.call_tool_streaming(mock_utcp_client, "calculator", {}, tool_template) + chunks = [c async for c in stream] + assert chunks == [result] @pytest.mark.asyncio @@ -366,12 +227,24 @@ async def test_text_call_template_with_auth_tools(): template = TextCallTemplate( name="test-template", - file_path="test.json", + content='{"test": true}', auth_tools=auth_tools ) assert template.auth_tools == auth_tools - assert template.auth is None # auth should still be None for file access + assert template.auth is None + + +@pytest.mark.asyncio +async def test_text_call_template_with_base_url(): + """Test that TextCallTemplate can be created with base_url.""" + template = TextCallTemplate( + name="test-template", + content='{"openapi": "3.0.0"}', + base_url="https://api.example.com/v1" + ) + + assert template.base_url == "https://api.example.com/v1" @pytest.mark.asyncio @@ -381,7 +254,7 @@ async def test_text_call_template_auth_tools_serialization(): template_dict = { "name": "test-template", "call_template_type": "text", - "file_path": "test.json", + "content": '{"test": true}', "auth_tools": { "auth_type": "api_key", "api_key": "test-key", diff --git a/plugins/communication_protocols/websocket/pyproject.toml b/plugins/communication_protocols/websocket/pyproject.toml index 5391418..09ce85c 100644 --- a/plugins/communication_protocols/websocket/pyproject.toml +++ b/plugins/communication_protocols/websocket/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-websocket" -version = "1.0.0" +version = "1.1.0" authors = [ { name = "UTCP Contributors" }, ] @@ -14,7 +14,7 @@ requires-python = ">=3.10" dependencies = [ "pydantic>=2.0", "aiohttp>=3.8", - "utcp>=1.0" + "utcp>=1.1" ] classifiers = [ "Development Status :: 4 - Beta", From 9f15e9978f8b60ec313587b400ae488f29a758ab Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Mon, 1 Dec 2025 02:21:30 +0100 Subject: [PATCH 07/10] Fix some issues --- core/pyproject.toml | 2 +- .../default_variable_substitutor.py | 9 +++++---- .../communication_protocols/http/pyproject.toml | 2 +- .../http/src/utcp_http/openapi_converter.py | 15 +++++++++++---- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/pyproject.toml b/core/pyproject.toml index e9214da..c5c8f07 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp" -version = "1.1.1" +version = "1.1.2" authors = [ { name = "UTCP Contributors" }, ] diff --git a/core/src/utcp/implementations/default_variable_substitutor.py b/core/src/utcp/implementations/default_variable_substitutor.py index ccc6dfb..b197d0b 100644 --- a/core/src/utcp/implementations/default_variable_substitutor.py +++ b/core/src/utcp/implementations/default_variable_substitutor.py @@ -100,8 +100,8 @@ def substitute(self, obj: dict | list | str, config: UtcpClientConfig, variable_ raise ValueError(f"Variable namespace '{variable_namespace}' contains invalid characters. Only alphanumeric characters and underscores are allowed.") if isinstance(obj, str): - # Skip substitution for JSON $ref strings - if '$ref' in obj: + # Skip substitution for JSON Schema $ref (but not variables like $refresh_token) + if re.search(r'\$ref(?![a-zA-Z0-9_])', obj): return obj # Use a regular expression to find all variables in the string, supporting ${VAR} and $VAR formats @@ -168,9 +168,10 @@ def find_required_variables(self, obj: dict | list | str, variable_namespace: Op result.extend(vars) return result elif isinstance(obj, str): - # Skip substitution for JSON $ref strings - if '$ref' in obj: + # Skip JSON Schema $ref (but not variables like $refresh_token) + if re.search(r'\$ref(?![a-zA-Z0-9_])', obj): return [] + # Find all variables in the string, supporting ${VAR} and $VAR formats variables = [] pattern = r'\${([a-zA-Z0-9_]+)}|\$([a-zA-Z0-9_]+)' diff --git a/plugins/communication_protocols/http/pyproject.toml b/plugins/communication_protocols/http/pyproject.toml index ae759ed..0ce7a83 100644 --- a/plugins/communication_protocols/http/pyproject.toml +++ b/plugins/communication_protocols/http/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-http" -version = "1.1.0" +version = "1.1.1" authors = [ { name = "UTCP Contributors" }, ] diff --git a/plugins/communication_protocols/http/src/utcp_http/openapi_converter.py b/plugins/communication_protocols/http/src/utcp_http/openapi_converter.py index c16412a..824df2a 100644 --- a/plugins/communication_protocols/http/src/utcp_http/openapi_converter.py +++ b/plugins/communication_protocols/http/src/utcp_http/openapi_converter.py @@ -83,11 +83,12 @@ class OpenApiConverter: Attributes: spec: The parsed OpenAPI specification dictionary. spec_url: Optional URL where the specification was retrieved from. + base_url: Optional base URL override for all API endpoints. placeholder_counter: Counter for generating unique placeholder variables. call_template_name: Normalized name for the call_template derived from the spec. """ - def __init__(self, openapi_spec: Dict[str, Any], spec_url: Optional[str] = None, call_template_name: Optional[str] = None, auth_tools: Optional[Auth] = None): + def __init__(self, openapi_spec: Dict[str, Any], spec_url: Optional[str] = None, call_template_name: Optional[str] = None, auth_tools: Optional[Auth] = None, base_url: Optional[str] = None): """Initializes the OpenAPI converter. Args: @@ -98,10 +99,13 @@ def __init__(self, openapi_spec: Dict[str, Any], spec_url: Optional[str] = None, the specification title is not provided. auth_tools: Optional auth configuration for generated tools. Applied only to endpoints that require authentication per OpenAPI spec. + base_url: Optional base URL override for all API endpoints. + When provided, this takes precedence over servers in the spec. """ self.spec = openapi_spec self.spec_url = spec_url self.auth_tools = auth_tools + self._base_url_override = base_url # Single counter for all placeholder variables self.placeholder_counter = 0 if call_template_name is None: @@ -141,9 +145,12 @@ def convert(self) -> UtcpManual: """ self.placeholder_counter = 0 tools = [] - servers = self.spec.get("servers") - if servers: - base_url = servers[0].get("url", "/") + + # Determine base URL: override > servers > spec_url > fallback + if self._base_url_override: + base_url = self._base_url_override + elif self.spec.get("servers"): + base_url = self.spec["servers"][0].get("url", "/") elif self.spec_url: parsed_url = urlparse(self.spec_url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" From 4e3205d13b630eade54b730e95d34732c5131db5 Mon Sep 17 00:00:00 2001 From: Mario Korte Date: Wed, 3 Dec 2025 17:39:14 +0100 Subject: [PATCH 08/10] Update `Tool` attributes and result processing to align with MCP naming conventions (#80) Co-authored-by: Mario Korte --- .../mcp/src/utcp_mcp/mcp_communication_protocol.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py b/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py index 3d419d4..c0c8e01 100644 --- a/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py +++ b/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py @@ -194,8 +194,8 @@ async def register_manual(self, caller: 'UtcpClient', manual_call_template: Call utcp_tool = Tool( name=mcp_tool.name, description=mcp_tool.description, - input_schema=mcp_tool.inputSchema, - output_schema=mcp_tool.outputSchema, + inputs=mcp_tool.inputSchema, + outputs=mcp_tool.outputSchema, tool_call_template=manual_call_template ) all_tools.append(utcp_tool) @@ -212,12 +212,12 @@ async def register_manual(self, caller: 'UtcpClient', manual_call_template: Call resource_tool = Tool( name=f"{server_name}.resource_{mcp_resource.name}", description=f"Read resource: {mcp_resource.description or mcp_resource.name}. URI: {mcp_resource.uri}", - input_schema={ + inputs={ "type": "object", "properties": {}, "required": [] }, - output_schema={ + outputs={ "type": "object", "properties": { "contents": { @@ -385,9 +385,9 @@ def _process_tool_result(self, result, tool_name: str) -> Any: self._log_info(f"Processing tool result for '{tool_name}', type: {type(result)}") # Check for structured output first - if hasattr(result, 'structured_output'): - self._log_info(f"Found structured_output: {result.structured_output}") - return result.structured_output + if hasattr(result, 'structuredContent'): + self._log_info(f"Found structuredContent: {result.structuredContent}") + return result.structuredContent # Process content if available if hasattr(result, 'content'): From 5dbad7c8facad38f897078dfc0176aa4f1921ef9 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Wed, 3 Dec 2025 18:13:21 +0100 Subject: [PATCH 09/10] Fix bug in mcp plugin --- plugins/communication_protocols/mcp/pyproject.toml | 2 +- .../mcp/src/utcp_mcp/mcp_communication_protocol.py | 8 ++++++-- .../mcp/tests/test_mcp_http_transport.py | 12 +++++------- .../mcp/tests/test_mcp_transport.py | 10 +++++----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/plugins/communication_protocols/mcp/pyproject.toml b/plugins/communication_protocols/mcp/pyproject.toml index 2cfd9a2..30a9ceb 100644 --- a/plugins/communication_protocols/mcp/pyproject.toml +++ b/plugins/communication_protocols/mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-mcp" -version = "1.1.0" +version = "1.1.1" authors = [ { name = "UTCP Contributors" }, ] diff --git a/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py b/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py index c0c8e01..3e94a29 100644 --- a/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py +++ b/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py @@ -384,12 +384,12 @@ async def call_tool_streaming(self, caller: 'UtcpClient', tool_name: str, tool_a def _process_tool_result(self, result, tool_name: str) -> Any: self._log_info(f"Processing tool result for '{tool_name}', type: {type(result)}") - # Check for structured output first + # Check for structured output first - this is the expected behavior if hasattr(result, 'structuredContent'): self._log_info(f"Found structuredContent: {result.structuredContent}") return result.structuredContent - # Process content if available + # Process content if available (fallback) if hasattr(result, 'content'): content = result.content self._log_info(f"Content type: {type(content)}") @@ -427,6 +427,10 @@ def _process_tool_result(self, result, tool_name: str) -> Any: return content + # Handle dictionary with 'result' key + if isinstance(result, dict) and 'result' in result: + return result['result'] + # Fallback to result attribute if hasattr(result, 'result'): return result.result diff --git a/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py b/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py index adaadb4..1fc87b8 100644 --- a/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py +++ b/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py @@ -136,7 +136,7 @@ async def test_http_unstructured_output( # Call the greet tool and verify the result result = await transport.call_tool(None, f"{HTTP_SERVER_NAME}.greet", {"name": "Alice"}, http_mcp_provider) - assert result == "Hello, Alice!" + assert result == {"result": "Hello, Alice!"} @pytest.mark.asyncio @@ -152,11 +152,9 @@ async def test_http_list_output( # Call the list_items tool and verify the result result = await transport.call_tool(None, f"{HTTP_SERVER_NAME}.list_items", {"count": 3}, http_mcp_provider) - assert isinstance(result, list) - assert len(result) == 3 - assert result[0] == "item_0" - assert result[1] == "item_1" - assert result[2] == "item_2" + assert isinstance(result, dict) + assert "result" in result + assert result == {"result": ["item_0", "item_1", "item_2"]} @pytest.mark.asyncio @@ -172,7 +170,7 @@ async def test_http_numeric_output( # Call the add_numbers tool and verify the result result = await transport.call_tool(None, f"{HTTP_SERVER_NAME}.add_numbers", {"a": 5, "b": 7}, http_mcp_provider) - assert result == 12 + assert result == {"result": 12} @pytest.mark.asyncio diff --git a/plugins/communication_protocols/mcp/tests/test_mcp_transport.py b/plugins/communication_protocols/mcp/tests/test_mcp_transport.py index d127791..26b8cc8 100644 --- a/plugins/communication_protocols/mcp/tests/test_mcp_transport.py +++ b/plugins/communication_protocols/mcp/tests/test_mcp_transport.py @@ -98,7 +98,7 @@ async def test_unstructured_string_output(transport: McpCommunicationProtocol, m await transport.register_manual(None, mcp_manual) result = await transport.call_tool(None, f"{SERVER_NAME}.greet", {"name": "Alice"}, mcp_manual) - assert result == "Hello, Alice!" + assert result == {"result": "Hello, Alice!"} @pytest.mark.asyncio @@ -108,9 +108,9 @@ async def test_list_output(transport: McpCommunicationProtocol, mcp_manual: McpC result = await transport.call_tool(None, f"{SERVER_NAME}.list_items", {"count": 3}, mcp_manual) - assert isinstance(result, list) - assert len(result) == 3 - assert result == ["item_0", "item_1", "item_2"] + assert isinstance(result, dict) + assert "result" in result + assert result == {"result": ["item_0", "item_1", "item_2"]} @pytest.mark.asyncio @@ -120,7 +120,7 @@ async def test_numeric_output(transport: McpCommunicationProtocol, mcp_manual: M result = await transport.call_tool(None, f"{SERVER_NAME}.add_numbers", {"a": 5, "b": 7}, mcp_manual) - assert result == 12 + assert result == {"result": 12} @pytest.mark.asyncio From 5f256432532ac83914d7b9f1ca87f68587e8d162 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Wed, 3 Dec 2025 18:56:01 +0100 Subject: [PATCH 10/10] unwrap mcp results --- plugins/communication_protocols/mcp/pyproject.toml | 2 +- .../mcp/src/utcp_mcp/mcp_communication_protocol.py | 3 +++ .../mcp/tests/test_mcp_http_transport.py | 10 +++++----- .../mcp/tests/test_mcp_transport.py | 10 +++++----- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/plugins/communication_protocols/mcp/pyproject.toml b/plugins/communication_protocols/mcp/pyproject.toml index 30a9ceb..87461b7 100644 --- a/plugins/communication_protocols/mcp/pyproject.toml +++ b/plugins/communication_protocols/mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp-mcp" -version = "1.1.1" +version = "1.1.2" authors = [ { name = "UTCP Contributors" }, ] diff --git a/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py b/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py index 3e94a29..7204b43 100644 --- a/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py +++ b/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_communication_protocol.py @@ -387,6 +387,9 @@ def _process_tool_result(self, result, tool_name: str) -> Any: # Check for structured output first - this is the expected behavior if hasattr(result, 'structuredContent'): self._log_info(f"Found structuredContent: {result.structuredContent}") + # If structuredContent has a 'result' key, unwrap it + if isinstance(result.structuredContent, dict) and 'result' in result.structuredContent: + return result.structuredContent['result'] return result.structuredContent # Process content if available (fallback) diff --git a/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py b/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py index 1fc87b8..ff82f68 100644 --- a/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py +++ b/plugins/communication_protocols/mcp/tests/test_mcp_http_transport.py @@ -136,7 +136,7 @@ async def test_http_unstructured_output( # Call the greet tool and verify the result result = await transport.call_tool(None, f"{HTTP_SERVER_NAME}.greet", {"name": "Alice"}, http_mcp_provider) - assert result == {"result": "Hello, Alice!"} + assert result == "Hello, Alice!" @pytest.mark.asyncio @@ -152,9 +152,9 @@ async def test_http_list_output( # Call the list_items tool and verify the result result = await transport.call_tool(None, f"{HTTP_SERVER_NAME}.list_items", {"count": 3}, http_mcp_provider) - assert isinstance(result, dict) - assert "result" in result - assert result == {"result": ["item_0", "item_1", "item_2"]} + assert isinstance(result, list) + assert len(result) == 3 + assert result == ["item_0", "item_1", "item_2"] @pytest.mark.asyncio @@ -170,7 +170,7 @@ async def test_http_numeric_output( # Call the add_numbers tool and verify the result result = await transport.call_tool(None, f"{HTTP_SERVER_NAME}.add_numbers", {"a": 5, "b": 7}, http_mcp_provider) - assert result == {"result": 12} + assert result == 12 @pytest.mark.asyncio diff --git a/plugins/communication_protocols/mcp/tests/test_mcp_transport.py b/plugins/communication_protocols/mcp/tests/test_mcp_transport.py index 26b8cc8..d127791 100644 --- a/plugins/communication_protocols/mcp/tests/test_mcp_transport.py +++ b/plugins/communication_protocols/mcp/tests/test_mcp_transport.py @@ -98,7 +98,7 @@ async def test_unstructured_string_output(transport: McpCommunicationProtocol, m await transport.register_manual(None, mcp_manual) result = await transport.call_tool(None, f"{SERVER_NAME}.greet", {"name": "Alice"}, mcp_manual) - assert result == {"result": "Hello, Alice!"} + assert result == "Hello, Alice!" @pytest.mark.asyncio @@ -108,9 +108,9 @@ async def test_list_output(transport: McpCommunicationProtocol, mcp_manual: McpC result = await transport.call_tool(None, f"{SERVER_NAME}.list_items", {"count": 3}, mcp_manual) - assert isinstance(result, dict) - assert "result" in result - assert result == {"result": ["item_0", "item_1", "item_2"]} + assert isinstance(result, list) + assert len(result) == 3 + assert result == ["item_0", "item_1", "item_2"] @pytest.mark.asyncio @@ -120,7 +120,7 @@ async def test_numeric_output(transport: McpCommunicationProtocol, mcp_manual: M result = await transport.call_tool(None, f"{SERVER_NAME}.add_numbers", {"a": 5, "b": 7}, mcp_manual) - assert result == {"result": 12} + assert result == 12 @pytest.mark.asyncio