#!/usr/bin/env python3
"""
Test script to verify factual intent detection in matrix_cmd and ollamatools.
This script tests the factual intent detection functionality to determine
if it's working correctly in both implementations.
"""
import logging
import sys
import asyncio
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def test_ollamatools_factual_intent():
"""Test the factual intent detection in ollamatools."""
try:
# Import from ollamatools
from ollamatools.plugins.standard.search import detect_factual_intent as ollamatools_detect
logger.info("Testing ollamatools factual intent detection...")
# Test cases - messages that should be detected as factual intent
factual_messages = [
"What is the weather like in New York?",
"Tell me about the history of the Roman Empire",
"What are the latest news about climate change?",
"Who is the author of the book 'To Kill a Mockingbird'?",
"What is the recipe for chocolate chip cookies?",
"What are the stock prices for Apple today?",
"Tell me about the scientific research on quantum computing"
]
# Test cases - messages that should NOT be detected as factual intent
non_factual_messages = [
"Hello there!",
"How are you doing?",
"Thanks for the help",
"Please join this room",
"I like this",
"Can you help me?",
"What" # Too short to be complex
]
# Test factual messages
logger.info("Testing messages that SHOULD be detected as factual intent:")
for message in factual_messages:
result = ollamatools_detect(message)
logger.info(f"Message: '{message}' -> Factual intent: {result}")
if not result:
logger.warning(f"FAILED: Message '{message}' was not detected as factual intent")
# Test non-factual messages
logger.info("\nTesting messages that should NOT be detected as factual intent:")
for message in non_factual_messages:
result = ollamatools_detect(message)
logger.info(f"Message: '{message}' -> Factual intent: {result}")
if result:
logger.warning(f"FAILED: Message '{message}' was incorrectly detected as factual intent")
return True
except ImportError as e:
logger.error(f"Import error: {e}")
return False
except Exception as e:
logger.error(f"Error: {e}")
return False
async def test_matrix_cmd_factual_intent():
"""Test the factual intent detection in matrix_cmd."""
try:
# Import from matrix_cmd
# First check if matrix_cmd is using the ollamatools implementation
from matrix_cmd.matrix_client import MatrixClient
logger.info("Testing matrix_cmd factual intent detection...")
# Create a mock client to test with
client = MatrixClient(
homeserver="https://example.org",
user_id="@test:example.org",
password="password",
enable_search=True
)
# Test if the client has access to the factual intent detection
# This is tricky because matrix_cmd doesn't expose the function directly
# We need to check if the ollamatools integration is available
# Check if ollamatools integration is available
if hasattr(client, 'client') and client.client is None:
# Client hasn't connected yet, so we can't test through it directly
# Let's try to import the function that matrix_cmd would use
try:
from ollamatools.plugins.integrations.matrix.search import (
detect_factual_intent as matrix_detect
)
# Test cases - messages that should be detected as factual intent
factual_messages = [
"What is the weather like in New York?",
"Tell me about the history of the Roman Empire",
"What are the latest news about climate change?",
"Who is the author of the book 'To Kill a Mockingbird'?",
"What is the recipe for chocolate chip cookies?",
"What are the stock prices for Apple today?",
"Tell me about the scientific research on quantum computing"
]
# Test cases - messages that should NOT be detected as factual intent
non_factual_messages = [
"Hello there!",
"How are you doing?",
"Thanks for the help",
"Please join this room",
"I like this",
"Can you help me?",
"What" # Too short to be complex
]
# Test factual messages
logger.info("Testing messages that SHOULD be detected as factual intent:")
for message in factual_messages:
result = matrix_detect(message)
logger.info(f"Message: '{message}' -> Factual intent: {result}")
if not result:
logger.warning(f"FAILED: Message '{message}' was not detected as factual intent")
# Test non-factual messages
logger.info("\nTesting messages that should NOT be detected as factual intent:")
for message in non_factual_messages:
result = matrix_detect(message)
logger.info(f"Message: '{message}' -> Factual intent: {result}")
if result:
logger.warning(f"FAILED: Message '{message}' was incorrectly detected as factual intent")
return True
except ImportError as e:
logger.error(f"Import error: {e}")
logger.info("Matrix integration might not be using ollamatools factual intent detection")
return False
else:
logger.warning("Cannot test matrix_cmd factual intent detection without connecting to a server")
logger.info("Checking if matrix_cmd is importing the correct function...")
# Check if the matrix_cmd is importing the function from ollamatools
import inspect
import ollamatools.plugins.integrations.matrix.search as matrix_search
if hasattr(matrix_search, 'detect_factual_intent'):
source = inspect.getsource(matrix_search.detect_factual_intent)
logger.info(f"Matrix search is using: {source[:100]}...")
return True
else:
logger.error("Matrix search module does not have detect_factual_intent function")
return False
except ImportError as e:
logger.error(f"Import error: {e}")
return False
except Exception as e:
logger.error(f"Error: {e}")
return False
async def test_execute_search_for_message():
"""Test the execute_search_for_message function in matrix integration."""
try:
from ollamatools.plugins.integrations.matrix.search import execute_search_for_message
logger.info("Testing execute_search_for_message function...")
# Mock OpenSearch config
opensearch_config = {
"base_url": "http://localhost:8081",
"engines": ["google", "bing", "duckduckgo"],
"num_results": 5
}
# Test messages
test_messages = [
"What is the weather like in New York?",
"Tell me about the history of the Roman Empire",
"Hello there!", # Non-factual message
"What" # Too short to be complex
]
for message in test_messages:
logger.info(f"Testing message: '{message}'")
try:
# This will likely fail without a real OpenSearch server
# But we can see if it detects factual intent
result = await execute_search_for_message(message, opensearch_config)
logger.info(f"Result: {result[:100] if result else 'No result'}")
except Exception as e:
# Check if the error is related to factual intent detection
if "No factual intent detected" in str(e):
logger.info(f"No factual intent detected for message: '{message}'")
else:
logger.error(f"Error executing search for message '{message}': {e}")
return True
except ImportError as e:
logger.error(f"Import error: {e}")
return False
except Exception as e:
logger.error(f"Error: {e}")
return False
async def main():
"""Run all tests."""
ollamatools_result = test_ollamatools_factual_intent()
matrix_cmd_result = await test_matrix_cmd_factual_intent()
search_execution_result = await test_execute_search_for_message()
if ollamatools_result and matrix_cmd_result and search_execution_result:
logger.info("All tests completed successfully!")
return 0
else:
logger.error("Some tests failed!")
return 1
if __name__ == "__main__":
sys.exit(asyncio.run(main()))