[WIP] Plan-Driven Retrieval-Augmented Generation Experinment

[IANTHEREAL]

test question:

RAG VM

panner(gpt-4o-mini)

Step(id=1, subquestion='What is the newest version of TiDB?', tool_used='retrieve_knowledge_graph', arguments={'query': 'Newest version of TiDB'}, output_tags='newest_version'),
Step(id=2, subquestion='What are the performance improvements of TiDB from version 6.5 to the newest version?', tool_used='retrieve_knowledge_embedded_chunks', arguments={'query': 'TiDB performance improvements between version 6.5 and {newest_version}'}, output_tags='performance_improvements'),
Step(id=3, subquestion='Can you summarize the performance improvements from version 6.5 to the newest version of TiDB?', tool_used='llm_generate', arguments={'prompt': 'Summarize the performance improvements of TiDB from version 6.5 to the newest version based on {performance_improvements}.'}, output_tags='performance_summary')

panner(gpt-4o)

Step(id=1, subquestion='What is the newest version of TiDB?', tool_used='retrieve_knowledge_graph', arguments={'query': 'Newest version of TiDB'}, output_tags='newest_version_graph_data'),
Step(id=2, subquestion='Determine the newest version number of TiDB from the retrieved data.', tool_used='llm_generate', arguments={'prompt': 'Extract the newest version number of TiDB from {newest_version_graph_data}.'}, output_tags='newest_version_number'),
Step(id=3, subquestion='What are the performance improvements between TiDB version 6.5 and the newest version?', tool_used='retrieve_knowledge_embedded_chunks', arguments={'query': 'TiDB version 6.5 to {newest_version_number} performance improvements'}, output_tags='performance_improvements_data'),
Step(id=4, subquestion='Summarize the performance improvements of TiDB from version 6.5 to the newest version.', tool_used='llm_generate', arguments={'prompt': 'Summarize the performance improvements of TiDB from version 6.5 to {newest_version_number} based on {performance_improvements_data}.'}, output_tags='performance_summary')]

RAG tools callings

call function GraphKnowledge {'query': 'List the TiDB versions from 6.5 to the latest version with release notes and performance improvements.'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.1 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.2 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.3 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.4 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.5 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.6 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.7 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.8 release notes'}
call function VectorChunks {'query': 'Performance improvements and new features in TiDB 6.5.9 release notes'}
call function VectorChunks {'query': 'Transaction throughput and query latency metrics for TiDB from version 6.5 to the latest version'}
call function VectorChunks {'query': 'Scalability optimizations in TiDB from version 6.5 to the latest version'}
call function VectorChunks {'query': 'User reviews and performance benchmarks comparing TiDB version 6.5 to the latest version'}
data: {"role": "assistant", "content": "### Summary of Performance Improvements from TiDB 6.5 to Latest Versions\n\n#### Key Performance...