mirrors/patchy631--ai-engineering-hub
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:37:47 +08:00
commit 7653f56fed
1422 changed files with 359026 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agentic_rag_deepseek/src/agentic_rag/__init__.py
View File
agentic_rag_deepseek/src/agentic_rag/agentic_rag.ipynb
+498
View File
@@ -0,0 +1,498 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# !pip install \"chonkie[semantic]\"\n",
"# !pip install markitdown\n",
"# !pip install qdrant-client\n",
"# !pip install fastembed"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"from crewai import Agent, Crew, Process, Task\n",
"from crewai.project import CrewBase, agent, crew, task\n",
"from crewai_tools import SerperDevTool\n",
"from crewai_tools import PDFSearchTool\n",
"# from tools.custom_tool import DocumentSearchTool\n",
"from tools.custom_tool import DocumentSearchTool\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/akshaypachaar/miniconda3/envs/env_crewai/lib/python3.10/site-packages/model2vec/hf_utils.py:152: ResourceWarning: unclosed file <_io.TextIOWrapper name='/Users/akshaypachaar/.cache/huggingface/hub/models--minishlab--potion-base-8M/snapshots/dcbec7aa2d52fc76754ac6291803feedd8c619ce/config.json' mode='r' encoding='UTF-8'>\n",
" config = json.load(open(config_path))\n",
"ResourceWarning: Enable tracemalloc to get the object allocation traceback\n"
]
}
],
"source": [
"pdf_tool = DocumentSearchTool(file_path='/Users/akshaypachaar/Eigen/ai-engineering/agentic_rag/knowledge/dspy.pdf')"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"web_search_tool = SerperDevTool()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"import yaml\n",
"with open('config/agents.yaml', 'r') as f:\n",
" agents_config = yaml.safe_load(f)\n",
"\n",
"with open('config/tasks.yaml', 'r') as f:\n",
" tasks_config = yaml.safe_load(f)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"retriever_agent = Agent(\n",
" config=agents_config['retriever_agent'],\n",
" verbose=True,\n",
" tools=[\n",
" pdf_tool,\n",
" web_search_tool\n",
" ]\n",
")\n",
"\n",
"response_synthesizer_agent = Agent(\n",
" config=agents_config['response_synthesizer_agent'],\n",
" verbose=True\n",
")\n",
"\n",
"retrieval_task = Task(\n",
" description=tasks_config['retrieval_task']['description'],\n",
" expected_output=tasks_config['retrieval_task']['expected_output'], \n",
" agent=retriever_agent\n",
")\n",
"\n",
"response_task = Task(\n",
" description=tasks_config['response_task']['description'],\n",
" expected_output=tasks_config['response_task']['expected_output'], \n",
" agent=response_synthesizer_agent\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2025-01-06 00:22:52,271 - 8394103616 - __init__.py-__init__:537 - WARNING: Overriding of current TracerProvider is not allowed\n"
]
}
],
"source": [
"crew = Crew(\n",
"\t\t\tagents=[retriever_agent, response_synthesizer_agent], \n",
"\t\t\ttasks=[retrieval_task, response_task],\n",
"\t\t\tprocess=Process.sequential,\n",
"\t\t\tverbose=True,\n",
"\t\t\t# process=Process.hierarchical, # In case you wanna use that instead https://docs.crewai.com/how-to/Hierarchical/\n",
"\t\t)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mRetrieve relevant information to answer the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Task:\u001b[00m \u001b[92mRetrieve the most relevant information from the available sources for the user query: Who is elon musk?\n",
"\u001b[00m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/akshaypachaar/miniconda3/envs/env_crewai/lib/python3.10/site-packages/crewai/tools/tool_usage.py:162: PydanticDeprecatedSince20: The `schema` method is deprecated; use `model_json_schema` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n",
" acceptable_args = tool.args_schema.schema()[\"properties\"].keys() # type: ignore # Item \"None\" of \"type[BaseModel] | None\" has no attribute \"schema\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mRetrieve relevant information to answer the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Thought:\u001b[00m \u001b[92mI will first search for relevant information in the available documents about Elon Musk.\u001b[00m\n",
"\u001b[95m## Using tool:\u001b[00m \u001b[92mDocumentSearchTool\u001b[00m\n",
"\u001b[95m## Tool Input:\u001b[00m \u001b[92m\n",
"\"{\\\"query\\\": \\\"Who is elon musk?\\\"}\"\u001b[00m\n",
"\u001b[95m## Tool Output:\u001b[00m \u001b[92m\n",
"Content: (b) if Google believes, in good faith, that the Distributor has violated or caused Google to violate any\n",
"Anti-Bribery Laws (as defined in Clause 8.5) or that such a violation is reasonably likely to occur,\n",
"Source: 4-pl\n",
"=========\n",
"FINAL ANSWER: This Agreement is governed by English law.\n",
"SOURCES: 28-pl\n",
"\n",
"QUESTION: What did the president say about Michael Jackson?\n",
"=========\n",
"Content: Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet.\n",
"Justices of the Supreme Court. My fellow Americans.\n",
"Last year COVID-19 kept us apart. This year we are finally together again.\n",
"Tonight, we meet as Democrats Republicans and Independents. But most importantly as Americans.\n",
"With a duty to one another to the American people to the Constitution.\n",
"And with an unwavering resolve that freedom will always triumph over tyranny.\n",
"Six days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to\n",
"his menacing ways. But he badly miscalculated.\n",
"He thought he could roll into Ukraine and the world would roll over. Instead he met a wall of strength he never imagined.\n",
"He met the Ukrainian people.\n",
"From President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world.\n",
"Groups of citizens blocking tanks with their bodies. Everyone from students to retirees teachers turned soldiers defending\n",
"their homeland.\n",
"Source: 0-pl\n",
"\n",
"___\n",
"result = total toys\n",
"return result\n",
"\n",
"Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to\n",
"Denny?\n",
"\n",
"\n",
"___\n",
"My fellow Americans|tonight , we have gathered in a sacred space|the citadel of our democracy.\n",
"In this Capitol, generation after generation, Americans have debated great questions amid great strife, and have done\n",
"great things.\n",
"We have fought for freedom, expanded liberty, defeated totalitarianism and terror.\n",
"And built the strongest, freest, and most prosperous nation the world has ever known.\n",
"Now is the hour.\n",
"Our moment of responsibility.\n",
"\n",
"___\n",
"To get there, I call on Congress to fund ARPA-H, the Advanced Research Projects Agency for Health.\n",
"It’s based on DARPA|the Defense Department project that led to the Internet, GPS, and so much more.\n",
"\n",
"___\n",
"well as the NSF under CAREER grant CNS-1651570. Any opinions, findings, and conclusions or\n",
"recommendations expressed in this material are those of the authors and do not necessarily reflect\n",
"the views of the National Science Foundation. Omar Khattab is supported by the Apple Scholars in\n",
"AI/ML fellowship.\n",
"\n",
"\\usepackage[pdftex]{graphicx} ...\n",
"\\includegraphics[width=0.8\\linewidth]{myfile.pdf}\n",
"\n",
"REFERENCES\n",
"\n",
"\n",
"___\n",
"I checked: {query}\n",
"\n",
"___\n",
" In International Conference on Machine\n",
"Learning, pp.\n",
"___\n",
" arXiv preprint arXiv:2305.03495, 2023.\n",
"\n",
"Peng Qi, Xiaowen Lin, Leo Mehr, Zijian Wang, and Christopher D. Manning. Answering complex\n",
"In Proceedings of the 2019 Con-\n",
"\n",
"___\n",
"(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it\n",
"will return some similar entities to search.\n",
"(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage.\n",
"(3) Finish[answer], which returns the answer and finishes the task.\n",
"Here are some examples.\n",
"Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into?\n",
"Action 1: Search[Colorado orogeny]\n",
"Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas.\n",
"Action 2: Lookup[eastern sector]\n",
"Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny.\n",
"Action 3: Search[High Plains]\n",
"Observation 3: High Plains refers to one of two distinct land regions:\n",
"Action 4: Search[High Plains (United States)]\n",
"Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation\n",
"from around 1,800 to 7,000 ft (550 to 2,130 m).[3]\n",
"Action 5: Finish[1,800 to 7,000 ft]\n",
"Question: Musician and satirist Allie Goertz wrote a song about the \"The Simpsons\" character Milhouse, who Matt Groening\n",
"named after who?\n",
"Action 1: Search[Milhouse]\n",
"Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons\n",
"voiced by Pamela Hayden and created by Matt Groening.\n",
"Action 2: Lookup[named after]\n",
"Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous.\n",
"Action 3: Finish[Richard Nixon]\n",
"Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture?\n",
"Action 1: Search[Adam Clayton Powell]\n",
"Observation 1: Could not find [Adam Clayton Powell]. Similar: [’Adam Clayton Powell III’, ’Seventh Avenue (Manhattan)’,\n",
"’Adam Clayton Powell Jr. State Office Building’, ’Isabel Washington Powell’, ’Adam Powell’, ’Adam Clayton Powell (film)’,\n",
"\n",
"___\n",
" He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give\n",
"\n",
"to Denny?\"\"\"\n",
"\n",
"jason lollipops initial = 20\n",
"jason lollipops after = 12\n",
"denny lollipops = jason lollipops initial - jason lollipops after\n",
"result = denny lollipops\n",
"return result\n",
"\n",
"Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\n",
"\n",
"# solution in Python:\n",
"\n",
"def solution():\n",
"\n",
"\"\"\"Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\"\"\"\n",
"leah chocolates = 32\n",
"sister chocolates = 42\n",
"total chocolates = leah chocolates + sister chocolates\n",
"chocolates eaten = 35\n",
"chocolates left = total chocolates - chocolates eaten\n",
"result = chocolates left\n",
"return result\n",
"\n",
"\u001b[00m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/akshaypachaar/miniconda3/envs/env_crewai/lib/python3.10/site-packages/crewai/tools/tool_usage.py:162: PydanticDeprecatedSince20: The `schema` method is deprecated; use `model_json_schema` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n",
" acceptable_args = tool.args_schema.schema()[\"properties\"].keys() # type: ignore # Item \"None\" of \"type[BaseModel] | None\" has no attribute \"schema\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mRetrieve relevant information to answer the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Thought:\u001b[00m \u001b[92mThought: The search in the document did not yield relevant information regarding Elon Musk. I will now conduct a web search to find the information.\u001b[00m\n",
"\u001b[95m## Using tool:\u001b[00m \u001b[92mSearch the internet\u001b[00m\n",
"\u001b[95m## Tool Input:\u001b[00m \u001b[92m\n",
"\"{\\\"search_query\\\": \\\"Who is Elon Musk?\\\"}\"\u001b[00m\n",
"\u001b[95m## Tool Output:\u001b[00m \u001b[92m\n",
"\n",
"Search results: Title: Who is Elon Musk and what is his net worth? - BBC\n",
"Link: https://www.bbc.com/news/business-61234231\n",
"Snippet: The boss of X (formerly Twitter), Tesla and SpaceX is a billionaire and a business celebrity.\n",
"---\n",
"Title: Mr Elon Musk FRS - Fellow Detail Page | Royal Society\n",
"Link: https://royalsociety.org/people/elon-musk-13829/\n",
"Snippet: In 2017, SpaceX made further history by re-flying both a Falcon 9 rocket and Dragon spacecraft for the first time. By pioneering the development of reusable ...\n",
"---\n",
"Title: Elon Musk - Tesla Investor Relations\n",
"Link: https://ir.tesla.com/corporate/elon-musk\n",
"Snippet: Elon is Technoking of Tesla and has served as our Chief Executive Officer since October 2008 and as a member of the Board since April 2004.\n",
"---\n",
"Title: Elon Musk - Forbes\n",
"Link: https://www.forbes.com/profile/elon-musk/\n",
"Snippet: Real Time Net Worth · Elon Musk cofounded seven companies, including electric car maker Tesla, rocket producer SpaceX and artificial intelligence startup xAI.\n",
"---\n",
"Title: Elon Musk changes his name to Kekius Maximus on X - BBC\n",
"Link: https://www.bbc.com/news/articles/cy53vz1qpx1o\n",
"Snippet: The world's richest man sparks speculation after changing his name and using a picture of Pepe the Frog.\n",
"---\n",
"Title: Elon Musk | Tesla\n",
"Link: https://www.tesla.com/elon-musk\n",
"Snippet: As the co-founder and CEO of Tesla, Elon leads all product design, engineering and global manufacturing of the company's electric vehicles, battery products ...\n",
"---\n",
"Title: Elon Musk - WSJ Spotlight Coverage, Recent News\n",
"Link: https://www.wsj.com/topics/person/elon-musk\n",
"Snippet: Elon Musk is the CEO of electric-vehicle maker Tesla, whose board he joined in 2004. During his two decades there, the company has grown to be the global ...\n",
"---\n",
"Title: Elon Musk | Biography, SpaceX, Tesla, Twitter, X, Trump, & Facts\n",
"Link: https://www.britannica.com/money/Elon-Musk\n",
"Snippet: Elon Musk, the South African-born entrepreneur widely known as the founder of SpaceX and Tesla, is renowned for his advances in electric vehicles and space ...\n",
"---\n",
"Title: How did Elon Musk become the most powerful man in Washington?\n",
"Link: https://www.youtube.com/watch?v=8-iFXyLQQsg\n",
"Snippet: Elon Musk, the world's richest man, is also the most powerful man in Washington right now. The panel discusses how this came to be and if ...\n",
"---\n",
"\n",
"\n",
"\n",
"You ONLY have access to the following tools, and should NEVER make up tools that are not listed here:\n",
"\n",
"Tool Name: DocumentSearchTool\n",
"Tool Arguments: {'query': {'description': 'Query to search the document.', 'type': 'str'}}\n",
"Tool Description: Search the document for the given query.\n",
"Tool Name: Search the internet\n",
"Tool Arguments: {'search_query': {'description': 'Mandatory search query you want to use to search the internet', 'type': 'str'}}\n",
"Tool Description: A tool that can be used to search the internet with a search_query.\n",
"\n",
"Use the following format:\n",
"\n",
"Thought: you should always think about what to do\n",
"Action: the action to take, only one name of [DocumentSearchTool, Search the internet], just the name, exactly as it's written.\n",
"Action Input: the input to the action, just a simple python dictionary, enclosed in curly braces, using \" to wrap keys and values.\n",
"Observation: the result of the action\n",
"\n",
"Once all necessary information is gathered:\n",
"\n",
"Thought: I now know the final answer\n",
"Final Answer: the final answer to the original input question\n",
"\u001b[00m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/akshaypachaar/miniconda3/envs/env_crewai/lib/python3.10/site-packages/crewai/tools/tool_usage.py:162: PydanticDeprecatedSince20: The `schema` method is deprecated; use `model_json_schema` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n",
" acceptable_args = tool.args_schema.schema()[\"properties\"].keys() # type: ignore # Item \"None\" of \"type[BaseModel] | None\" has no attribute \"schema\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mRetrieve relevant information to answer the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Thought:\u001b[00m \u001b[92mThought: I have found multiple relevant sources from the web about Elon Musk. I will now extract the complete content from the best source.\u001b[00m\n",
"\u001b[95m## Using tool:\u001b[00m \u001b[92mSearch the internet\u001b[00m\n",
"\u001b[95m## Tool Input:\u001b[00m \u001b[92m\n",
"\"{\\\"search_query\\\": \\\"Elon Musk biography\\\"}\"\u001b[00m\n",
"\u001b[95m## Tool Output:\u001b[00m \u001b[92m\n",
"\n",
"Search results: Title: Elon Musk - Wikipedia\n",
"Link: https://en.wikipedia.org/wiki/Elon_Musk\n",
"Snippet: Elon Reeve Musk is a businessman known for his key roles in the space company SpaceX and the automotive company Tesla, Inc. He is also known for his ...\n",
"---\n",
"Title: Elon Musk: Biography, Entrepreneur, SpaceX and Tesla Founder\n",
"Link: https://www.biography.com/business-leaders/elon-musk\n",
"Snippet: Elon Reeve Musk was born on June 28, 1971, in Pretoria, South Africa. His mother, Maye Musk, is a Canadian model and the oldest woman to star in ...\n",
"---\n",
"Title: Elon Musk: Isaacson, Walter: 9781982181284 - Amazon.com\n",
"Link: https://www.amazon.com/Elon-Musk-Walter-Isaacson/dp/1982181281\n",
"Snippet: The #1 New York Times bestseller from the author of Steve Jobs--this is the astonishingly intimate story of the most fascinating and controversial innovator of our era--a rule-breaking visionary who helped to lead the world into the era of electric vehicles, private space exploration, and artificial intelligence.\n",
"---\n",
"Title: Elon Musk | Book by Walter Isaacson | Official Publisher Page\n",
"Link: https://www.simonandschuster.com/books/Elon-Musk/Walter-Isaacson/9781982181284\n",
"Snippet: The #1 New York Times bestseller from the author of Steve Jobs--this is the astonishingly intimate story of the most fascinating and controversial innovator of our era--a rule-breaking visionary who helped to lead the world into the era of electric vehicles, private space exploration, and artificial intelligence.\n",
"---\n",
"Title: Elon Musk | Biography, SpaceX, Tesla, Twitter, X, Trump, & Facts\n",
"Link: https://www.britannica.com/money/Elon-Musk\n",
"Snippet: South African-born American entrepreneur who cofounded the electronic-payment firm PayPal and formed SpaceX, maker of launch vehicles and spacecraft.\n",
"---\n",
"Title: Book Review: 'Elon Musk,' by Walter Isaacson - The New York Times\n",
"Link: https://www.nytimes.com/2023/09/09/books/review/elon-musk-walter-isaacson.html\n",
"Snippet: Walter Isaacson's biography of the billionaire entrepreneur depicts a mercurial “man-child” with grandiose ambitions and an ego to match.\n",
"---\n",
"Title: Who is Elon Musk and what is his net worth? - BBC\n",
"Link: https://www.bbc.com/news/business-61234231\n",
"Snippet: Born in Pretoria, South Africa, Mr Musk showed his talents for entrepreneurship early, going door-to-door with his brother selling homemade ...\n",
"---\n",
"Title: Elon Musk: Tesla, SpaceX, and the Quest for a Fantastic Future\n",
"Link: https://www.amazon.com/Elon-Musk-SpaceX-Fantastic-Future/dp/0062301233\n",
"Snippet: In the spirit of Steve Jobs and Moneyball, Elon Musk is both an illuminating and authorized look at the extraordinary life of one of Silicon Valley's most exciting, unpredictable, and ambitious entrepreneurs--a real-life Tony Stark--and a fascinating exploration of the renewal of American invention and its new ``makers.''\n",
"---\n",
"Title: Elon Musk's Biography Summary - Dante\n",
"Link: https://dantekim.com/notes/elon-musk-tesla-spacex-and-the-quest-for-a-fantastic-future/\n",
"Snippet: In his biography you get to dig much deeper and see his flaws, extreme perseverance and genius in such close detail.\n",
"---\n",
"Title: Elon Musk By Walter Isaacson: A Biography Book On Hardcore Hope\n",
"Link: https://www.youtube.com/watch?v=srlhWHvRVuQ\n",
"Snippet: Elon Musk: the name synonymous with Tesla, SpaceX, and audacious dreams of colonizing Mars. But what's the man behind the ambition?\n",
"---\n",
"\u001b[00m\n",
"\n",
"\n",
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mRetrieve relevant information to answer the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Final Answer:\u001b[00m \u001b[92m\n",
"**Elon Reeve Musk** (born June 28, 1971) is a South African-born American entrepreneur and business magnate. He is the founder, CEO, and chief engineer of SpaceX; early investor, CEO, and product architect of Tesla, Inc.; founder of The Boring Company; co-founder of Neuralink; and co-founder and initial co-chairman of OpenAI. Musk is known for his ambitious vision for the future, including the colonization of Mars and the advent of sustainable energy. \n",
"\n",
"Born in Pretoria, South Africa, Musk showed an early interest in computing and technology. He taught himself computer programming at a young age and created a video game, which he sold at the age of 12. After moving to the U.S. to study at the University of Pennsylvania, Musk dropped out to pursue his entrepreneurial ambitions, co-founding Zip2, X.com, which later became PayPal, and then aiming to revolutionize transportation with his work on electric vehicles through Tesla.\n",
"\n",
"Musk established SpaceX in 2002 to reduce space transportation costs and enable the colonization of Mars. He made significant advancements with the Falcon rocket series and the Dragon spacecraft. With Tesla, Musk aims to accelerate the world's transition to sustainable energy, to help combat climate change.\n",
"\n",
"Musk has been recognized for his contributions and endeavors in the technology industry and has been listed among the most powerful and influential people in the world.\n",
"\n",
"For a more detailed exploration of his life, contributions, and impact, you can read further on his [Wikipedia page](https://en.wikipedia.org/wiki/Elon_Musk).\u001b[00m\n",
"\n",
"\n",
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mResponse synthesizer agent for the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Task:\u001b[00m \u001b[92mSynthesize the final response for the user query: Who is elon musk?\n",
"\u001b[00m\n",
"\n",
"\n",
"\u001b[1m\u001b[95m# Agent:\u001b[00m \u001b[1m\u001b[92mResponse synthesizer agent for the user query: Who is elon musk?\u001b[00m\n",
"\u001b[95m## Final Answer:\u001b[00m \u001b[92m\n",
"**Elon Reeve Musk** (born June 28, 1971) is a South African-born American entrepreneur and business magnate, renowned for his groundbreaking contributions in various technology sectors. He is the founder, CEO, and chief engineer of SpaceX; an early investor, CEO, and product architect of Tesla, Inc.; the founder of The Boring Company; and a co-founder of Neuralink and OpenAI. Musk is celebrated for his ambitious vision for the future, which includes plans for the colonization of Mars and the promotion of sustainable energy.\n",
"\n",
"Born in Pretoria, South Africa, Musk displayed a keen interest in computing from a young age, teaching himself programming and creating a video game that he sold by the age of 12. He moved to the United States to study at the University of Pennsylvania but left before completing his degree to pursue entrepreneurial ventures, co-founding Zip2 and later X.com, which evolved into PayPal. His ambitions continued with Tesla, where he aims to revolutionize transportation through electric vehicles and combat climate change.\n",
"\n",
"Musk started SpaceX in 2002 with the goal of reducing space transportation costs and enabling the colonization of Mars, achieving significant milestones with the Falcon rocket series and the Dragon spacecraft. Recognized globally, Musk has been cited as one of the most powerful and influential people in the world due to his innovative works and contributions to technology. For further details about his life and impact, refer to his [Wikipedia page](https://en.wikipedia.org/wiki/Elon_Musk).\u001b[00m\n",
"\n",
"\n",
"**Elon Reeve Musk** (born June 28, 1971) is a South African-born American entrepreneur and business magnate, renowned for his groundbreaking contributions in various technology sectors. He is the founder, CEO, and chief engineer of SpaceX; an early investor, CEO, and product architect of Tesla, Inc.; the founder of The Boring Company; and a co-founder of Neuralink and OpenAI. Musk is celebrated for his ambitious vision for the future, which includes plans for the colonization of Mars and the promotion of sustainable energy.\n",
"\n",
"Born in Pretoria, South Africa, Musk displayed a keen interest in computing from a young age, teaching himself programming and creating a video game that he sold by the age of 12. He moved to the United States to study at the University of Pennsylvania but left before completing his degree to pursue entrepreneurial ventures, co-founding Zip2 and later X.com, which evolved into PayPal. His ambitions continued with Tesla, where he aims to revolutionize transportation through electric vehicles and combat climate change.\n",
"\n",
"Musk started SpaceX in 2002 with the goal of reducing space transportation costs and enabling the colonization of Mars, achieving significant milestones with the Falcon rocket series and the Dragon spacecraft. Recognized globally, Musk has been cited as one of the most powerful and influential people in the world due to his innovative works and contributions to technology. For further details about his life and impact, refer to his [Wikipedia page](https://en.wikipedia.org/wiki/Elon_Musk).\n"
]
}
],
"source": [
"result = crew.kickoff(inputs={\"query\": \"Who is elon musk?\"})\n",
"print(result)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "env_crewai",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.15"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
agentic_rag_deepseek/src/agentic_rag/config/agents.yaml
+24
View File
@@ -0,0 +1,24 @@
# routing_agent:
# role: >
# Understand the user query: {query}, and route it to the most appropriate knowledge base
# goal: >
# Route user queries to the most appropriate knowledge base
# backstory: >
# You are a user query routing agent. You are knowln for your ability to carefully understand user queries and route them to the most appropriate knowledge base.
retriever_agent:
role: >
Retrieve relevant information to answer the user query: {query}
goal: >
Retrieve the most relevant information from the available sources for the user query: {query}, always try to use the pdf search tool first. If you are not able to retrieve the information from the pdf search tool then try to use the web search tool.
backstory: >
You're a meticulous analyst with a keen eye for detail. You're known for
your ability understand the user query: {query} and retrieve knowlege from the most suitable knowledge base.
response_synthesizer_agent:
role: >
Response synthesizer agent for the user query: {query}
goal: >
Synthesize the retrieved information into a concise and coherent response based on the user query: {query}. If you are not ble to retrieve the information then respond with "I'm sorry, I couldn't find the information you're looking for."
backstory: >
You're a skilled communicator with a knack for turning complex information into clear and concise responses.
agentic_rag_deepseek/src/agentic_rag/config/tasks.yaml
+20
View File
@@ -0,0 +1,20 @@
# routing_task:
# description: >
# Understand the user query: {query}, and route it to the most appropriate knowledge base
# expected_output: >
# The most appropriate knowledge base to route the user query to, the answer should be either 'pdf_search' or 'pdf_search'
# agent: routing_agent
retrieval_task:
description: >
Retrieve the most relevant information from the available sources for the user query: {query}
expected_output: >
The most relevant information in form of text as retrieved from the sources.
agent: retriever_agent
response_task:
description: >
Synthesize the final response for the user query: {query}
expected_output: >
A concise and coherent response based on the retrieved infromation from the right source for the user query: {query}. If you are not ble to retrieve the information then respond with "I'm sorry, I couldn't find the information you're looking for."
agent: response_synthesizer_agent
agentic_rag_deepseek/src/agentic_rag/crew.py
+76
View File
@@ -0,0 +1,76 @@
from crewai import Agent, Crew, Process, Task
from crewai.project import CrewBase, agent, crew, task
from crewai_tools import SerperDevTool
from crewai_tools import PDFSearchTool
# from tools.custom_tool import DocumentSearchTool
from agentic_rag.tools.custom_tool import DocumentSearchTool
# Initialize the tool with a specific PDF path for exclusive search within that document
pdf_tool = DocumentSearchTool(pdf='/Users/akshaypachaar/Eigen/ai-engineering/agentic_rag/knowledge/dspy.pdf')
web_search_tool = SerperDevTool()
@CrewBase
class AgenticRag():
"""AgenticRag crew"""
agents_config = 'config/agents.yaml'
tasks_config = 'config/tasks.yaml'
# If you would like to add tools to your agents, you can learn more about it here:
# https://docs.crewai.com/concepts/agents#agent-tools
# @agent
# def routing_agent(self) -> Agent:
# return Agent(
# config=self.agents_config['routing_agent'],
# verbose=True
# )
@agent
def retriever_agent(self) -> Agent:
return Agent(
config=self.agents_config['retriever_agent'],
verbose=True,
tools=[
pdf_tool,
web_search_tool
]
)
@agent
def response_synthesizer_agent(self) -> Agent:
return Agent(
config=self.agents_config['response_synthesizer_agent'],
verbose=True
)
# @task
# def routing_task(self) -> Task:
# return Task(
# config=self.tasks_config['routing_task'],
# )
@task
def retrieval_task(self) -> Task:
return Task(
config=self.tasks_config['retrieval_task'],
)
@task
def response_task(self) -> Task:
return Task(
config=self.tasks_config['response_task'],
)
@crew
def crew(self) -> Crew:
"""Creates the AgenticRag crew"""
# To learn how to add knowledge sources to your crew, check out the documentation:
# https://docs.crewai.com/concepts/knowledge#what-is-knowledge
return Crew(
agents=self.agents, # Automatically created by the @agent decorator
tasks=self.tasks, # Automatically created by the @task decorator
process=Process.sequential,
verbose=True,
# process=Process.hierarchical, # In case you wanna use that instead https://docs.crewai.com/how-to/Hierarchical/
)
agentic_rag_deepseek/src/agentic_rag/main.py
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python
import sys
import warnings
from agentic_rag.crew import AgenticRag
warnings.filterwarnings("ignore", category=SyntaxWarning, module="pysbd")
# This main file is intended to be a way for you to run your
# crew locally, so refrain from adding unnecessary logic into this file.
# Replace with inputs you want to test with, it will automatically
# interpolate any tasks and agents information
def run():
"""
Run the crew.
"""
inputs = {
'query': 'Who is elon musk?'
}
AgenticRag().crew().kickoff(inputs=inputs)
def train():
"""
Train the crew for a given number of iterations.
"""
inputs = {
"topic": "AI LLMs"
}
try:
AgenticRag().crew().train(n_iterations=int(sys.argv[1]), filename=sys.argv[2], inputs=inputs)
except Exception as e:
raise Exception(f"An error occurred while training the crew: {e}")
def replay():
"""
Replay the crew execution from a specific task.
"""
try:
AgenticRag().crew().replay(task_id=sys.argv[1])
except Exception as e:
raise Exception(f"An error occurred while replaying the crew: {e}")
def test():
"""
Test the crew execution and returns the results.
"""
inputs = {
"topic": "AI LLMs"
}
try:
AgenticRag().crew().test(n_iterations=int(sys.argv[1]), openai_model_name=sys.argv[2], inputs=inputs)
except Exception as e:
raise Exception(f"An error occurred while replaying the crew: {e}")
agentic_rag_deepseek/src/agentic_rag/tools/__init__.py
View File
agentic_rag_deepseek/src/agentic_rag/tools/custom_tool.py
+92
View File
@@ -0,0 +1,92 @@
import os
from crewai.tools import BaseTool
from typing import Type
from pydantic import BaseModel, Field, ConfigDict
from groundx import Document, GroundX
from dotenv import load_dotenv
load_dotenv()
class DocumentSearchToolInput(BaseModel):
"""Input schema for DocumentSearchTool."""
query: str = Field(..., description="Query to search the document.")
class DocumentSearchTool(BaseTool):
name: str = "DocumentSearchTool"
description: str = "Search the document for the given query."
args_schema: Type[BaseModel] = DocumentSearchToolInput
model_config = ConfigDict(extra="allow")
def __init__(self, file_path: str):
"""Initialize the searcher with a PDF file path and set up the Qdrant collection."""
super().__init__()
self.file_path = file_path
self.client = GroundX(
api_key=os.getenv("GROUNDX_API_KEY")
)
self.bucket_id = self._create_bucket()
self.process_id = self._upload_document()
def _upload_document(self):
ingest = self.client.ingest(
documents=[
Document(
bucket_id=self.bucket_id,
file_name=os.path.basename(self.file_path),
file_path=self.file_path,
file_type="pdf",
search_data=dict(
key = "value",
),
)
]
)
return ingest.ingest.process_id
def _create_bucket(self):
response = self.client.buckets.create(
name="agentic_rag"
)
return response.bucket.bucket_id
def _run(self, query: str) -> str:
# Check processing status
status_response = self.client.documents.get_processing_status_by_id(
process_id=self.process_id
)
# Check if processing is complete
if status_response.ingest.status != "complete":
return "Document is still being processed..."
# If processing is complete, proceed with search
search_response = self.client.search.content(
id=self.bucket_id,
query=query,
n=10,
verbosity=2
)
# Format the results with separators
formatted_results = ""
for result in search_response.search.results:
formatted_results += f"{result.text}\n____\n"
return formatted_results.rstrip('____\n')
# Test the implementation
def test_document_searcher():
# Test file path
pdf_path = "/Users/akshay/Eigen/ai-engineering-hub/agentic_rag_deepseek/knowledge/dspy.pdf"
# Create instance
searcher = DocumentSearchTool(file_path=pdf_path)
# Test search
result = searcher._run("What is the purpose of DSpy?")
print("Search Results:", result)
if __name__ == "__main__":
test_document_searcher()
agentic_rag_deepseek/src/agentic_rag/tools/test.ipynb
View File