1786 lines
106 KiB
Plaintext
1786 lines
106 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9f97dd1e",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Libraries"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "9e974df6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"from typing import TypedDict, List, Optional, Annotated\n",
|
|
"from IPython.display import Image, display\n",
|
|
"\n",
|
|
"from langchain_core.documents import Document\n",
|
|
"from langchain_core.messages import BaseMessage, SystemMessage\n",
|
|
"from langchain_core.tools import tool\n",
|
|
"from langgraph.checkpoint.memory import InMemorySaver\n",
|
|
"from langgraph.graph.message import add_messages\n",
|
|
"from langchain_ollama import ChatOllama, OllamaEmbeddings\n",
|
|
"from langchain_elasticsearch import ElasticsearchStore\n",
|
|
"from langgraph.graph import StateGraph, END\n",
|
|
"from langgraph.prebuilt import ToolNode\n",
|
|
"from langfuse import get_client\n",
|
|
"from langfuse.langchain import CallbackHandler"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "30edcecc",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ES_URL = os.getenv(\"ELASTICSEARCH_LOCAL_URL\")\n",
|
|
"INDEX_NAME = os.getenv(\"ELASTICSEARCH_INDEX\")\n",
|
|
"BASE_URL = os.getenv(\"LLM_BASE_LOCAL_URL\")\n",
|
|
"MODEL_NAME = os.getenv(\"OLLAMA_MODEL_NAME\")\n",
|
|
"EMB_MODEL_NAME = os.getenv(\"OLLAMA_EMB_MODEL_NAME\")\n",
|
|
"LANGFUSE_PUBLIC_KEY = os.getenv(\"LANGFUSE_PUBLIC_KEY\")\n",
|
|
"LANGFUSE_SECRET_KEY = os.getenv(\"LANGFUSE_SECRET_KEY\")\n",
|
|
"LANGFUSE_HOST = os.getenv(\"LANGFUSE_HOST\")\n",
|
|
"\n",
|
|
"# langfuse = get_client()\n",
|
|
"# langfuse_handler = CallbackHandler()\n",
|
|
"\n",
|
|
"embeddings = OllamaEmbeddings(base_url=BASE_URL, model=EMB_MODEL_NAME)\n",
|
|
"llm = ChatOllama(base_url=BASE_URL, model=MODEL_NAME)\n",
|
|
"\n",
|
|
"vector_store = ElasticsearchStore(\n",
|
|
" es_url=ES_URL,\n",
|
|
" index_name=INDEX_NAME,\n",
|
|
" embedding=embeddings,\n",
|
|
" query_field=\"text\",\n",
|
|
" vector_query_field=\"vector\",\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ad98841b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# if langfuse.auth_check():\n",
|
|
"# print(\"Langfuse client is authenticated and ready!\")\n",
|
|
"# else:\n",
|
|
"# print(\"Authentication failed. Please check your credentials and host.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "873ea2f6",
|
|
"metadata": {},
|
|
"source": [
|
|
"### State"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "5f8c88cf",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class AgentState(TypedDict):\n",
|
|
" messages: Annotated[list, add_messages]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "1d60c120",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Tools"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "f0a21230",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"retrieve_kwargs = {\"k\": 5}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "f9359747",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def format_context(docs: List[Document]) -> str:\n",
|
|
" chunks: List[str] = []\n",
|
|
" for i, doc in enumerate(docs, 1):\n",
|
|
" source = (doc.metadata or {}).get(\"source\", \"Untitled\")\n",
|
|
" source_id = (doc.metadata or {}).get(\"id\", f\"chunk-{i}\")\n",
|
|
" text = doc.page_content or \"\"\n",
|
|
" chunks.append(f\"[{i}] id={source_id} source={source}\\n{text}\")\n",
|
|
" return \"\\n\\n\".join(chunks)\n",
|
|
"\n",
|
|
"\n",
|
|
"@tool\n",
|
|
"def retrieve(query: str) -> str:\n",
|
|
" \"\"\"This tool retrieves relevant documents from the vector store based on the input query and formats them for the agent's response.\n",
|
|
" Args:\n",
|
|
" query (str): The input query for which to retrieve relevant documents.\n",
|
|
" \"\"\"\n",
|
|
" retriever = vector_store.as_retriever(\n",
|
|
" search_type=\"similarity\",\n",
|
|
" search_kwargs=retrieve_kwargs,\n",
|
|
" )\n",
|
|
" docs = retriever.invoke(query)\n",
|
|
" return format_context(docs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "e5247ab9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def should_continue(state: AgentState) -> str:\n",
|
|
" last = state[\"messages\"][-1]\n",
|
|
" \n",
|
|
" if getattr(last, \"tool_calls\", None):\n",
|
|
" return \"tools\"\n",
|
|
" return \"end\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "a644f6fa",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"tools = [retrieve]\n",
|
|
"tool_node = ToolNode(tools)\n",
|
|
"memory = InMemorySaver()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "395966e2",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Agent"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"id": "36d0f54e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def agent(state: AgentState) -> AgentState:\n",
|
|
" messages: List[BaseMessage] = state[\"messages\"]\n",
|
|
"\n",
|
|
" system = SystemMessage(\n",
|
|
" content=(\n",
|
|
" \"\"\" \n",
|
|
" You are a specialized agent whose job is to respond questions about a new language code called AVAP, which is a language designed to build APIs.\n",
|
|
" To responde to the user's question, you have access to a tool called \"retrieve\" that allows you to retrieve relevant documents about AVAP from a vector store.\n",
|
|
" You must respond in the language the user is assking you.\n",
|
|
" \"\"\"\n",
|
|
" )\n",
|
|
" )\n",
|
|
"\n",
|
|
" model = llm.bind_tools(tools)\n",
|
|
" resp = model.invoke([system, *messages])\n",
|
|
"\n",
|
|
" return {\"messages\": [*messages, resp]}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ef55bca3",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Graph"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"id": "fae46a58",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"graph = StateGraph(AgentState)\n",
|
|
"graph.add_node(\"agent\", agent)\n",
|
|
"graph.add_node(\"tools\", tool_node)\n",
|
|
"\n",
|
|
"graph.set_entry_point(\"agent\")\n",
|
|
"graph.add_conditional_edges(\"agent\", should_continue, {\"tools\": \"tools\", \"end\": END})\n",
|
|
"graph.add_edge(\"tools\", \"agent\")\n",
|
|
"\n",
|
|
"agent_graph = graph.compile()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"id": "2fec3fdb",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAANgAAAERCAIAAACW0v5yAAAQAElEQVR4nOydB3xTVfvHn3tvRvfeLW0pZbVlCsgLgmyVWZa+TBV52fxBBUSRoYIIojhAFAUZgqCCLEFUppRZkC2ztLSldK+Upk2T+39ubglpSRdtkpPkfD98ys29Jzdp88s5zzjnORKe54FCMTcSoFAIgAqRQgRUiBQioEKkEAEVIoUIqBApRECFWAOSbiqvxeZlpxYXKzWaEl6jLt+AYYHXGDiJPH4eOABDd9DwPMMz5W+LUTb9k3jI8KBhyj+fLX9SImdkdqyjszQw3L7Fsy5AKgyNI1bJv6cUZw9m5WWXaNQajmOkdqydHccwoC4pLy5GwvAl5f+eghAZ4B/THMexavVjd+AY0EC5D4VhUXSoMP1TQsvHXwvwtcreEt+tugRUxXzRA7W6hJc7sMGNHXuN9AHCoEKsjBux+Ud2ZKiKeK8AWatnPRq2dgBLprAQ/t6Wevf6A+zR64U79BvvD8RAhVghmxYn5mYUN2rl3IO8/qOWJPxbeHBrmqpIPXhqsGcAEeYZFaJhVs287eIuHfFOMFgvp/fnnv0rI7KDW+eBnmBuqBANsGrW7eadPDr2cwcb4OvZcX3GBNRrZAdmhQqxPKtmxnWK9onq6AQ2w7dv3wlr7tx9mBeYDxYoeqx+O65VVw+bUiHyv8X1b57PvX5GAeaDCvERm5cmOrlJ2vd2A9vjhVcCDvyUCuaDCrGUhKvKnNSi4W9Zs3dSCSFN7T395T8uSQQzQYVYyoEf74dEOIMN89IbQZmpRfkZGjAHVIgCiTeLlQ/UfV7zBdvGy1++Z20ymAMqRIFjv6Y5e8nAtMyePXvnzp1QQ27fvt23b18wDh36e2elFYE5oEIUyM0sjmxv6gkBV69ehZrzZM+qJsGN7RiGOftXLpgcGkcERZZm/aI7kz9pAMYhJiZmw4YNV65c8fLyatGixdSpU/GgTZs24lUnJ6fDhw9jP/fLL7+cOXPm3r17YWFh0dHRQ4YMERt079597NixBw8e/Oeff0aNGrVx40bx/Ouvvz5ixAioazZ+mGDvwA2ZHgSmhU4Dg6un8zgJA8bh2rVr06ZNmzBhwnvvvRcXF/fll18uWLBgxYoVqM6OHTvOnTt3wIAB2OyTTz5BCc6ZMwc7pPj4+CVLlvj7+2MDvCSVSn/99dd27dqhHJ966ils8Mcff+zZsweMg4evPC1JCSaHChEy7imldsYyUc6fP29nZzdmzBiWZf38/CIiIm7duvV4s8WLFxcUFAQEBOAxdpa7du06fvy4KERUnqur64wZM8AkuPvIkm49AJNDhQhFDzQSqbF6xJYtWyqVyunTpz/99NOdO3euV6+eblDWBw2kLVu2YDeZkJAgngkMDNRdRfmCqXBw4dQaM1hr1FkBtTD52Vh/+iZNmnzxxRfe3t44KA8cOHDSpEkXLlwo10aj0eDwjQbilClTDh06FBsbi6akfgOZzHQePcsIk27B5FAhgoODRGPMIG6HDh3QFty9ezdah7m5udg7lpSU6DdAOxJdGXQ+unbt6uwsBNXz8/PBTBQq1IwZdEiFCODqLSkqNJYSz549i9YeHmCniPG/N998E0WWkpKi3yYnJwd/+viUTr+N0wJmIvO+SsqZQRVUiNCwlYtaZSwh4kA8a9as7du3Z2dnX758GQ1BVCR6xHK5HJV38uRJHIiDg4MlEgnGZfLy8tBl/vjjj9u3b19OrDqwcUZGBkZ8dNZk3ZJ5v8jBlQOTQ4UIPvWk6JtePZEHRmDkyJFoGi5btqxnz57jxo1zdHRcvXo1yg4voSuNdiH2kegUL1y48NKlS926dcMBevLkyRhERNXqQon6PPPMM+gAoRO9f/9+MAL5WcVBjcywNIcGtAW+XxAvs2NHzLbRqTc6clJLNn50Z+ryhmByaI8o8PRzXnnZKrB59m24Z+9khnEZaBxRJOI/Toe3pR7cmt7tJW+DDdDbFVMgj4M5OoXC8NxmTNatXbsWjMM6LVDDt4RB8kWLFkEFZNwr6jMmAMwBHZpLOXsg9+Te9MmfhBu8iqG++/fvG7yE8WrMnRi8hLagzheuc/K1QA3fEjpJnp6G1+zt+DolN6345XkhYA6oEB+xZt4dd2/poKmmzveTAK+Br2bequh7aAKojfiI196vfz+hKOGqGVL+Zmf1nDuR7c25WIcKsQyj3grbu848U5TNyMYPEj18ZF2GmnM5KR2ay6NUaNYsuDN8VrC7jxRsgG+xL/yPS4e+Zi72QIVoAEWOZt37ceHNnZ9/xZpXseSmq7cuT/Dwlw+ZGgjmhgqxQla/E8cwzLODvRu1tsL19j9/lpyWVNi6i+d/+hJRWYUKsTIObE6/fi5XJudCmzn1+K83WD7/nsz/52hOTlqxi6d05NsEZZKoEKvmz42p8dcKipUalmMcXaQOzpy9I8dIGXXxo+KbnIQtrdvJYCxEe4Zj1Oqyf1sGOLb8SZYVKnmqy8wLw5NCzVhebeCj4fB1VbzuuboJbJxEuIn+GREJx5aUwIN81YM8tfKBGljG3Uf2wqgAVx+y/FQqxGqjhMN70tMSihR5Jag5XlNGUizHa9Rl6w0Lf9ryM/sYluc15ZppyxVrJY1hc1Z4zGirHZdvKcJxvPrhC2Fb3afHcsJN9M/o2ktknJ096+ojb9rWOawZobVGqRAJYvjw4QsWLGjUqBHYHjTXTBAlJSXiDDEbhAqRIKgQKURAhUghApVKJZXaRDrncagQCYL2iBQioEKkEAEVIoUIqI1IIQK1Wk17RIqZQRVynHlW0JEAFSIp2LKBCFSI5ECFSCEC9FSoECnmh/aIFCKgQqQQARUihQhsOZoNVIjkQHtEChFQIVKIgAqRQgRUiBQioM4KhQhoj0ghAoZh3N2JKENjFqgQSYFl2YyMDLBVqBBJAcflcluj2RRUiKSAQlSr1WCrUCGSAu0RKURAhUghAipEChFQIVKIgAqRQgRUiBQioEKkEAEVIoUIqBApRECFSCECKkQKEXAcZ8u5ZrpNLkGgFm22U6RCJAhbHp3pzlPmp2XLlizLMoywsZlGoxEPRo8ePX36dLAZaI9ofpo0aSIKEcHRGY/r1as3bNgwsCWoEM3P4MGDZTKZ/plOnTr5+lrznuWPQ4VofoYOHRoaGqp7iBLEM2BjUCESwfDhwx0cSjewbdu2bUhICNgYVIhE0LdvX7FTxO4QRQm2B/Waa0BBLpzZn1FYgDGWMtvEsxJGo+ah7B8SfQ4Nr9E/yYjfer7M3t4MK/jIDPCpaenXrl3z9PSMaBohPJ0TPhpeU/494GsB3vex83hzBhiNxvCnKbeXBIY7RrZ3BFKhQqwumz66m5epksk5FKFGVUYIDKfdbb7sH1LYrJ4ve1K3Ib2+EBleuIANNYLABM9Z247hhDPw2IeD5wUpPyZE/CRR03wFqRm5HatS8ZwEBkwI9A6SAXlQIVaLH5cmaUqY/pMDwZK58nfe+aMZQ18P8vQnTotUiFWzeUkSy7F9/hcAlo8iB3asjJu4NAwIgzorVVCYBbkZRdahQsTJDZzdZNtWpABhUCFWwZnDmRK5Ve1M5h1kl5NeBIRBp4FVQWG+WlNiZdYLX6LUAGFQIVaBmteo1cR9bLWBF34j4r5aVIg2B5neKRWizSGEvhkgDSpEm0NI95DXKVIh2hwEdodAhVglmKljrCvGRWYIgAqxKnjG2nJP1FmxRAQVWpcQS2dVEAYVos3B8yR+s6gQbQ6hR2RpQJtiboQeUUPc4EyFWAUsK/yzJoQekQa0LQ7e6rxmnicxoE2ngVWBsHCEYCG+9/7svft2guVDhWjZXL9+FawCOjTXPQqF4udffjh95kR8/G1PD68OHZ4d8+pEOzs7vJSdnbX4o3lXrl4Mrhc6YMDQpKS7fx87tP77X0C7Te6atV+dPHUsLe1+VFTLgQNebN/+GfGG0YN6vPrKhNzcnPUbVtvb27dt858pk2d4enp17d4Gr3687INVXy/fvfMwWDK0R6wSvqam/fZft2z+cd1LL476cNFn48dPO3zkTxSQeGnpsvfvJsZ/vPSrhR98eupUDP5jH7pCX3y59JdtmwdGv7R50+5nO3ef/96sI0cPiJekUunWrRuw5Y5fD6z/ftuly+fXrf8Gz/++NwZ/zpwxt0YqpM6KZaJd71kjXhw6EpUUElJffHj58oXTZ46PH/d/2KWdPHls6pSZEU2j8Pybb7w7bHhfL28fPC4qKtr/x57hw17p328wPuz9wgB81oaN3+J9xJsEBtYbOWKMcOTkjD3ijRv/wpNCprNChVgVNU/xYQd2JvbER0vm37p9Q6x36O7ugT9vx93En1FRLcRmTk5OrVu3ww4Sj1FYxcXFqDDdTVq2eGrf77ty83JdXVzxYaNGTXWXnJ1dCgoU8KQwRAakqBDrntXffrl37w4clFFYvr5+361ZKTq2+fl5+NPR0UnX0kUrMhDMynz8OXXaa+VulZ2VKQqRqbvRFLtDDXldIhViHYPBnt17tg0ZPLxvn4HiGVFkiFwu+Cuq4mJd4+ycLPHA08sbhMF6Dg7B+nfz8fGDun+Lgt0LhEGFWAWCiViTgQzH4sLCQi8vH/EhDrjHTxwVj+vVE2p83Ym/HRoqrG9H5/rcudO+vv54HBQYLJfL8aBVyzZiY/SvUdO6EmF1CJnOCvWaq4SpkY2IBmJwcCiad8n3ktA7QTe5WVRLHJQLCgoCA4LQg0EPGi+hCj/7fLG/f2kNExTcKy+PR+/k0qXzqF30l2fMmvTZ5x9V/lqoXW9vn9jYk/+cj61+2J1mViySJ5iPOHfOh3Zyu1deHTJydPRTrduNHTsFHw4c3CPl/r1ZM+ZhFGbU6IGvvzEO/Y+oyBZSiVR81n9fGj1zxrzNW9b1G9Dl8y+WBPgHvfnmu1W+1ojhY879c2buvDc1Gste80pr31TB3nX3468UjHq3AdQF2EcqlUr0YMSHb8+ZLuEkH7y/DExIzM7UO5cUEz+um9+orqA9oknB1DD2hZhNQUVu/GHN2bOn+vcfAqaFruKzSOp28dT8+Us+Xvb+t9+tSE9PDQmuP3/uR23btAcKFWKV8BqGrzvrC4OCC9//BMwKXWBPIQJacoRCBjTXbIkImVmOyNoITwqt9GCRCDXWySviVhvo0EyhVAgVIoUIqBCrgJWARGptNiIN31gemhIoUVmbjUi9ZgrFMFSIFCKgQqwCqYyRyq3KRpRKJTJ74naOobNvqsDX30Gjtioh5uUUy+2I+9ypEKugeVdntO3jLz0AayHznrJRK2cgDCrEqmnbw+f47vtgFfz6ZaKdPft0b3cgDDpDu1pkp6h+Wp7oEWgf3NhJbs+WqPX2RWbKrCXgtV9u8QTPAKstzyo2edSQh8erB4v7NvPl71emPfPw0eMvzosBwtKXE/9/dJUFJiO5KOmmwsNfHj3RH8iDCrG63LqWuXdNklziolaV2UJMDA7r/or4UP9YvMQYWvfCPFSP2ECEr+i2eurltVUbrpy9FgAAEABJREFUxXCgdo9x/QO+dA00/+jmCPpbMjkX0tip+3AvIBIqxOoyduzYRYsW+fr6gtEYOXLk3LlzGzduDE/ExYsXp02b5uTk1KlTp+jo6EaNGoHlQG3Eqvnzzz/x53fffWdUFSJ4f3t7e3hSmjVr5unpmZKSsmXLltdff3369OkHDhwAC4H2iJWh0Wj69ev36aefPnEvZWJmzpyJ4hMrjOGbd3Nz8/Pze/7550ePHg1kQ3vECklOTi4sLFy7dq3JVIivKBZtemLatGnDcaXBapRjXl7etWvX1q1bB8RDhWiYt956S6FQODo6Gns41mfy5MmpqalQC6Kiory8yrgj3t7eBw8eBOKhQiwP9klnzpzp1auX6YfjgIAAqVQKtSAyMhK/PLqHLi4u+/fvB0uACrEMGzduxOGsdevW3bt3B5Pz1Vdf+fj4QO0ICwvTaAkNDW3btq2l+Ct00sMj9u3bl5WV5eHhAWYiKSnJ399fZ+Q9Ge3bt8exODY2VnyIIaHAwMAmTZoA2VCvWeDff/9t2rRpYmJivXr1wHz07Nnzp59+cnev4/xb165dd+3a5exMXH5ZHzo0A1pRa9asAaF+oTlVCEKh7ECZTAZ1zc6dOwcMGABkY9M9IhpSGOPYu3dv7969warBLv/DDz9ECxhIxXZ7RLSi5syZgwfkqDAhIcFI/QIaHi+//PLs2bOBVGxXiGiNLV68GEjipZdeUuvP66lTevTogXL88ssvgUhsTogFBQW//fYbHixduhQIA41UicSIcQzsFPPz87dv3w7kYVs2IqbsMPG6bdu2cukHmwLzN6jIdu3aAUnYkBAxOuPg4ODp6QmkgjZiSEgIGB90ojF4jk46EINNDM1FRUVDhw6Vy+Ukq1ClUv33v/8Fk4ABnejoaCAJ6xcixmhiYmLQIqx99syo4Pts0MB0BdZ37NhBlBatfGheuHAhxiyM6gFYLqdPn16/fv3KlSuBAKy5R1y9enVUVJSlqBB7xLt374IJQX+le/fuGOgGArBOIR46dAi0YTnSLKFKyMnJGTt2LJiWQYMGYQ4a+0UwN1YoRPQHb9++jQeurq5gOTAMExoaCiZn6tSpmAD866+/wKxYlY2Ynp7u7e196tSpp59+Gig1YdSoUe+88w6mXsBMWI8Qt2zZkpubO378eLBMMLmXkpISFBQEZqJbt27oSru4uIA5MIUQMatmgi0L0QdE65vwWXeVkJycjDkPlAKYCcz+9e/fXzSvTY8pPEqMJxtPiBgHxr7Ezs6uRYsW+EKOjo7iYkqLA23E4OBgMB/4HV61atXIkSN/+OEHMDmm6BGzsrKMJES8bV5enpubm+6Mh4eHhQqREA4cOPDHH38sWbIETIsFf2bijCl9FVo0xcXF9+7dA3ODkcXIyEjTzxazSCFiR4gOMqsFrIW4uLhZs2YBAYwePVqhUJh4tphFfpBoF2Lo64UXXsAgMFgLmAEy+6IZHW+//TaO0RgIA1NhSUJEcxYDNHggl8vB6ggPDydqxjjmoPH9JCUlgUmwJCGKNUDASkGX//59surSYixp4MCBYBLMI8SrV6/OmTNnyJAhr7322urVqx88KK1QvWvXrmHDhiUmJmJc+vnnn584cSJ6cKCdWY0/t27digmAMWPGbNiwoZbFigjkypUr8+bNA8JALZpmKaoZhIiRW8wmKZXK5cuX45/+zp07M2fOFIUllUqx28Nk8fTp0/ft29epUydsg4ljdEr2aJk0adLnn3/u5+e3adMmsC5kMhlRU6ZF8C1hl4F/djAyZhAixu7RMEcJom0eEhKCmkOpHT9+XLyKjsiIESMw6YkB3i5duqBdiAMWGoU7d+7spAXjrr169WrZsiVYF1FRUfPnzwfywHxVz549Fy1aBMbEDELEcblx48a6qTG+vr7+/v6XL1/WNRDLcGHX6ODggAc4cKMcMcamn3ho2LAhWBdoftSyJp3xQEsRPy+j1lk0w6RRVNiNGzfQBNQ/mZ2drTvGvhCVx3GcLkyIWsTwtX5ZX8zpgXWBJsrmzZsXLlwIRDJlypS5c+deunSpWbNmYATMIETMwmHsvlwx3XKTPlCLKDudE4NdI+oS/UpdA9F9sSYiIiL69esXExPTsWNHIJKDBw++++67YBzMIMT69etjsBS/WLoOLyEhoZydjq6MftYEdenj44NBbN2Z06dPg9VB8jRKNBvc3d2NF8E1g404aNAgzNF9/fXXqDaMl65Zs2bChAnx8fH6bdCJLld8o3PnzseOHTt69Choq4Vcu3YNrJGCggIMWgF53L1716iJHzMIEd1eVCEaeVOnTh07duzFixfRcca8gn4bvFquQBvGF9GsXLVqFf7E1NO4ceMAwPqWIGLEHoMGK1asAMJAIRp1lpplTwN7HDoNzEhgQBfjG8OHDwfjQOhnVqQFbBhMOKHpAsRghUNzdXjcRrQ1OnTogKYzEIONDs2oQnxjT7A23pqGZoxeiWEsIIC2bdueOXMGjAahnxlGDWmdEIye3rx5E/1oMDeJiYnGXl5IbUSiQbOMhGIVxh6XgVgh4tBsfRO9ngCMIa9fv14/EW8WTFC40RTDn5ubW01tRMxHoxCfYGGU9cVuAgIC/Pz80GJmGAbMBA7NYWFhYExMIcQnWOVkliowxIJ/vWeeeQbzouZaI4FDc9euXcGYENp/YO5/9+7dQHnIpk2btm7dCmYCh2YbtRExB22t2eQnA000c23+rVKpMjMz0TwAY0KoEDt27Ni/f3+glGX+/PmmX4SP47IJSswTKkSMWpl+u2TymTx5sukXWBk7uSdCqBAxiL9t2zaglMXHx+e7774D02KCICIQK8SUlJQrV64AxRD79+9H7wFMhU0PzZjZHDJkCFAM8dxzzxl1175y2PTQ7O/vHxERAZQKOHLkiMnq/tj00HzhwoXNmzcDpQIwsq1UKtPT08HIFBQUYNLfBDt2ESpE/BNfvHgRKBUTGBg4fvx4Y2/NYppxGcyyiq86NG/e3NfXFyiVsnHjxhMnThh13DTNuAzECtFHC1AqxdHRsUePHmBMTLZhKqFDM8ZuNmzYAJRqMG3aNONV1ExMTDTN0EyoELOzs8+dOweUarB8+fI9e/aAcTDZ0Ezohj+YZcfvovWV/LI4unTpgip3cnICI0Noj4jxAqrCGrF169aYmBjdw969e0OtwXFJKpWaQIVArBBv3br17bffAqXaYK7lm2++SUtL69Onz1NPPcWy7J07d6B2mCa5J0KoEHNzc2NjY4FSE9C9Gzx4cGpqKsMwhYWFycnJUDtMMB9WB6Hhm/DwcLG6DaX6tGrViuM48Ri/ybXfEMBkLjMQ2yO6urri+AKU6tG5c2d9FYJ2TyRx0+raQIdm4bu4cuVKoFSPo0eP1q9fHx0L3RkcnWtfCNmUQzOhQlQoFCdPngRKtdm+ffukSZMCAgLEChkYlUtJSYHaYcqhmVAbEX//qVOnAqVSrp8pUJVoZyUyKD1oFT4gakbv4zHHL1+6nJOf68Q5nfgjydm5tCY0w6A6sRXor45mtE8sF0lmWOA1aGXmRQb3vvlPEfBF4kswupbaZ5X+fNjeICzL+ATJvQJlUBVkBbTHjh0r1m3XVQNDW0epVIrb/lB0bFx0Nz9HxbKgKhY+voeSAFEgouaEY+Go9EKp/hjtUv2H7bVH2LbM0n2OY9Tq8qrQb1lWh6Ctvc/oXkX/mRIpXmOkMqbFM+7tXqisXAJZPWLz5s0fTzF7e3sDRY/Vs+O8gxyixwVD1R0NEVyOyT13KNMvRB4cUWFlM7JsxNGjR5czSrBHbNu2LVAesvqduGYdvXqM8rMUFSJRHV1HzAnbv+l+7B+5FbUhS4hubm6Ym9Iv8uLj4zNs2DCgaNm3Pk0i5aI6u4AF0ugp1/NHMiu6SpzXjLLT7xRxsG7atClQtKTeVXr5W+pOR627e6hUfLHC8FXihIgp9kGDBokxCE9PzxEjRgDlIaqiEomdBZc702ggI9XwTk0k/lYvvviiuP9PREREixYtgPKQkmK+pFgFFotGzWsqqHpZK6+5+AEc35uedrf4QX5JkRKjLQy+EsMyvEb4qXXx+dIIk9at5yRCA17n+pfGGTCcwOJTQDyB0SoN3zX0o5J6aiknWTUrjuWEZ4lPEW+ubQkMB49+K11EAco0K/0l8bdkGamUdXBhgxo6dOhr9DVplJryhEL8fV3q3esFqiKelbBoPrMyVu4o5XlRVYK4RH8D1aIR45QPZQQaIYBaJo6lpbSV9iE2kJWNdelinbpjbUsDQVBxQ8lyJyUSDgcFdbE6K1WVlph97mC23J5r0talUzRVJCnUWIj7vk+Nu6zgpIyzl1NgpEV+kHwxf/dy+sVjOZdP5LR61q19bypHEyH0IxXUva2ZEL95+w4OtSEt/Z28zFO6tE5gZExIa2GJYHpc3tmDWVdPKca8Z6I5JjYOmkssGM7kVddZSb6uXPHGLWcvxyZdgi1ahfp4h7lEdg9lOO6rN2s7Y8pEMGC+Qtp1AANQUUa5WkLMTS/ZsTo5olv9gAgrHMXqt/X3a+L91QwL0KIgQmvbBrOUqoV468KDTUsTInuEshxYKx5BjqGtA1cSr0Wet2wdCj1iBT161ULcv/5eeDsTTUozIw7uUq8Qt6/figOK0dAG9AxfqkKIq+fEO/s6y5ystzPUwzfcjZNxm5cmArEw2hCYxcJDhTZuZUI8/EtmiUoT3NwLbIaGHYKy7helxBcDkWhtRAsenJ/QWbl8PNs7tMZ7P1k6Du52u7+p7fo3IyHYiJZsJGqzEIa7xAqFeGJ3FmZNvOu7ApGcv/TXjLlPKwqyoa4Ja+OP6crcDBJ3ixYSm2Bqogf12LCxzirIV7QioEIhXj6Va+dsJfHCmiKVS/78obYrj4zBE3jN770/e+++nUAG5VbM6FOhEJUFav9GHmCTuPo4Z9wn1EysKdevXwVLwHCK78aZAomEtXcx1mz0+LsX/zj0XWLSVSdH96aNn+nVdaydnSOejzn5859H1k4cs2rDlrdT0+L8fcM7dxjWtnVf8Vl7fv8y9sJeucyhVfPnfLyMuN7Wt4FrZpKJSqUbla7d2+DPj5d9sOrr5bt3HgZhk8Mj6zesTrh7x9XVLTy88bSpb/n6lu5tVsklERxVt23/cf/+PYlJCSHB9du0aT/m1Yn6q/qrRY285luX8sFoYYKMzMRv1k1VqYqmjPvu5eFLUlJvrlo7Ua0WZnRxEmlhYf6O35a9GP3Ox++fbB7V7acdC7Nz7uOl46e3HT/9y6A+M6eN/97TPeDPQ2vAaLAyFqMkN84owML5fa9QH2zmjLmiCmPPnpq3YGavXn1+2rJ3/tyPUlNTPvviI7FlJZd0bN++5YdNa4cMHr5l855+/Qb/tnfHlq01K6ZaY69ZkVsikRprzuy5C79LOOkrw5b4eof6+YQNHTAnOeX65X+PiFfValXPrmND6jVjGKZNyz74LUxOuYHnj534qXlkd0rPGUQAAAcYSURBVJSmg4ML9pHhYW3AmHASNv0ecaNzLZ2Vtd+v6typGyoJ+7zIyOaTJr5x8uSxa9qxu5JLOi5cPNe4ccRzz/V1c3Pv22fgyhXrnm7XEWpCjW3EElX5ta51CI7L9YIiHB1LA0Me7v6eHkF3Es7rGgQHRooHDvbCKqFCZT7KMSMr0denvq5NUEATMCoaXqEgToi1TPHFxd1s0iRS97BxI2Enm2vXrlR+SUdUVIuzZ08t/fj93/fvzs3LDQwICg9vBHWEYRuRYTTGC1cVKhWJyVcx+KJ/Mi8/U+/Vy38HlEUFGo1aLnfQnZHJ7MGoMAzHGmtMqAVPvo+9QqEoKiqSyx+tvXJwEP6eDx4UVHJJ/w7YXzo4OMYcP7Jk6XsSiaRLl57j//d/Xl51s+rcsBClMgkDxgqkOTt71g9p+Vy3MlXnHB0rC1jayR1ZllOplLozRcUPwJhgH2xnT15iU3+2eg2xsxN0plQ+WrtUoNWZp4dXJZf078CyLI7I+C8+Pu7cudPrNqwuKFB8uHA5VBtGexeDlwwL0c1TmpFirIEpwLfh2Qt7w0JbsQ/f0/20OG/Pyrxg7Abc3fzj71569qFN8u/1GDAmGg3vV9/Ine4TUIuhGfuwxo2aXrnyaBsl8TisQcNKLunfAf3lRo2a1q/fIDQ0DP/lK/J/2/sr1BBeY7hMjmF5NmjhpFZVUFen1mBERqPR7Nq3vLhYmZaesGf/ik9WDE9JvVX5s1pE9bh09RAmVPD44N8bEpIug9EoVqjRRgxv4QCEwbA1s9zlcrm3t09s7Ml/zseWlJQMjH7pWMzhbdt+zMvPwzNfrfq0dau2DcOFfbEruaTjwMHf0bM+fvwoGojoyvx97GBUZM3WWFbirBjuEes3c8Bn5GcUORthMja6vTOmbD7098bPvn45LT0+OChyaPScKp2PHs++WlCQvWPvJz/8NAdH9v4vTN/88zwjVZBKu5MtlZM44YjX1LhHHDF8zPfrvj595viPm/dgdCY9I23rzxtXfPUJxgjbPNX+f2OniM0quaTjzTfeXbFy2Zy5b+Cxh4cnjtFDh4yEOqLCamDr3ktQ81yDp/3B9rh+JNE3RB49kbjffdWs24Hh9l1fCgDLZN2CWwMnBAY1NmDzVOgYtuzsWqQoAptEWaSKnkDiN1CII1r6opUKFFfhKr6WXd1O7MtK/jczsKnhdSo5uanLVgw3eMle7lRYZDgt4ecdNmVcXe5b8e6i7hVdwmwNxxn4BUODm48dVaGvd+tkiqu7DIj8uPmKZ69YBEKpT77my0nb9vI49XuFQnR28nxj0kaDl9ALkckM1wpi2TquyFjRexDehqpIJjVg40q4ynLoynzlhI/CgUwsfOWUWPvD4KXKZNGmh9uV47nxsfdD2/g9fhU7Gw938xsrdfsebvydWK+ho4Tg0oMV9SiWThXJg5fnhRTmKXNSjBs9JoSki+kcBwPI81EewQDLWHCvWFq1yBBVZ7EmLmmQdCUNrJ17VzIVmQ9e+yAUSMbyl5NCTWdo6zeZuLTB5T/vZN+z2n4x8RKqUDFhaRhQjMmTrFnRBwesKZ+G37uaGneGxAn0teT634kPshXjFlMVmgK+lrVvkMmfhIOm5NrhhPs3637JklmI/yftyoF4VzfJeKpCk1DJAvuaBVPGLAg9vT/nn8NZWYm59i523g08nNwtp7j9Q7KTC7IScpWFxVIZO3BcvYBGFvMrsKxlx7MFKnj/NY7qtXvODf/F/pVzOSY3/mwyywqz6vGvI5FxGp7X7UCkvwmMiLY+J1Om6ib/qBLKoz1qHhoSYrVP7QRdXns7/WZlGoD+PjMsD5ryRT5ZjufVwhsqKS6d2+bqKesxLDAkwsIKo2s0Fh3P1lInPaIODDHiPzy4df7BrQv5ORnFmhK+WKknRAnwJY9es7QULGqT1UqyVCaPlMiyQqVvsdyr0JgREvwPTwrnxdlD4pmH9xceimvOdbtwMRzwQvnk0odie4mUYTjG3knq4i6J/I9rQAMbXSZLMrXNc4S3dMB/QKHUDkI3haQYRCrjJFILLoglkWBE3vD7p0K0JKR2TNEDY01YNgFoQwWFGXYNLXj3GBsktKlz5n1LnZt3fFeG3J6DCjp0KkRL4tnBHviBHdxskRnXhCt53Yb6VHSVrP2aKdVhw8K7DMu26uIVEmkB4SdFDn/ur/SEa/kvvxvq6FqhgUuFaJH8/FlyZkqRRs3r7/BdbmmSbtulcmh3DmfKPansOtWHd9LF1ype9SQ20QVuHzXUvjzLCRVu7R0lz4/29wurLHFAhWjJFENhod7y84fRWu2x9gz/WOgfym3lVaogntUrqqCTlbBTWNlEgnhG3MZeVw3koZi1yQNdpFd7nuPsnaA6UCFSiICGbyhEQIVIIQIqRAoRUCFSiIAKkUIEVIgUIvh/AAAA//8K91KcAAAABklEQVQDAAPvFDLgENXIAAAAAElFTkSuQmCC",
|
|
"text/plain": [
|
|
"<IPython.core.display.Image object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"try:\n",
|
|
" display(Image(agent_graph.get_graph().draw_mermaid_png()))\n",
|
|
"except Exception:\n",
|
|
" pass"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "1e9aff05",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Test"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"id": "8569cf39",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"config = {\"configurable\": {\"thread_id\": \"5\"}, \n",
|
|
" #\"callbacks\": [langfuse_handler],\n",
|
|
" #\"run_name\": \"rag-local-test\"differences between getDatetime() and getTimeStamp() functions in AVAP\n",
|
|
" }\n",
|
|
"\n",
|
|
"def stream_graph_updates(user_input: str):\n",
|
|
" for event in agent_graph.stream(\n",
|
|
" {\"messages\": [{\"role\": \"user\", \"content\": user_input}]},\n",
|
|
" #config=config,\n",
|
|
" stream_mode=\"values\",\n",
|
|
" ):\n",
|
|
" event[\"messages\"][-1].pretty_print()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"id": "a1a1f3cf",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"user_input = \"Create a small snippet that adds 2 numbers using AVAP language. Use the provided tool.\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "53b89690",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"a = stream_graph_updates(user_input)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "367b898a",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Evaluation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "f4119e2c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from dataclasses import dataclass\n",
|
|
"from typing import Any, Iterable\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"import mteb\n",
|
|
"from mteb.types import Array\n",
|
|
"from mteb.models import SearchEncoderWrapper\n",
|
|
"\n",
|
|
"\n",
|
|
"def _l2_normalize(x: np.ndarray, eps: float = 1e-12) -> np.ndarray:\n",
|
|
" norms = np.linalg.norm(x, axis=1, keepdims=True)\n",
|
|
" return x / np.clip(norms, eps, None)\n",
|
|
"\n",
|
|
"\n",
|
|
"def _to_text_list(batch: dict[str, Any]) -> list[str]:\n",
|
|
" \"\"\"\n",
|
|
" MTEB batched inputs can be:\n",
|
|
" - TextInput: {\"text\": [..]}\n",
|
|
" - CorpusInput: {\"title\": [..], \"body\": [..], \"text\": [..]}\n",
|
|
" - QueryInput: {\"query\": [..], \"instruction\": [..], \"text\": [..]}\n",
|
|
" We prefer \"text\" if present; otherwise compose from title/body or query/instruction.\n",
|
|
" \"\"\"\n",
|
|
" if \"text\" in batch and batch[\"text\"] is not None:\n",
|
|
" return list(batch[\"text\"])\n",
|
|
"\n",
|
|
" if \"title\" in batch and \"body\" in batch:\n",
|
|
" titles = batch[\"title\"] or [\"\"] * len(batch[\"body\"])\n",
|
|
" bodies = batch[\"body\"] or [\"\"] * len(batch[\"title\"])\n",
|
|
" return [f\"{t} {b}\".strip() for t, b in zip(titles, bodies)]\n",
|
|
"\n",
|
|
" if \"query\" in batch:\n",
|
|
" queries = list(batch[\"query\"])\n",
|
|
" instructions = batch.get(\"instruction\")\n",
|
|
" if instructions:\n",
|
|
" return [f\"{i} {q}\".strip() for q, i in zip(queries, instructions)]\n",
|
|
" return queries\n",
|
|
"\n",
|
|
" raise ValueError(f\"Unsupported batch keys: {sorted(batch.keys())}\")\n",
|
|
"\n",
|
|
"\n",
|
|
"@dataclass\n",
|
|
"class OllamaLangChainEncoder:\n",
|
|
" lc_embeddings: Any # OllamaEmbeddings implements embed_documents()\n",
|
|
" normalize: bool = True\n",
|
|
"\n",
|
|
" # Optional metadata hook used by some wrappers; safe to keep as None for local runs\n",
|
|
" mteb_model_meta: Any = None\n",
|
|
"\n",
|
|
" def encode(\n",
|
|
" self,\n",
|
|
" inputs: Iterable[dict[str, Any]],\n",
|
|
" *,\n",
|
|
" task_metadata: Any,\n",
|
|
" hf_split: str,\n",
|
|
" hf_subset: str,\n",
|
|
" prompt_type: Any = None,\n",
|
|
" **kwargs: Any,\n",
|
|
" ) -> Array:\n",
|
|
" all_vecs: list[np.ndarray] = []\n",
|
|
"\n",
|
|
" for batch in inputs:\n",
|
|
" texts = _to_text_list(batch)\n",
|
|
" vecs = self.lc_embeddings.embed_documents(texts)\n",
|
|
" arr = np.asarray(vecs, dtype=np.float32)\n",
|
|
" if self.normalize:\n",
|
|
" arr = _l2_normalize(arr)\n",
|
|
" all_vecs.append(arr)\n",
|
|
"\n",
|
|
" if not all_vecs:\n",
|
|
" return np.zeros((0, 0), dtype=np.float32)\n",
|
|
"\n",
|
|
" return np.vstack(all_vecs)\n",
|
|
"\n",
|
|
" def similarity(self, embeddings1: Array, embeddings2: Array) -> Array:\n",
|
|
" a = np.asarray(embeddings1, dtype=np.float32)\n",
|
|
" b = np.asarray(embeddings2, dtype=np.float32)\n",
|
|
" if self.normalize:\n",
|
|
" # dot == cosine if already normalized\n",
|
|
" return a @ b.T\n",
|
|
" a = _l2_normalize(a)\n",
|
|
" b = _l2_normalize(b)\n",
|
|
" return a @ b.T\n",
|
|
"\n",
|
|
" def similarity_pairwise(self, embeddings1: Array, embeddings2: Array) -> Array:\n",
|
|
" a = np.asarray(embeddings1, dtype=np.float32)\n",
|
|
" b = np.asarray(embeddings2, dtype=np.float32)\n",
|
|
" if not self.normalize:\n",
|
|
" a = _l2_normalize(a)\n",
|
|
" b = _l2_normalize(b)\n",
|
|
" return np.sum(a * b, axis=1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "f99b0712",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "93602dede149489ca26083ecd40477e8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Evaluating tasks: 0%| | 0/4 [00:00<?, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "cb1ae9433b774a139d6b5feeee9109ac",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filtering queries by qrels: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "aeef5c6478aa457bb742ce3a675fffaa",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Processing queries for dataloading: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "b4154824082c4777814cf12edbb0bf8c",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "dde212727c714ed4bfb4a998c658d553",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filtering queries by qrels: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e47d5f222d9f430d89f31517e601d758",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Processing queries for dataloading: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "aff05c0e82f244698aad60281530a940",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "c3e3c6b083d441f5944465fe4fbee349",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filtering queries by qrels: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "91040cf797204368b76dcdb0a0a2ad5d",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Processing queries for dataloading: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "8a8735049f824a5b8d271f9a88d7b569",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e3742f5ae3c340b4aebe0b8b4970d049",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filtering queries by qrels: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "89f57073615d4f46829afcca58a0801a",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Processing queries for dataloading: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "8a575b19d2d049c18075ac9458e014e5",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "04e6120e40c441c69ee41759ba3e7b04",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filtering queries by qrels: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e852ccab35314bc39c468b2a603a42a4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Processing queries for dataloading: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "00bbeda82d414947923d59e9b1d13982",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/1000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "2fb562b60cc244a0b17444cc0091bed1",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"README.md: 0.00B [00:00, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "7b8b8554f78943cea9a43ef98bae4759",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"python-qrels/train-00000-of-00001.parque(…): 0%| | 0.00/2.86M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "62551166b334404a8fdd029a5853f984",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"python-qrels/valid-00000-of-00001.parque(…): 0%| | 0.00/162k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "105fc5d624c74cf192526518ca42b649",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"python-qrels/test-00000-of-00001.parquet: 0%| | 0.00/173k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "9684bb4101cc478d83117bdea8f513fe",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating train split: 0%| | 0/251820 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "44c17e939ca84d2fbc4b01397b41404e",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating valid split: 0%| | 0/13914 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "17683aa7fc0f46b78c919266cb910941",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating test split: 0%| | 0/14918 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a79083758d4448088f5659ee1dc40511",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"python-corpus/corpus-00000-of-00001.parq(…): 0%| | 0.00/45.7M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "b88e879424b944449a16728c315492a1",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating corpus split: 0%| | 0/280652 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a764d586e96f4150aa4b04886960f3fb",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"python-queries/queries-00000-of-00001.pa(…): 0%| | 0.00/68.8M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "0c686988a0e4467a8639de5d11ea66f8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating queries split: 0%| | 0/280652 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "4fcfc211375041d08d8707b08e39ec46",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filter: 0%| | 0/280652 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "6f05dd1467244b5782ede555870152e2",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"javascript-qrels/train-00000-of-00001.pa(…): 0%| | 0.00/659k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "5f3ed10365b6419eac111509f599744a",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"javascript-qrels/valid-00000-of-00001.pa(…): 0%| | 0.00/45.2k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e6da433a24314cbb9398f0c49f9c9a45",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"javascript-qrels/test-00000-of-00001.par(…): 0%| | 0.00/39.6k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "68388f56d6604894bf4b0caf4a3b00b6",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating train split: 0%| | 0/58025 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "2e23701740664132b903a0c1c790c063",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating valid split: 0%| | 0/3885 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "48adfa854b8d49829859fd25aafa6bc3",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating test split: 0%| | 0/3291 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "3446299236094c1eb47ac73a6836aae2",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"javascript-corpus/corpus-00000-of-00001.(…): 0%| | 0.00/8.60M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "d84a2426cf0f48a9a67bc7f45d891c8c",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating corpus split: 0%| | 0/65201 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "6ac041bf57a549779f6ff8bfadbcccd1",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"javascript-queries/queries-00000-of-0000(…): 0%| | 0.00/12.1M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "faad96d058a04b61bebfc7c7c7cc5ae8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating queries split: 0%| | 0/65201 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "b5331e488b1248ab97b70905b8aadea4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filter: 0%| | 0/65201 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "32f9fe24cfa9487e8d6f0502737c3e2a",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"go-qrels/train-00000-of-00001.parquet: 0%| | 0.00/1.90M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "cf486dff97734d728d73e3b0246302f4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"go-qrels/valid-00000-of-00001.parquet: 0%| | 0.00/85.8k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "cdfee983c61245dd90ee4cdfb586f13a",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"go-qrels/test-00000-of-00001.parquet: 0%| | 0.00/95.2k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "dadfd7cdefdf4db2962c227d1f12872d",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating train split: 0%| | 0/167288 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "dee90cd582e242feb1f512de251010a4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating valid split: 0%| | 0/7325 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "18b96c08b6254fb2a6029ef15d619867",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating test split: 0%| | 0/8122 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "6ea49ca07e494f8ea3350f1a902254a8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"go-corpus/corpus-00000-of-00001.parquet: 0%| | 0.00/16.4M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e6c7ca2c40a5448fb853a26022d29634",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating corpus split: 0%| | 0/182735 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "f0af062718c14c5fad05cbc0108b5b76",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"go-queries/queries-00000-of-00001.parque(…): 0%| | 0.00/22.5M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "7dd644ae97c848ac93a3fe7722434ed9",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating queries split: 0%| | 0/182735 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "5a94c799c6714cb487d8eec7fb5711b4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filter: 0%| | 0/182735 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "b1a6579d14874ad48ab6dcf6178c63b3",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"ruby-qrels/train-00000-of-00001.parquet: 0%| | 0.00/283k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "1f35d3e72a8d419697ea334da328d949",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"ruby-qrels/valid-00000-of-00001.parquet: 0%| | 0.00/17.6k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "29cf20656dca49b4be10953340b09712",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"ruby-qrels/test-00000-of-00001.parquet: 0%| | 0.00/16.0k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "d74838ba909049f5a7d8fe53d8bd0da2",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating train split: 0%| | 0/24927 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "98ea4ec5d9be4a95b6b48d1645b5263d",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating valid split: 0%| | 0/1400 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "f68525af8e4743978ce3e4e656684713",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating test split: 0%| | 0/1261 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "764e9b9fdbf0457d92e731b99b4cc97d",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"ruby-corpus/corpus-00000-of-00001.parque(…): 0%| | 0.00/2.61M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e76834f109224d3d9aeaf3ec49b6c3dd",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating corpus split: 0%| | 0/27588 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "53f27cb9efd1480895dfb6bac2a39562",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"ruby-queries/queries-00000-of-00001.parq(…): 0%| | 0.00/3.95M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "21217a84ba894ee1b50017e0524d33d4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating queries split: 0%| | 0/27588 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a5d8a6542917484f80c060616c26182e",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filter: 0%| | 0/27588 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "c661eab9c893421fbf91b042f28414e0",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"java-qrels/train-00000-of-00001.parquet: 0%| | 0.00/1.87M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "37cac267e08643d18cd294ae862c000e",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"java-qrels/valid-00000-of-00001.parquet: 0%| | 0.00/61.2k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a161ebc931f748c6b63425a3d7a76689",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"java-qrels/test-00000-of-00001.parquet: 0%| | 0.00/127k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "2bc084f9d21c4d1e83b4ce92389dd176",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating train split: 0%| | 0/164923 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "cf8c32eed2a644a4b2aecc00d6b82f11",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating valid split: 0%| | 0/5183 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "c707440dd7a54c17aca2139cebec5a01",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating test split: 0%| | 0/10955 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "53c1b3db4b054359ae55f5d9c94d7411",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"java-corpus/corpus-00000-of-00001.parque(…): 0%| | 0.00/20.1M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a5550ec81c6b42b3819da18483ba6e4b",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating corpus split: 0%| | 0/181061 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "04cbe229636642c7a6d4c5c1978c0897",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"java-queries/queries-00000-of-00001.parq(…): 0%| | 0.00/29.7M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "9c87801ff24c4ae8b82faf8dde366e7b",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating queries split: 0%| | 0/181061 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e3a6384960734306a8125a7598bd6079",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filter: 0%| | 0/181061 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "edbdfb7ff091489bba4405b30e882728",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"php-qrels/train-00000-of-00001.parquet: 0%| | 0.00/2.74M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "dc340b9249db4e088d207da7585f9400",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"php-qrels/valid-00000-of-00001.parquet: 0%| | 0.00/151k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "251c577f2fc945cf9b73ffb595afe99c",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"php-qrels/test-00000-of-00001.parquet: 0%| | 0.00/163k [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "b3e74ffecb8a4bcd8ca70b711da75fbf",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating train split: 0%| | 0/241241 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "5802a8890a894c1ba4e854c3cdc5bace",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating valid split: 0%| | 0/12982 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "21237de330ca4ebaac5c98095581f420",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating test split: 0%| | 0/14014 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "644ab1e05eb4408eab2d8434296d317b",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"php-corpus/corpus-00000-of-00001.parquet: 0%| | 0.00/28.6M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "73c87dad0a95460f8fb646ca4e60e34d",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating corpus split: 0%| | 0/268237 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "7f58e8aab9854229bbcf8b72c4b39b08",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"php-queries/queries-00000-of-00001.parqu(…): 0%| | 0.00/41.9M [00:00<?, ?B/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "50bceaf8a83d422698929d819dfb7c67",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Generating queries split: 0%| | 0/268237 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "66e03865675a4027a5c92186375066f8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filter: 0%| | 0/268237 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "00f6dfcba9a34d24827ff9929f23ac1e",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Filtering queries by qrels: 0%| | 0/14918 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "cb9181bb83ff4d0585390b3fad7742bd",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Processing queries for dataloading: 0%| | 0/14918 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "c7139c9c103b42709940168130dd84de",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/50000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "dafb9d036ed94575985b01f4f1b59858",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/50000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "994d09e7a4454b81b64bab20716967f8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Converting corpus dict: 0%| | 0/50000 [00:00<?, ? examples/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"ename": "KeyboardInterrupt",
|
|
"evalue": "",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
|
"\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 10\u001b[39m\n\u001b[32m 2\u001b[39m search_model = SearchEncoderWrapper(encoder)\n\u001b[32m 4\u001b[39m tasks = mteb.get_tasks([\n\u001b[32m 5\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mCodeSearchNetRetrieval\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 6\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mCodeSearchNetCCRetrieval\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 7\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mAppsRetrieval\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 8\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mStackOverflowDupQuestions\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 9\u001b[39m ])\n\u001b[32m---> \u001b[39m\u001b[32m10\u001b[39m results = \u001b[43mmteb\u001b[49m\u001b[43m.\u001b[49m\u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 11\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43msearch_model\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 12\u001b[39m \u001b[43m \u001b[49m\u001b[43mtasks\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtasks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 13\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43m{\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mbatch_size\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m32\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mshow_progress_bar\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m}\u001b[49m\n\u001b[32m 14\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 16\u001b[39m \u001b[38;5;28mprint\u001b[39m(results)\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/evaluate.py:388\u001b[39m, in \u001b[36mevaluate\u001b[39m\u001b[34m(model, tasks, co2_tracker, raise_error, encode_kwargs, cache, overwrite_strategy, prediction_folder, show_progress_bar, public_only, num_proc)\u001b[39m\n\u001b[32m 386\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m i, task \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(tasks_tqdm):\n\u001b[32m 387\u001b[39m tasks_tqdm.set_description(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mEvaluating task \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtask.metadata.name\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m--> \u001b[39m\u001b[32m388\u001b[39m _res = \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 389\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 390\u001b[39m \u001b[43m \u001b[49m\u001b[43mtask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 391\u001b[39m \u001b[43m \u001b[49m\u001b[43mco2_tracker\u001b[49m\u001b[43m=\u001b[49m\u001b[43mco2_tracker\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 392\u001b[39m \u001b[43m \u001b[49m\u001b[43mraise_error\u001b[49m\u001b[43m=\u001b[49m\u001b[43mraise_error\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 393\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 394\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 395\u001b[39m \u001b[43m \u001b[49m\u001b[43moverwrite_strategy\u001b[49m\u001b[43m=\u001b[49m\u001b[43moverwrite_strategy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 396\u001b[39m \u001b[43m \u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 397\u001b[39m \u001b[43m \u001b[49m\u001b[43mshow_progress_bar\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 398\u001b[39m \u001b[43m \u001b[49m\u001b[43mpublic_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpublic_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 399\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 400\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 401\u001b[39m evaluate_results.extend(_res.task_results)\n\u001b[32m 402\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _res.exceptions:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/evaluate.py:487\u001b[39m, in \u001b[36mevaluate\u001b[39m\u001b[34m(model, tasks, co2_tracker, raise_error, encode_kwargs, cache, overwrite_strategy, prediction_folder, show_progress_bar, public_only, num_proc)\u001b[39m\n\u001b[32m 485\u001b[39m result = TaskError(task_name=task.metadata.name, exception=\u001b[38;5;28mstr\u001b[39m(e))\n\u001b[32m 486\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m487\u001b[39m result = \u001b[43m_evaluate_task\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 488\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 489\u001b[39m \u001b[43m \u001b[49m\u001b[43msplits\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmissing_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 490\u001b[39m \u001b[43m \u001b[49m\u001b[43mtask\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 491\u001b[39m \u001b[43m \u001b[49m\u001b[43mco2_tracker\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 492\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 493\u001b[39m \u001b[43m \u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mpublic_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpublic_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 497\u001b[39m logger.info(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m✓ Finished evaluation for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtask.metadata.name\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m 499\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(result, TaskError):\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/evaluate.py:161\u001b[39m, in \u001b[36m_evaluate_task\u001b[39m\u001b[34m(model, task, splits, co2_tracker, encode_kwargs, prediction_folder, public_only, num_proc)\u001b[39m\n\u001b[32m 159\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m split, hf_subsets \u001b[38;5;129;01min\u001b[39;00m splits.items():\n\u001b[32m 160\u001b[39m tick = time()\n\u001b[32m--> \u001b[39m\u001b[32m161\u001b[39m task_results[split] = \u001b[43mtask\u001b[49m\u001b[43m.\u001b[49m\u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 162\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 163\u001b[39m \u001b[43m \u001b[49m\u001b[43msplit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 164\u001b[39m \u001b[43m \u001b[49m\u001b[43msubsets_to_run\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_subsets\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 165\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 166\u001b[39m \u001b[43m \u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 167\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 168\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 169\u001b[39m tock = time()\n\u001b[32m 171\u001b[39m logger.debug(\n\u001b[32m 172\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mEvaluation for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtask.metadata.name\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m on \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msplit\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m took \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtock\u001b[38;5;250m \u001b[39m-\u001b[38;5;250m \u001b[39mtick\u001b[38;5;132;01m:\u001b[39;00m\u001b[33m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m seconds\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 173\u001b[39m )\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/abstasks/retrieval.py:342\u001b[39m, in \u001b[36mAbsTaskRetrieval.evaluate\u001b[39m\u001b[34m(self, model, split, subsets_to_run, encode_kwargs, prediction_folder, num_proc, **kwargs)\u001b[39m\n\u001b[32m 339\u001b[39m \u001b[38;5;66;03m# TODO: convert all tasks directly https://github.com/embeddings-benchmark/mteb/issues/2030\u001b[39;00m\n\u001b[32m 340\u001b[39m \u001b[38;5;28mself\u001b[39m.convert_v1_dataset_format_to_v2(num_proc=num_proc)\n\u001b[32m--> \u001b[39m\u001b[32m342\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 343\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 344\u001b[39m \u001b[43m \u001b[49m\u001b[43msplit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 345\u001b[39m \u001b[43m \u001b[49m\u001b[43msubsets_to_run\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 346\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 347\u001b[39m \u001b[43m \u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 348\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 349\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 350\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/abstasks/abstask.py:199\u001b[39m, in \u001b[36mAbsTask.evaluate\u001b[39m\u001b[34m(self, model, split, subsets_to_run, encode_kwargs, prediction_folder, num_proc, **kwargs)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 198\u001b[39m data_split = \u001b[38;5;28mself\u001b[39m.dataset[hf_subset][split]\n\u001b[32m--> \u001b[39m\u001b[32m199\u001b[39m scores[hf_subset] = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_evaluate_subset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43mdata_split\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m=\u001b[49m\u001b[43msplit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 205\u001b[39m \u001b[43m \u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprediction_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 206\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 207\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 208\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 209\u001b[39m \u001b[38;5;28mself\u001b[39m._add_main_score(scores[hf_subset])\n\u001b[32m 210\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m scores\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/abstasks/retrieval.py:409\u001b[39m, in \u001b[36mAbsTaskRetrieval._evaluate_subset\u001b[39m\u001b[34m(self, model, data_split, encode_kwargs, hf_split, hf_subset, prediction_folder, num_proc, **kwargs)\u001b[39m\n\u001b[32m 404\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[32m 405\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mRetrievalEvaluator expects a SearchInterface, Encoder, or CrossEncoder, got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(model)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 406\u001b[39m )\n\u001b[32m 408\u001b[39m start_time = time()\n\u001b[32m--> \u001b[39m\u001b[32m409\u001b[39m results = \u001b[43mretriever\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 410\u001b[39m \u001b[43m \u001b[49m\u001b[43msearch_model\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 411\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 412\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 413\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 414\u001b[39m end_time = time()\n\u001b[32m 415\u001b[39m logger.debug(\n\u001b[32m 416\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mRunning retrieval task - Time taken to retrieve: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mend_time\u001b[38;5;250m \u001b[39m-\u001b[38;5;250m \u001b[39mstart_time\u001b[38;5;132;01m:\u001b[39;00m\u001b[33m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m seconds\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 417\u001b[39m )\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/_evaluators/retrieval_evaluator.py:70\u001b[39m, in \u001b[36mRetrievalEvaluator.__call__\u001b[39m\u001b[34m(self, search_model, encode_kwargs, num_proc)\u001b[39m\n\u001b[32m 61\u001b[39m search_model.index(\n\u001b[32m 62\u001b[39m corpus=\u001b[38;5;28mself\u001b[39m.corpus,\n\u001b[32m 63\u001b[39m task_metadata=\u001b[38;5;28mself\u001b[39m.task_metadata,\n\u001b[32m (...)\u001b[39m\u001b[32m 67\u001b[39m num_proc=num_proc,\n\u001b[32m 68\u001b[39m )\n\u001b[32m 69\u001b[39m logger.info(\u001b[33m\"\u001b[39m\u001b[33mRunning retrieval task - Searching queries...\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m70\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msearch_model\u001b[49m\u001b[43m.\u001b[49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 71\u001b[39m \u001b[43m \u001b[49m\u001b[43mqueries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mqueries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 72\u001b[39m \u001b[43m \u001b[49m\u001b[43mtop_k\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtop_k\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 73\u001b[39m \u001b[43m \u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 75\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 76\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 77\u001b[39m \u001b[43m \u001b[49m\u001b[43mtop_ranked\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtop_ranked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 78\u001b[39m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 79\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/models/search_wrappers.py:183\u001b[39m, in \u001b[36mSearchEncoderWrapper.search\u001b[39m\u001b[34m(self, queries, task_metadata, hf_split, hf_subset, top_k, encode_kwargs, top_ranked, num_proc)\u001b[39m\n\u001b[32m 181\u001b[39m logger.info(\u001b[33m\"\u001b[39m\u001b[33mPerforming full corpus search...\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 182\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.index_backend \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m183\u001b[39m result_heaps = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_full_corpus_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 184\u001b[39m \u001b[43m \u001b[49m\u001b[43mquery_idx_to_id\u001b[49m\u001b[43m=\u001b[49m\u001b[43mquery_idx_to_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 185\u001b[39m \u001b[43m \u001b[49m\u001b[43mquery_embeddings\u001b[49m\u001b[43m=\u001b[49m\u001b[43mquery_embeddings\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 186\u001b[39m \u001b[43m \u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 187\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 188\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 189\u001b[39m \u001b[43m \u001b[49m\u001b[43mtop_k\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtop_k\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 190\u001b[39m \u001b[43m \u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 191\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 192\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 193\u001b[39m cos_scores_top_k_values, cos_scores_top_k_idx = (\n\u001b[32m 194\u001b[39m \u001b[38;5;28mself\u001b[39m.index_backend.search(\n\u001b[32m 195\u001b[39m query_embeddings,\n\u001b[32m (...)\u001b[39m\u001b[32m 200\u001b[39m )\n\u001b[32m 201\u001b[39m )\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/mteb/models/search_wrappers.py:253\u001b[39m, in \u001b[36mSearchEncoderWrapper._full_corpus_search\u001b[39m\u001b[34m(self, query_idx_to_id, query_embeddings, task_metadata, hf_subset, hf_split, top_k, encode_kwargs)\u001b[39m\n\u001b[32m 249\u001b[39m sub_corpus = \u001b[38;5;28mself\u001b[39m.task_corpus.select(\n\u001b[32m 250\u001b[39m \u001b[38;5;28mrange\u001b[39m(corpus_start_idx, corpus_end_idx)\n\u001b[32m 251\u001b[39m )\n\u001b[32m 252\u001b[39m sub_corpus_ids = sub_corpus[\u001b[33m\"\u001b[39m\u001b[33mid\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m--> \u001b[39m\u001b[32m253\u001b[39m sub_corpus_embeddings = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 254\u001b[39m \u001b[43m \u001b[49m\u001b[43mcreate_dataloader\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 255\u001b[39m \u001b[43m \u001b[49m\u001b[43msub_corpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 256\u001b[39m \u001b[43m \u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 257\u001b[39m \u001b[43m \u001b[49m\u001b[43mprompt_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mPromptType\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdocument\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 258\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 259\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 260\u001b[39m \u001b[43m \u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtask_metadata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 261\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_split\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 262\u001b[39m \u001b[43m \u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhf_subset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 263\u001b[39m \u001b[43m \u001b[49m\u001b[43mprompt_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mPromptType\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdocument\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 264\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mencode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 265\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 267\u001b[39m \u001b[38;5;66;03m# Compute similarities using either cosine-similarity or dot product\u001b[39;00m\n\u001b[32m 268\u001b[39m logger.info(\u001b[33m\"\u001b[39m\u001b[33mComputing Similarities...\u001b[39m\u001b[33m\"\u001b[39m)\n",
|
|
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 64\u001b[39m, in \u001b[36mOllamaLangChainEncoder.encode\u001b[39m\u001b[34m(self, inputs, task_metadata, hf_split, hf_subset, prompt_type, **kwargs)\u001b[39m\n\u001b[32m 62\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m inputs:\n\u001b[32m 63\u001b[39m texts = _to_text_list(batch)\n\u001b[32m---> \u001b[39m\u001b[32m64\u001b[39m vecs = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mlc_embeddings\u001b[49m\u001b[43m.\u001b[49m\u001b[43membed_documents\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 65\u001b[39m arr = np.asarray(vecs, dtype=np.float32)\n\u001b[32m 66\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.normalize:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/langchain_ollama/embeddings.py:301\u001b[39m, in \u001b[36mOllamaEmbeddings.embed_documents\u001b[39m\u001b[34m(self, texts)\u001b[39m\n\u001b[32m 296\u001b[39m msg = (\n\u001b[32m 297\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mOllama client is not initialized. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 298\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mPlease ensure Ollama is running and the model is loaded.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 299\u001b[39m )\n\u001b[32m 300\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg)\n\u001b[32m--> \u001b[39m\u001b[32m301\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43membed\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 302\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_default_params\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeep_alive\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mkeep_alive\u001b[49m\n\u001b[32m 303\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m[\u001b[33m\"\u001b[39m\u001b[33membeddings\u001b[39m\u001b[33m\"\u001b[39m]\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/ollama/_client.py:393\u001b[39m, in \u001b[36mClient.embed\u001b[39m\u001b[34m(self, model, input, truncate, options, keep_alive, dimensions)\u001b[39m\n\u001b[32m 384\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34membed\u001b[39m(\n\u001b[32m 385\u001b[39m \u001b[38;5;28mself\u001b[39m,\n\u001b[32m 386\u001b[39m model: \u001b[38;5;28mstr\u001b[39m = \u001b[33m'\u001b[39m\u001b[33m'\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 391\u001b[39m dimensions: Optional[\u001b[38;5;28mint\u001b[39m] = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m 392\u001b[39m ) -> EmbedResponse:\n\u001b[32m--> \u001b[39m\u001b[32m393\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 394\u001b[39m \u001b[43m \u001b[49m\u001b[43mEmbedResponse\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 395\u001b[39m \u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mPOST\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 396\u001b[39m \u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m/api/embed\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 397\u001b[39m \u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mEmbedRequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 398\u001b[39m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 399\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m=\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 400\u001b[39m \u001b[43m \u001b[49m\u001b[43mtruncate\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtruncate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 401\u001b[39m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m=\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 402\u001b[39m \u001b[43m \u001b[49m\u001b[43mkeep_alive\u001b[49m\u001b[43m=\u001b[49m\u001b[43mkeep_alive\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 403\u001b[39m \u001b[43m \u001b[49m\u001b[43mdimensions\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdimensions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 404\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmodel_dump\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexclude_none\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 405\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/ollama/_client.py:189\u001b[39m, in \u001b[36mClient._request\u001b[39m\u001b[34m(self, cls, stream, *args, **kwargs)\u001b[39m\n\u001b[32m 185\u001b[39m \u001b[38;5;28;01myield\u001b[39;00m \u001b[38;5;28mcls\u001b[39m(**part)\n\u001b[32m 187\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m inner()\n\u001b[32m--> \u001b[39m\u001b[32m189\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mcls\u001b[39m(**\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_request_raw\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m.json())\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/ollama/_client.py:129\u001b[39m, in \u001b[36mClient._request_raw\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 127\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_request_raw\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs):\n\u001b[32m 128\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m129\u001b[39m r = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 130\u001b[39m r.raise_for_status()\n\u001b[32m 131\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m r\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpx/_client.py:825\u001b[39m, in \u001b[36mClient.request\u001b[39m\u001b[34m(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[39m\n\u001b[32m 810\u001b[39m warnings.warn(message, \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m, stacklevel=\u001b[32m2\u001b[39m)\n\u001b[32m 812\u001b[39m request = \u001b[38;5;28mself\u001b[39m.build_request(\n\u001b[32m 813\u001b[39m method=method,\n\u001b[32m 814\u001b[39m url=url,\n\u001b[32m (...)\u001b[39m\u001b[32m 823\u001b[39m extensions=extensions,\n\u001b[32m 824\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m825\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[43m=\u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpx/_client.py:914\u001b[39m, in \u001b[36mClient.send\u001b[39m\u001b[34m(self, request, stream, auth, follow_redirects)\u001b[39m\n\u001b[32m 910\u001b[39m \u001b[38;5;28mself\u001b[39m._set_timeout(request)\n\u001b[32m 912\u001b[39m auth = \u001b[38;5;28mself\u001b[39m._build_request_auth(request, auth)\n\u001b[32m--> \u001b[39m\u001b[32m914\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_handling_auth\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 915\u001b[39m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 916\u001b[39m \u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[43m=\u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 917\u001b[39m \u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 918\u001b[39m \u001b[43m \u001b[49m\u001b[43mhistory\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 919\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 920\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 921\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m stream:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpx/_client.py:942\u001b[39m, in \u001b[36mClient._send_handling_auth\u001b[39m\u001b[34m(self, request, auth, follow_redirects, history)\u001b[39m\n\u001b[32m 939\u001b[39m request = \u001b[38;5;28mnext\u001b[39m(auth_flow)\n\u001b[32m 941\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m942\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_handling_redirects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 943\u001b[39m \u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 944\u001b[39m \u001b[43m \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 945\u001b[39m \u001b[43m \u001b[49m\u001b[43mhistory\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhistory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 946\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 947\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 948\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpx/_client.py:979\u001b[39m, in \u001b[36mClient._send_handling_redirects\u001b[39m\u001b[34m(self, request, follow_redirects, history)\u001b[39m\n\u001b[32m 976\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m._event_hooks[\u001b[33m\"\u001b[39m\u001b[33mrequest\u001b[39m\u001b[33m\"\u001b[39m]:\n\u001b[32m 977\u001b[39m hook(request)\n\u001b[32m--> \u001b[39m\u001b[32m979\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_single_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 980\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 981\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m._event_hooks[\u001b[33m\"\u001b[39m\u001b[33mresponse\u001b[39m\u001b[33m\"\u001b[39m]:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpx/_client.py:1014\u001b[39m, in \u001b[36mClient._send_single_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 1009\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m 1010\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mAttempted to send an async request with a sync Client instance.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 1011\u001b[39m )\n\u001b[32m 1013\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m request_context(request=request):\n\u001b[32m-> \u001b[39m\u001b[32m1014\u001b[39m response = \u001b[43mtransport\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1016\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response.stream, SyncByteStream)\n\u001b[32m 1018\u001b[39m response.request = request\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpx/_transports/default.py:250\u001b[39m, in \u001b[36mHTTPTransport.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 237\u001b[39m req = httpcore.Request(\n\u001b[32m 238\u001b[39m method=request.method,\n\u001b[32m 239\u001b[39m url=httpcore.URL(\n\u001b[32m (...)\u001b[39m\u001b[32m 247\u001b[39m extensions=request.extensions,\n\u001b[32m 248\u001b[39m )\n\u001b[32m 249\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[32m--> \u001b[39m\u001b[32m250\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 252\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp.stream, typing.Iterable)\n\u001b[32m 254\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m Response(\n\u001b[32m 255\u001b[39m status_code=resp.status,\n\u001b[32m 256\u001b[39m headers=resp.headers,\n\u001b[32m 257\u001b[39m stream=ResponseStream(resp.stream),\n\u001b[32m 258\u001b[39m extensions=resp.extensions,\n\u001b[32m 259\u001b[39m )\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py:256\u001b[39m, in \u001b[36mConnectionPool.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 253\u001b[39m closing = \u001b[38;5;28mself\u001b[39m._assign_requests_to_connections()\n\u001b[32m 255\u001b[39m \u001b[38;5;28mself\u001b[39m._close_connections(closing)\n\u001b[32m--> \u001b[39m\u001b[32m256\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exc \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 258\u001b[39m \u001b[38;5;66;03m# Return the response. Note that in this case we still have to manage\u001b[39;00m\n\u001b[32m 259\u001b[39m \u001b[38;5;66;03m# the point at which the response is closed.\u001b[39;00m\n\u001b[32m 260\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response.stream, typing.Iterable)\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py:236\u001b[39m, in \u001b[36mConnectionPool.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 232\u001b[39m connection = pool_request.wait_for_connection(timeout=timeout)\n\u001b[32m 234\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 235\u001b[39m \u001b[38;5;66;03m# Send the request on the assigned connection.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m236\u001b[39m response = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 237\u001b[39m \u001b[43m \u001b[49m\u001b[43mpool_request\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\n\u001b[32m 238\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 239\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[32m 240\u001b[39m \u001b[38;5;66;03m# In some cases a connection may initially be available to\u001b[39;00m\n\u001b[32m 241\u001b[39m \u001b[38;5;66;03m# handle a request, but then become unavailable.\u001b[39;00m\n\u001b[32m 242\u001b[39m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[32m 243\u001b[39m \u001b[38;5;66;03m# In this case we clear the connection and try again.\u001b[39;00m\n\u001b[32m 244\u001b[39m pool_request.clear_connection()\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/connection.py:103\u001b[39m, in \u001b[36mHTTPConnection.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 100\u001b[39m \u001b[38;5;28mself\u001b[39m._connect_failed = \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 101\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exc\n\u001b[32m--> \u001b[39m\u001b[32m103\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_connection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/http11.py:136\u001b[39m, in \u001b[36mHTTP11Connection.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 134\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Trace(\u001b[33m\"\u001b[39m\u001b[33mresponse_closed\u001b[39m\u001b[33m\"\u001b[39m, logger, request) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[32m 135\u001b[39m \u001b[38;5;28mself\u001b[39m._response_closed()\n\u001b[32m--> \u001b[39m\u001b[32m136\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exc\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/http11.py:106\u001b[39m, in \u001b[36mHTTP11Connection.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 95\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m 97\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Trace(\n\u001b[32m 98\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mreceive_response_headers\u001b[39m\u001b[33m\"\u001b[39m, logger, request, kwargs\n\u001b[32m 99\u001b[39m ) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[32m 100\u001b[39m (\n\u001b[32m 101\u001b[39m http_version,\n\u001b[32m 102\u001b[39m status,\n\u001b[32m 103\u001b[39m reason_phrase,\n\u001b[32m 104\u001b[39m headers,\n\u001b[32m 105\u001b[39m trailing_data,\n\u001b[32m--> \u001b[39m\u001b[32m106\u001b[39m ) = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_receive_response_headers\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 107\u001b[39m trace.return_value = (\n\u001b[32m 108\u001b[39m http_version,\n\u001b[32m 109\u001b[39m status,\n\u001b[32m 110\u001b[39m reason_phrase,\n\u001b[32m 111\u001b[39m headers,\n\u001b[32m 112\u001b[39m )\n\u001b[32m 114\u001b[39m network_stream = \u001b[38;5;28mself\u001b[39m._network_stream\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/http11.py:177\u001b[39m, in \u001b[36mHTTP11Connection._receive_response_headers\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m 174\u001b[39m timeout = timeouts.get(\u001b[33m\"\u001b[39m\u001b[33mread\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m 176\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m177\u001b[39m event = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_receive_event\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 178\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(event, h11.Response):\n\u001b[32m 179\u001b[39m \u001b[38;5;28;01mbreak\u001b[39;00m\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_sync/http11.py:217\u001b[39m, in \u001b[36mHTTP11Connection._receive_event\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m 214\u001b[39m event = \u001b[38;5;28mself\u001b[39m._h11_state.next_event()\n\u001b[32m 216\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m event \u001b[38;5;129;01mis\u001b[39;00m h11.NEED_DATA:\n\u001b[32m--> \u001b[39m\u001b[32m217\u001b[39m data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_network_stream\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 218\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mREAD_NUM_BYTES\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\n\u001b[32m 219\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 221\u001b[39m \u001b[38;5;66;03m# If we feed this case through h11 we'll raise an exception like:\u001b[39;00m\n\u001b[32m 222\u001b[39m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[32m 223\u001b[39m \u001b[38;5;66;03m# httpcore.RemoteProtocolError: can't handle event type\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 227\u001b[39m \u001b[38;5;66;03m# perspective. Instead we handle this case distinctly and treat\u001b[39;00m\n\u001b[32m 228\u001b[39m \u001b[38;5;66;03m# it as a ConnectError.\u001b[39;00m\n\u001b[32m 229\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m data == \u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m._h11_state.their_state == h11.SEND_RESPONSE:\n",
|
|
"\u001b[36mFile \u001b[39m\u001b[32m~/PycharmProjects/assistance-engine/.venv/lib/python3.11/site-packages/httpcore/_backends/sync.py:128\u001b[39m, in \u001b[36mSyncStream.read\u001b[39m\u001b[34m(self, max_bytes, timeout)\u001b[39m\n\u001b[32m 126\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m map_exceptions(exc_map):\n\u001b[32m 127\u001b[39m \u001b[38;5;28mself\u001b[39m._sock.settimeout(timeout)\n\u001b[32m--> \u001b[39m\u001b[32m128\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrecv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmax_bytes\u001b[49m\u001b[43m)\u001b[49m\n",
|
|
"\u001b[31mKeyboardInterrupt\u001b[39m: "
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"encoder = OllamaLangChainEncoder(lc_embeddings=embeddings, normalize=True)\n",
|
|
"search_model = SearchEncoderWrapper(encoder)\n",
|
|
"\n",
|
|
"tasks = mteb.get_tasks([\n",
|
|
" \"CodeSearchNetRetrieval\",\n",
|
|
" \"CodeSearchNetCCRetrieval\",\n",
|
|
" \"AppsRetrieval\",\n",
|
|
" \"StackOverflowDupQuestions\",\n",
|
|
"])\n",
|
|
"results = mteb.evaluate(\n",
|
|
" model=search_model,\n",
|
|
" tasks=tasks,\n",
|
|
" encode_kwargs={\"batch_size\": 32, \"show_progress_bar\": True}\n",
|
|
")\n",
|
|
"\n",
|
|
"print(results)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "b2c5d9f6",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"ModelResult(model_name=no_model_name/available, model_revision=no_revision_available, task_results=[...](#1))"
|
|
]
|
|
},
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"results"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "7364562f",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "AttributeError",
|
|
"evalue": "'list' object has no attribute 'scores'",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
|
"\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)",
|
|
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[10]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mresults\u001b[49m\u001b[43m.\u001b[49m\u001b[43mtask_results\u001b[49m\u001b[43m.\u001b[49m\u001b[43mscores\u001b[49m\n",
|
|
"\u001b[31mAttributeError\u001b[39m: 'list' object has no attribute 'scores'"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"results.task_results.scores"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a657676e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "assistance-engine",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.13"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|