Add BEIR analysis notebooks for different datasets and models
- Created `n00 Beir Analysis.ipynb` for analyzing BEIR dataset with Ollama embeddings. - Added `n00 Beir Analysis_cosqa.ipynb` for evaluating the CosQA dataset using similar embedding techniques. - Introduced `n00 first Analysis.ipynb` for initial analysis with Ragas embeddings and semantic similarity evaluation. - Implemented data loading and processing for each notebook, including downloading datasets and saving results. - Included evaluation metrics such as NDCG, MAP, Recall, and Precision for model performance assessment.
This commit is contained in:
acano
parent
752bf9c7d9
commit
dd3bde2ec9
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,318 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9f97dd1e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9e974df6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from __future__ import annotations\n",
|
||||
"\n",
|
||||
"from typing import List, Dict, Any, Optional, TypedDict\n",
|
||||
"from pydantic import BaseModel, Field\n",
|
||||
"import os\n",
|
||||
"from elasticsearch import Elasticsearch\n",
|
||||
"from langchain_community.embeddings import OllamaEmbeddings\n",
|
||||
"from langchain_community.llms import Ollama\n",
|
||||
"from langchain_core.messages import HumanMessage, SystemMessage\n",
|
||||
"from langchain_elasticsearch import ElasticsearchStore\n",
|
||||
"from langgraph.graph import StateGraph, END"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "30edcecc",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/tmp/ipykernel_837760/4144511709.py:6: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the `langchain-ollama package and should be used instead. To use it run `pip install -U `langchain-ollama` and import as `from `langchain_ollama import OllamaEmbeddings``.\n",
|
||||
" embeddings = OllamaEmbeddings(\n",
|
||||
"/tmp/ipykernel_837760/4144511709.py:8: LangChainDeprecationWarning: The class `Ollama` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the `langchain-ollama package and should be used instead. To use it run `pip install -U `langchain-ollama` and import as `from `langchain_ollama import OllamaLLM``.\n",
|
||||
" llm = Ollama(base_url=base_url, model=model_name)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"es = Elasticsearch(os.getenv(\"ELASTICSEARCH_LOCAL_URL\"))\n",
|
||||
"index_name = os.getenv(\"ELASTICSEARCH_INDEX\")\n",
|
||||
"base_url = os.getenv(\"LLM_BASE_LOCAL_URL\")\n",
|
||||
"model_name = os.getenv(\"OLLAMA_MODEL_NAME\")\n",
|
||||
"\n",
|
||||
"embeddings = OllamaEmbeddings(base_url=base_url, model=model_name)\n",
|
||||
"llm = Ollama(base_url=base_url, model=model_name)\n",
|
||||
"\n",
|
||||
"vector_store = ElasticsearchStore(\n",
|
||||
" es_url=es,\n",
|
||||
" index_name=index_name,\n",
|
||||
" embedding=embeddings,\n",
|
||||
" query_field=\"text\",\n",
|
||||
" vector_query_field=\"embedding\",\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1e1dd107",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# State"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "19e723e2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class RAGState(TypedDict, total=False):\n",
|
||||
" question: str\n",
|
||||
" query_embedding: List[float]\n",
|
||||
" docs: List[Dict[str, Any]]\n",
|
||||
" answer: str\n",
|
||||
" parsed: Dict[str, Any]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "90162558",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Schema"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "acfe33d8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class AnswerSchema(BaseModel):\n",
|
||||
" answer: str = Field(..., description=\"Final answer to the user\")\n",
|
||||
" citations: List[str] = Field(default_factory=list, description=\"List of sources/ids used\")\n",
|
||||
" confidence: str = Field(..., description=\"low|medium|high\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "88a4aba1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# ES Retrieval"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "6c9e9b89",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def es_vector_search(\n",
|
||||
" es_client: Elasticsearch,\n",
|
||||
" index: str,\n",
|
||||
" query_vector: List[float],\n",
|
||||
" k: int = 5,\n",
|
||||
" num_candidates: int = 50,\n",
|
||||
" title_filter: Optional[str] = None,\n",
|
||||
"):\n",
|
||||
" \"\"\"\n",
|
||||
" Uses Elasticsearch kNN search against your field: 'vector'\n",
|
||||
" and returns _source: text + metadata.title (matches your mapping).\n",
|
||||
" \"\"\"\n",
|
||||
" body: Dict[str, Any] = {\n",
|
||||
" \"knn\": {\n",
|
||||
" \"field\": \"vector\", # <-- your mapping\n",
|
||||
" \"query_vector\": query_vector,\n",
|
||||
" \"k\": k,\n",
|
||||
" \"num_candidates\": num_candidates,\n",
|
||||
" },\n",
|
||||
" \"_source\": [\"text\", \"metadata.title\"], # <-- your mapping\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # Optional: filter by title (exact via keyword subfield, or fuzzy via text)\n",
|
||||
" if title_filter:\n",
|
||||
" body[\"query\"] = {\n",
|
||||
" \"bool\": {\n",
|
||||
" \"filter\": [\n",
|
||||
" {\"term\": {\"metadata.title.keyword\": title_filter}}\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" res = es_client.search(index=index, body=body)\n",
|
||||
" return res[\"hits\"][\"hits\"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def normalize_hits(hits: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n",
|
||||
" docs: List[Dict[str, Any]] = []\n",
|
||||
" for h in hits:\n",
|
||||
" src = h.get(\"_source\", {}) or {}\n",
|
||||
" meta = src.get(\"metadata\", {}) or {}\n",
|
||||
" docs.append(\n",
|
||||
" {\n",
|
||||
" \"id\": h.get(\"_id\"),\n",
|
||||
" \"score\": h.get(\"_score\"),\n",
|
||||
" \"text\": src.get(\"text\", \"\"),\n",
|
||||
" \"title\": meta.get(\"title\", None),\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
" return docs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def format_context(docs: List[Dict[str, Any]]) -> str:\n",
|
||||
" chunks = []\n",
|
||||
" for i, d in enumerate(docs, 1):\n",
|
||||
" title = d.get(\"title\") or \"Untitled\"\n",
|
||||
" doc_id = d.get(\"id\") or f\"chunk-{i}\"\n",
|
||||
" chunks.append(f\"[{i}] id={doc_id} title={title}\\n{d.get('text','')}\")\n",
|
||||
" return \"\\n\\n\".join(chunks)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2591e778",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Nodes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "69c41a89",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def embed_query(state: RAGState) -> RAGState:\n",
|
||||
" q = state[\"question\"]\n",
|
||||
" vec = embeddings.embed_query(q)\n",
|
||||
" return {\"query_embedding\": vec}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def retrieve(state: RAGState) -> RAGState:\n",
|
||||
" base_retriever = vector_store.as_retriever(\n",
|
||||
" search_type=\"similarity\",\n",
|
||||
" search_kwargs={\"k\": 4}\n",
|
||||
" ) \n",
|
||||
" docs = base_retriever.invoke(\"What does the text say about agricultural techniques?\")\n",
|
||||
" docs = normalize_hits(hits)\n",
|
||||
" return {\"docs\": docs}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def generate_answer(state: RAGState) -> RAGState:\n",
|
||||
" question = state[\"question\"]\n",
|
||||
" docs = state.get(\"docs\", [])\n",
|
||||
" context = format_context(docs)\n",
|
||||
"\n",
|
||||
" system = SystemMessage(\n",
|
||||
" content=(\n",
|
||||
" \"You are a helpful RAG assistant. Use ONLY the provided context. \"\n",
|
||||
" \"If the context is insufficient, say what is missing and ask a precise follow-up question. \"\n",
|
||||
" \"Cite sources like [1], [2] based on the context chunks.\"\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" user = HumanMessage(\n",
|
||||
" content=f\"Question:\\n{question}\\n\\nContext:\\n{context}\\n\\nWrite the best possible answer.\"\n",
|
||||
" )\n",
|
||||
" resp = llm.invoke([system, user])\n",
|
||||
" # Handle both string and message object responses\n",
|
||||
" answer = resp.content if hasattr(resp, 'content') else resp\n",
|
||||
" return {\"answer\": answer}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f3933c13",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Graph"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d6810eb8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"graph = StateGraph(RAGState)\n",
|
||||
"\n",
|
||||
"graph.add_node(\"embed_query\", embed_query)\n",
|
||||
"graph.add_node(\"retrieve\", retrieve)\n",
|
||||
"graph.add_node(\"generate_answer\", generate_answer)\n",
|
||||
"\n",
|
||||
"graph.set_entry_point(\"embed_query\")\n",
|
||||
"graph.add_edge(\"embed_query\", \"retrieve\")\n",
|
||||
"graph.add_edge(\"retrieve\", \"generate_answer\")\n",
|
||||
"graph.add_edge(\"generate_answer\", END)\n",
|
||||
"\n",
|
||||
"dag_app = graph.compile()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6528efac",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Retrieve"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "568708ba",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"RAW ANSWER:\n",
|
||||
" New agricultural techniques include vertical farming, which integrates plants into multi-level structures in densely populated cities to optimize space and reduce transportation dependency. This approach uses hydroponic systems and automated nutrient control to grow food without traditional soil. Additionally, agriculture buildings are designed to seamlessly integrate with urban environments, addressing economic challenges while enhancing sustainability through the use of renewable energy sources.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"out = dag_app.invoke({\"question\": \"What are new agricultural techniques about?\"})\n",
|
||||
"\n",
|
||||
"print(\"RAW ANSWER:\\n\", out[\"answer\"])"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "assistance-engine",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
|
|
@ -0,0 +1,54 @@
|
|||
{
|
||||
"qwen3-0.6B-emb:latest": {
|
||||
"NDCG": {
|
||||
"NDCG@1": 0.94971,
|
||||
"NDCG@3": 0.96956,
|
||||
"NDCG@5": 0.97166,
|
||||
"NDCG@10": 0.97342
|
||||
},
|
||||
"MAP": {
|
||||
"MAP@1": 0.94971,
|
||||
"MAP@3": 0.96504,
|
||||
"MAP@5": 0.9662,
|
||||
"MAP@10": 0.96694
|
||||
},
|
||||
"Recall": {
|
||||
"Recall@1": 0.94971,
|
||||
"Recall@3": 0.98251,
|
||||
"Recall@5": 0.98761,
|
||||
"Recall@10": 0.99297
|
||||
},
|
||||
"Precision": {
|
||||
"P@1": 0.94971,
|
||||
"P@3": 0.3275,
|
||||
"P@5": 0.19752,
|
||||
"P@10": 0.0993
|
||||
}
|
||||
},
|
||||
"qwen2.5:1.5b": {
|
||||
"NDCG": {
|
||||
"NDCG@1": 0.00031,
|
||||
"NDCG@3": 0.00061,
|
||||
"NDCG@5": 0.00086,
|
||||
"NDCG@10": 0.00118
|
||||
},
|
||||
"MAP": {
|
||||
"MAP@1": 0.00031,
|
||||
"MAP@3": 0.00051,
|
||||
"MAP@5": 0.00065,
|
||||
"MAP@10": 0.00078
|
||||
},
|
||||
"Recall": {
|
||||
"Recall@1": 0.00031,
|
||||
"Recall@3": 0.00088,
|
||||
"Recall@5": 0.00151,
|
||||
"Recall@10": 0.0025
|
||||
},
|
||||
"Precision": {
|
||||
"P@1": 0.00031,
|
||||
"P@3": 0.00029,
|
||||
"P@5": 0.0003,
|
||||
"P@10": 0.00025
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
{
|
||||
"qwen3-0.6B-emb:latest": {
|
||||
"NDCG": {
|
||||
"NDCG@1": 0.56333,
|
||||
"NDCG@3": 0.64367,
|
||||
"NDCG@5": 0.66577,
|
||||
"NDCG@10": 0.68551,
|
||||
"NDCG@100": 0.71285
|
||||
},
|
||||
"MAP": {
|
||||
"MAP@1": 0.52994,
|
||||
"MAP@3": 0.6117,
|
||||
"MAP@5": 0.62815,
|
||||
"MAP@10": 0.6383,
|
||||
"MAP@100": 0.64466
|
||||
},
|
||||
"Recall": {
|
||||
"Recall@1": 0.52994,
|
||||
"Recall@3": 0.7035,
|
||||
"Recall@5": 0.75967,
|
||||
"Recall@10": 0.81611,
|
||||
"Recall@100": 0.94
|
||||
},
|
||||
"Precision": {
|
||||
"P@1": 0.56333,
|
||||
"P@3": 0.25889,
|
||||
"P@5": 0.17067,
|
||||
"P@10": 0.093,
|
||||
"P@100": 0.0107
|
||||
}
|
||||
},
|
||||
"qwen2.5:1.5b": {
|
||||
"NDCG": {
|
||||
"NDCG@1": 0.02333,
|
||||
"NDCG@3": 0.03498,
|
||||
"NDCG@5": 0.0404,
|
||||
"NDCG@10": 0.04619,
|
||||
"NDCG@100": 0.07768
|
||||
},
|
||||
"MAP": {
|
||||
"MAP@1": 0.02083,
|
||||
"MAP@3": 0.03083,
|
||||
"MAP@5": 0.03375,
|
||||
"MAP@10": 0.03632,
|
||||
"MAP@100": 0.04123
|
||||
},
|
||||
"Recall": {
|
||||
"Recall@1": 0.02083,
|
||||
"Recall@3": 0.04417,
|
||||
"Recall@5": 0.0575,
|
||||
"Recall@10": 0.07417,
|
||||
"Recall@100": 0.23144
|
||||
},
|
||||
"Precision": {
|
||||
"P@1": 0.02333,
|
||||
"P@3": 0.01556,
|
||||
"P@5": 0.01267,
|
||||
"P@10": 0.00833,
|
||||
"P@100": 0.00277
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
{
|
||||
"qwen3-0.6B-emb:latest": {
|
||||
"NDCG": {
|
||||
"NDCG@1": 0.174,
|
||||
"NDCG@3": 0.27374,
|
||||
"NDCG@5": 0.33509,
|
||||
"NDCG@10": 0.39086,
|
||||
"NDCG@100": 0.45099
|
||||
},
|
||||
"MAP": {
|
||||
"MAP@1": 0.174,
|
||||
"MAP@3": 0.247,
|
||||
"MAP@5": 0.2808,
|
||||
"MAP@10": 0.30466,
|
||||
"MAP@100": 0.31702
|
||||
},
|
||||
"Recall": {
|
||||
"Recall@1": 0.174,
|
||||
"Recall@3": 0.352,
|
||||
"Recall@5": 0.502,
|
||||
"Recall@10": 0.67,
|
||||
"Recall@100": 0.952
|
||||
},
|
||||
"Precision": {
|
||||
"P@1": 0.174,
|
||||
"P@3": 0.11733,
|
||||
"P@5": 0.1004,
|
||||
"P@10": 0.067,
|
||||
"P@100": 0.00952
|
||||
}
|
||||
},
|
||||
"qwen2.5:1.5b": {
|
||||
"NDCG": {
|
||||
"NDCG@1": 0.0,
|
||||
"NDCG@3": 0.0,
|
||||
"NDCG@5": 0.0,
|
||||
"NDCG@10": 0.0,
|
||||
"NDCG@100": 0.0021
|
||||
},
|
||||
"MAP": {
|
||||
"MAP@1": 0.0,
|
||||
"MAP@3": 0.0,
|
||||
"MAP@5": 0.0,
|
||||
"MAP@10": 0.0,
|
||||
"MAP@100": 0.00043
|
||||
},
|
||||
"Recall": {
|
||||
"Recall@1": 0.0,
|
||||
"Recall@3": 0.0,
|
||||
"Recall@5": 0.0,
|
||||
"Recall@10": 0.0,
|
||||
"Recall@100": 0.01
|
||||
},
|
||||
"Precision": {
|
||||
"P@1": 0.0,
|
||||
"P@3": 0.0,
|
||||
"P@5": 0.0,
|
||||
"P@10": 0.0,
|
||||
"P@100": 0.0001
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,333 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "66cbbaf8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "c01c19dc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import Dict, List, Union\n",
|
||||
"import numpy as np\n",
|
||||
"from langchain_ollama import OllamaEmbeddings\n",
|
||||
"from beir.datasets.data_loader import GenericDataLoader\n",
|
||||
"from beir.retrieval.search.dense import DenseRetrievalExactSearch\n",
|
||||
"from beir.retrieval.evaluation import EvaluateRetrieval\n",
|
||||
"from beir import util\n",
|
||||
"import json\n",
|
||||
"from datasets import load_dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ac011c1c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Utils"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "b83e7900",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class BEIROllamaEmbeddings:\n",
|
||||
" \"\"\"\n",
|
||||
" Adapter that makes LangChain's OllamaEmbeddings compatible with BEIR.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(\n",
|
||||
" self,\n",
|
||||
" base_url: str,\n",
|
||||
" model: str,\n",
|
||||
" batch_size: int = 64,\n",
|
||||
" ) -> None:\n",
|
||||
" self.batch_size = batch_size\n",
|
||||
" self.embeddings = OllamaEmbeddings(\n",
|
||||
" base_url=base_url,\n",
|
||||
" model=model,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" def _batch_embed(self, texts: List[str]) -> np.ndarray:\n",
|
||||
" vectors = []\n",
|
||||
"\n",
|
||||
" for i in range(0, len(texts), self.batch_size):\n",
|
||||
" batch = texts[i : i + self.batch_size]\n",
|
||||
" batch_vectors = self.embeddings.embed_documents(batch)\n",
|
||||
" vectors.extend(batch_vectors)\n",
|
||||
"\n",
|
||||
" return np.asarray(vectors, dtype=np.float32)\n",
|
||||
"\n",
|
||||
" def encode_queries(self, queries: List[str], **kwargs) -> np.ndarray:\n",
|
||||
" \"\"\"\n",
|
||||
" BEIR query encoder\n",
|
||||
" \"\"\"\n",
|
||||
" return self._batch_embed(queries)\n",
|
||||
"\n",
|
||||
" def encode_corpus(\n",
|
||||
" self,\n",
|
||||
" corpus: Union[List[Dict[str, str]], Dict[str, Dict[str, str]]],\n",
|
||||
" **kwargs,\n",
|
||||
" ) -> np.ndarray:\n",
|
||||
" \"\"\"\n",
|
||||
" BEIR corpus encoder\n",
|
||||
" \"\"\"\n",
|
||||
" if isinstance(corpus, dict):\n",
|
||||
" corpus = list(corpus.values())\n",
|
||||
"\n",
|
||||
" texts = []\n",
|
||||
" for doc in corpus:\n",
|
||||
" title = (doc.get(\"title\") or \"\").strip()\n",
|
||||
" text = (doc.get(\"text\") or \"\").strip()\n",
|
||||
"\n",
|
||||
" if title:\n",
|
||||
" texts.append(f\"{title}\\n{text}\")\n",
|
||||
" else:\n",
|
||||
" texts.append(text)\n",
|
||||
"\n",
|
||||
" return self._batch_embed(texts)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "af3eb66d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def convert_hf_to_beir(hf_dataset):\n",
|
||||
" corpus, queries, qrels = {}, {}, {}\n",
|
||||
" \n",
|
||||
" for i, data in enumerate(hf_dataset):\n",
|
||||
" docid = f\"doc_{i}\"\n",
|
||||
" queryid = f\"q_{i}\"\n",
|
||||
" \n",
|
||||
" # El código es el documento (lo que el agente debe recuperar)\n",
|
||||
" corpus[docid] = {\"title\": data.get(\"func_name\", \"\"), \"text\": data['code']}\n",
|
||||
" \n",
|
||||
" # El docstring es la consulta (lo que el usuario pide)\n",
|
||||
" queries[queryid] = data['docstring']\n",
|
||||
" \n",
|
||||
" # Relación 1 a 1: la query i busca el código i\n",
|
||||
" qrels[queryid] = {docid: 1}\n",
|
||||
" \n",
|
||||
" return corpus, queries, qrels"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c9528fb6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "230aae25",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"raw_dataset = load_dataset(\"google/code_x_glue_tc_nl_code_search_adv\", split=\"test\")\n",
|
||||
"corpus, queries, qrels = convert_hf_to_beir(raw_dataset)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "13050d31",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test qwen3-0.6B-emb:latest"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "514540af",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = BEIROllamaEmbeddings(\n",
|
||||
" base_url=\"http://localhost:11434\",\n",
|
||||
" model=\"qwen3-0.6B-emb:latest\",\n",
|
||||
" batch_size=64,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Inicializar buscador y evaluador\n",
|
||||
"retriever = DenseRetrievalExactSearch(model, batch_size=64)\n",
|
||||
"evaluator = EvaluateRetrieval(retriever, score_function=\"cos_sim\")\n",
|
||||
"\n",
|
||||
"# Ejecutar recuperación\n",
|
||||
"results = evaluator.retrieve(corpus, queries)\n",
|
||||
"\n",
|
||||
"# Evaluar métricas (NDCG, MAP, Recall, Precision)\n",
|
||||
"ndcg, _map, recall, precision = evaluator.evaluate(\n",
|
||||
" qrels, results, [1, 3, 5, 10]\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "5c0f9845",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Resultados para CodeXGLUE:\n",
|
||||
"NDCG: {'NDCG@1': 0.94971, 'NDCG@3': 0.96956, 'NDCG@5': 0.97166, 'NDCG@10': 0.97342}\n",
|
||||
"MAP: {'MAP@1': 0.94971, 'MAP@3': 0.96504, 'MAP@5': 0.9662, 'MAP@10': 0.96694}\n",
|
||||
"Recall: {'Recall@1': 0.94971, 'Recall@3': 0.98251, 'Recall@5': 0.98761, 'Recall@10': 0.99297}\n",
|
||||
"Precision: {'P@1': 0.94971, 'P@3': 0.3275, 'P@5': 0.19752, 'P@10': 0.0993}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(f\"Resultados para CodeXGLUE:\")\n",
|
||||
"print(\"NDCG:\", ndcg)\n",
|
||||
"print(\"MAP:\", _map)\n",
|
||||
"print(\"Recall:\", recall)\n",
|
||||
"print(\"Precision:\", precision)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c4e643ca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test qwen2.5:1.5b"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "5ced1c25",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_q2 = BEIROllamaEmbeddings(\n",
|
||||
" base_url=\"http://localhost:11434\",\n",
|
||||
" model=\"qwen2.5:1.5b\",\n",
|
||||
" batch_size=64,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Inicializar buscador y evaluador\n",
|
||||
"retriever_q2 = DenseRetrievalExactSearch(model_q2, batch_size=64)\n",
|
||||
"evaluator_q2 = EvaluateRetrieval(retriever_q2, score_function=\"cos_sim\")\n",
|
||||
"\n",
|
||||
"# Ejecutar recuperación\n",
|
||||
"results_q2 = evaluator_q2.retrieve(corpus, queries)\n",
|
||||
"\n",
|
||||
"# Evaluar métricas (NDCG, MAP, Recall, Precision)\n",
|
||||
"ndcg_qwen_2, _map_qwen_2, recall_qwen_2, precision_qwen_2 = evaluator_q2.evaluate(\n",
|
||||
" qrels, results_q2, [1, 3, 5, 10]\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "6a95189e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Resultados para CodeXGLUE:\n",
|
||||
"NDCG: {'NDCG@1': 0.00031, 'NDCG@3': 0.00061, 'NDCG@5': 0.00086, 'NDCG@10': 0.00118}\n",
|
||||
"MAP: {'MAP@1': 0.00031, 'MAP@3': 0.00051, 'MAP@5': 0.00065, 'MAP@10': 0.00078}\n",
|
||||
"Recall: {'Recall@1': 0.00031, 'Recall@3': 0.00088, 'Recall@5': 0.00151, 'Recall@10': 0.0025}\n",
|
||||
"Precision: {'P@1': 0.00031, 'P@3': 0.00029, 'P@5': 0.0003, 'P@10': 0.00025}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(f\"Resultados para CodeXGLUE:\")\n",
|
||||
"print(\"NDCG:\", ndcg_qwen_2)\n",
|
||||
"print(\"MAP:\", _map_qwen_2)\n",
|
||||
"print(\"Recall:\", recall_qwen_2)\n",
|
||||
"print(\"Precision:\", precision_qwen_2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3dad9811",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Save data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "f875dd8d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Resultados guardados en /home/pseco/VsCodeProjects/assistance-engine/data/interim/beir_CodeXGlue_results.json\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results_data = {\n",
|
||||
" \"qwen3-0.6B-emb:latest\": {\n",
|
||||
" \"NDCG\": ndcg,\n",
|
||||
" \"MAP\": _map,\n",
|
||||
" \"Recall\": recall,\n",
|
||||
" \"Precision\": precision,\n",
|
||||
" },\n",
|
||||
" \"qwen2.5:1.5b\": {\n",
|
||||
" \"NDCG\": ndcg_qwen_2,\n",
|
||||
" \"MAP\": _map_qwen_2,\n",
|
||||
" \"Recall\": recall_qwen_2,\n",
|
||||
" \"Precision\": precision_qwen_2,\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"output_file = \"/home/pseco/VsCodeProjects/assistance-engine/data/interim/beir_CodeXGlue_results.json\"\n",
|
||||
"with open(output_file, \"w\") as f:\n",
|
||||
" json.dump(results_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"print(f\"Resultados guardados en {output_file}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "assistance-engine",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
|
|
@ -0,0 +1,323 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "66cbbaf8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "c01c19dc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import Dict, List, Union\n",
|
||||
"import numpy as np\n",
|
||||
"from langchain_ollama import OllamaEmbeddings\n",
|
||||
"from beir.datasets.data_loader import GenericDataLoader\n",
|
||||
"from beir.retrieval.search.dense import DenseRetrievalExactSearch\n",
|
||||
"from beir.retrieval.evaluation import EvaluateRetrieval\n",
|
||||
"from beir import util\n",
|
||||
"import json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ac011c1c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Utils"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "b83e7900",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class BEIROllamaEmbeddings:\n",
|
||||
" \"\"\"\n",
|
||||
" Adapter that makes LangChain's OllamaEmbeddings compatible with BEIR.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(\n",
|
||||
" self,\n",
|
||||
" base_url: str,\n",
|
||||
" model: str,\n",
|
||||
" batch_size: int = 64,\n",
|
||||
" ) -> None:\n",
|
||||
" self.batch_size = batch_size\n",
|
||||
" self.embeddings = OllamaEmbeddings(\n",
|
||||
" base_url=base_url,\n",
|
||||
" model=model,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" def _batch_embed(self, texts: List[str]) -> np.ndarray:\n",
|
||||
" vectors = []\n",
|
||||
"\n",
|
||||
" for i in range(0, len(texts), self.batch_size):\n",
|
||||
" batch = texts[i : i + self.batch_size]\n",
|
||||
" batch_vectors = self.embeddings.embed_documents(batch)\n",
|
||||
" vectors.extend(batch_vectors)\n",
|
||||
"\n",
|
||||
" return np.asarray(vectors, dtype=np.float32)\n",
|
||||
"\n",
|
||||
" def encode_queries(self, queries: List[str], **kwargs) -> np.ndarray:\n",
|
||||
" \"\"\"\n",
|
||||
" BEIR query encoder\n",
|
||||
" \"\"\"\n",
|
||||
" return self._batch_embed(queries)\n",
|
||||
"\n",
|
||||
" def encode_corpus(\n",
|
||||
" self,\n",
|
||||
" corpus: Union[List[Dict[str, str]], Dict[str, Dict[str, str]]],\n",
|
||||
" **kwargs,\n",
|
||||
" ) -> np.ndarray:\n",
|
||||
" \"\"\"\n",
|
||||
" BEIR corpus encoder\n",
|
||||
" \"\"\"\n",
|
||||
" if isinstance(corpus, dict):\n",
|
||||
" corpus = list(corpus.values())\n",
|
||||
"\n",
|
||||
" texts = []\n",
|
||||
" for doc in corpus:\n",
|
||||
" title = (doc.get(\"title\") or \"\").strip()\n",
|
||||
" text = (doc.get(\"text\") or \"\").strip()\n",
|
||||
"\n",
|
||||
" if title:\n",
|
||||
" texts.append(f\"{title}\\n{text}\")\n",
|
||||
" else:\n",
|
||||
" texts.append(text)\n",
|
||||
"\n",
|
||||
" return self._batch_embed(texts)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "af3eb66d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def convert_codexglue_to_beir(input_file):\n",
|
||||
" corpus, queries, qrels = {}, {}, {}\n",
|
||||
" with open(input_file, 'r') as f:\n",
|
||||
" for i, line in enumerate(f):\n",
|
||||
" data = json.loads(line)\n",
|
||||
" docid = f\"doc_{i}\"\n",
|
||||
" queryid = f\"q_{i}\"\n",
|
||||
" \n",
|
||||
" # El código es nuestro documento (Corpus)\n",
|
||||
" corpus[docid] = {\"title\": \"\", \"text\": data['code']}\n",
|
||||
" # El docstring es nuestra consulta (Query)\n",
|
||||
" queries[queryid] = data['docstring']\n",
|
||||
" # En CodeXGLUE, la consulta i corresponde al código i\n",
|
||||
" qrels[queryid] = {docid: 1}\n",
|
||||
" \n",
|
||||
" return corpus, queries, qrels\n",
|
||||
"\n",
|
||||
"# Carga tus datos (ejemplo con el set de test de AdvTest)\n",
|
||||
"corpus, queries, qrels = convert_codexglue_to_beir(\"test.jsonl\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c9528fb6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "230aae25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1915c67ec20f4806b30b48eff9a132e2",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/5183 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"dataset=\"scifact\"\n",
|
||||
"url=f\"https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{dataset}.zip\"\n",
|
||||
"data_path=util.download_and_unzip(url, out_dir=\"datasets\")\n",
|
||||
"corpus, queries, qrels=GenericDataLoader(data_path).load(split=\"test\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "13050d31",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test qwen3-0.6B-emb:latest"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "514540af",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"NDCG: {'NDCG@1': 0.56333, 'NDCG@3': 0.64367, 'NDCG@5': 0.66577, 'NDCG@10': 0.68551, 'NDCG@100': 0.71285}\n",
|
||||
"MAP: {'MAP@1': 0.52994, 'MAP@3': 0.6117, 'MAP@5': 0.62815, 'MAP@10': 0.6383, 'MAP@100': 0.64466}\n",
|
||||
"Recall: {'Recall@1': 0.52994, 'Recall@3': 0.7035, 'Recall@5': 0.75967, 'Recall@10': 0.81611, 'Recall@100': 0.94}\n",
|
||||
"Precision: {'P@1': 0.56333, 'P@3': 0.25889, 'P@5': 0.17067, 'P@10': 0.093, 'P@100': 0.0107}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = BEIROllamaEmbeddings(\n",
|
||||
" base_url=\"http://localhost:11434\",\n",
|
||||
" model=\"qwen3-0.6B-emb:latest\",\n",
|
||||
" batch_size=64,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"retriever = DenseRetrievalExactSearch(model, batch_size=64)\n",
|
||||
"evaluator = EvaluateRetrieval(retriever, score_function=\"cos_sim\")\n",
|
||||
"\n",
|
||||
"results = evaluator.retrieve(corpus, queries)\n",
|
||||
"ndcg, _map, recall, precision = evaluator.evaluate(\n",
|
||||
" qrels, results, [1, 3, 5, 10, 100]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"NDCG:\", ndcg)\n",
|
||||
"print(\"MAP:\", _map)\n",
|
||||
"print(\"Recall:\", recall)\n",
|
||||
"print(\"Precision:\", precision)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c4e643ca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test qwen2.5:1.5b"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "5ced1c25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"NDCG: {'NDCG@1': 0.02333, 'NDCG@3': 0.03498, 'NDCG@5': 0.0404, 'NDCG@10': 0.04619, 'NDCG@100': 0.07768}\n",
|
||||
"MAP: {'MAP@1': 0.02083, 'MAP@3': 0.03083, 'MAP@5': 0.03375, 'MAP@10': 0.03632, 'MAP@100': 0.04123}\n",
|
||||
"Recall: {'Recall@1': 0.02083, 'Recall@3': 0.04417, 'Recall@5': 0.0575, 'Recall@10': 0.07417, 'Recall@100': 0.23144}\n",
|
||||
"Precision: {'P@1': 0.02333, 'P@3': 0.01556, 'P@5': 0.01267, 'P@10': 0.00833, 'P@100': 0.00277}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model_qwen2 = BEIROllamaEmbeddings(\n",
|
||||
" base_url=\"http://localhost:11434\",\n",
|
||||
" model=\"qwen2.5:1.5b\",\n",
|
||||
" batch_size=64,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"retriever_qwen_2 = DenseRetrievalExactSearch(model_qwen2, batch_size=64)\n",
|
||||
"evaluator_qwen_2 = EvaluateRetrieval(retriever_qwen_2, score_function=\"cos_sim\")\n",
|
||||
"\n",
|
||||
"results_qwen_2 = evaluator_qwen_2.retrieve(corpus, queries)\n",
|
||||
"ndcg_qwen_2, _map_qwen_2, recall_qwen_2, precision_qwen_2 = evaluator_qwen_2.evaluate(\n",
|
||||
" qrels, results_qwen_2, [1, 3, 5, 10, 100]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"NDCG:\", ndcg_qwen_2)\n",
|
||||
"print(\"MAP:\", _map_qwen_2)\n",
|
||||
"print(\"Recall:\", recall_qwen_2)\n",
|
||||
"print(\"Precision:\", precision_qwen_2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b9402837",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Save Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "c281d5e1",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Resultados guardados en /home/pseco/VsCodeProjects/assistance-engine/data/interim/beir_Scifact_results.json\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results_data = {\n",
|
||||
" \"qwen3-0.6B-emb:latest\": {\n",
|
||||
" \"NDCG\": ndcg,\n",
|
||||
" \"MAP\": _map,\n",
|
||||
" \"Recall\": recall,\n",
|
||||
" \"Precision\": precision,\n",
|
||||
" },\n",
|
||||
" \"qwen2.5:1.5b\": {\n",
|
||||
" \"NDCG\": ndcg_qwen_2,\n",
|
||||
" \"MAP\": _map_qwen_2,\n",
|
||||
" \"Recall\": recall_qwen_2,\n",
|
||||
" \"Precision\": precision_qwen_2,\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"output_file = \"/home/pseco/VsCodeProjects/assistance-engine/data/interim/beir_Scifact_results.json\"\n",
|
||||
"with open(output_file, \"w\") as f:\n",
|
||||
" json.dump(results_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"print(f\"Resultados guardados en {output_file}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "assistance-engine",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
|
|
@ -0,0 +1,335 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "66cbbaf8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "c01c19dc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"from typing import Dict, List, Union\n",
|
||||
"import numpy as np\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from langchain_ollama import OllamaEmbeddings\n",
|
||||
"from beir.datasets.data_loader import GenericDataLoader\n",
|
||||
"from beir.retrieval.search.dense import DenseRetrievalExactSearch\n",
|
||||
"from beir.retrieval.evaluation import EvaluateRetrieval\n",
|
||||
"from beir import util"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ac011c1c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Utils"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "b83e7900",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class BEIROllamaEmbeddings:\n",
|
||||
" \"\"\"\n",
|
||||
" Adapter that makes LangChain's OllamaEmbeddings compatible with BEIR.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(\n",
|
||||
" self,\n",
|
||||
" base_url: str,\n",
|
||||
" model: str,\n",
|
||||
" batch_size: int = 64,\n",
|
||||
" ) -> None:\n",
|
||||
" self.batch_size = batch_size\n",
|
||||
" self.embeddings = OllamaEmbeddings(\n",
|
||||
" base_url=base_url,\n",
|
||||
" model=model,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" def _batch_embed(self, texts: List[str]) -> np.ndarray:\n",
|
||||
" vectors = []\n",
|
||||
"\n",
|
||||
" for i in range(0, len(texts), self.batch_size):\n",
|
||||
" batch = texts[i : i + self.batch_size]\n",
|
||||
" batch_vectors = self.embeddings.embed_documents(batch)\n",
|
||||
" vectors.extend(batch_vectors)\n",
|
||||
"\n",
|
||||
" return np.asarray(vectors, dtype=np.float32)\n",
|
||||
"\n",
|
||||
" def encode_queries(self, queries: List[str], **kwargs) -> np.ndarray:\n",
|
||||
" \"\"\"\n",
|
||||
" BEIR query encoder\n",
|
||||
" \"\"\"\n",
|
||||
" return self._batch_embed(queries)\n",
|
||||
"\n",
|
||||
" def encode_corpus(\n",
|
||||
" self,\n",
|
||||
" corpus: Union[List[Dict[str, str]], Dict[str, Dict[str, str]]],\n",
|
||||
" **kwargs,\n",
|
||||
" ) -> np.ndarray:\n",
|
||||
" \"\"\"\n",
|
||||
" BEIR corpus encoder\n",
|
||||
" \"\"\"\n",
|
||||
" if isinstance(corpus, dict):\n",
|
||||
" corpus = list(corpus.values())\n",
|
||||
"\n",
|
||||
" texts = []\n",
|
||||
" for doc in corpus:\n",
|
||||
" title = (doc.get(\"title\") or \"\").strip()\n",
|
||||
" text = (doc.get(\"text\") or \"\").strip()\n",
|
||||
"\n",
|
||||
" if title:\n",
|
||||
" texts.append(f\"{title}\\n{text}\")\n",
|
||||
" else:\n",
|
||||
" texts.append(text)\n",
|
||||
"\n",
|
||||
" return self._batch_embed(texts)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c9528fb6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "230aae25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Descargando datos de Hugging Face...\n",
|
||||
"Cargando con BEIR GenericDataLoader...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "0e67479e959248f598db3415efbb13ae",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/20604 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"dataset_name = \"cosqa\"\n",
|
||||
"data_path = f\"/home/acano/PycharmProjects/assistance-engine/data/external/{dataset_name}\"\n",
|
||||
"\n",
|
||||
"os.makedirs(f\"{data_path}/qrels\", exist_ok=True)\n",
|
||||
"\n",
|
||||
"# 1. Cargar desde Hugging Face con los nombres de configuración correctos\n",
|
||||
"print(\"Descargando datos de Hugging Face...\")\n",
|
||||
"hf_corpus = load_dataset(\"CoIR-Retrieval/cosqa\", \"corpus\", split=\"corpus\")\n",
|
||||
"hf_queries = load_dataset(\"CoIR-Retrieval/cosqa\", \"queries\", split=\"queries\")\n",
|
||||
"# Los qrels están en la config 'default'\n",
|
||||
"hf_qrels = load_dataset(\"CoIR-Retrieval/cosqa\", \"default\", split=\"test\")\n",
|
||||
"\n",
|
||||
"# 2. Guardar Corpus\n",
|
||||
"with open(f\"{data_path}/corpus.jsonl\", \"w\") as f:\n",
|
||||
" for item in hf_corpus:\n",
|
||||
" f.write(json.dumps({\"_id\": str(item[\"_id\"]), \"text\": item[\"text\"], \"title\": \"\"}) + \"\\n\")\n",
|
||||
"\n",
|
||||
"# 3. Guardar Queries\n",
|
||||
"with open(f\"{data_path}/queries.jsonl\", \"w\") as f:\n",
|
||||
" for item in hf_queries:\n",
|
||||
" f.write(json.dumps({\"_id\": str(item[\"_id\"]), \"text\": item[\"text\"]}) + \"\\n\")\n",
|
||||
"\n",
|
||||
"# 4. Guardar Qrels (Formato TSV para BEIR)\n",
|
||||
"with open(f\"{data_path}/qrels/test.tsv\", \"w\") as f:\n",
|
||||
" f.write(\"query-id\\tcorpus-id\\tscore\\n\")\n",
|
||||
" for item in hf_qrels:\n",
|
||||
" # En la config 'default', los campos suelen ser 'query-id' y 'corpus-id'\n",
|
||||
" f.write(f\"{item['query-id']}\\t{item['corpus-id']}\\t{item['score']}\\n\")\n",
|
||||
"\n",
|
||||
"print(\"Cargando con BEIR GenericDataLoader...\")\n",
|
||||
"corpus, queries, qrels = GenericDataLoader(data_path).load(split=\"test\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "13050d31",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test qwen3-0.6B-emb:latest"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "514540af",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"NDCG: {'NDCG@1': 0.174, 'NDCG@3': 0.27374, 'NDCG@5': 0.33509, 'NDCG@10': 0.39086, 'NDCG@100': 0.45099}\n",
|
||||
"MAP: {'MAP@1': 0.174, 'MAP@3': 0.247, 'MAP@5': 0.2808, 'MAP@10': 0.30466, 'MAP@100': 0.31702}\n",
|
||||
"Recall: {'Recall@1': 0.174, 'Recall@3': 0.352, 'Recall@5': 0.502, 'Recall@10': 0.67, 'Recall@100': 0.952}\n",
|
||||
"Precision: {'P@1': 0.174, 'P@3': 0.11733, 'P@5': 0.1004, 'P@10': 0.067, 'P@100': 0.00952}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = BEIROllamaEmbeddings(\n",
|
||||
" base_url=\"http://localhost:11434\",\n",
|
||||
" model=\"qwen3-0.6B-emb:latest\",\n",
|
||||
" batch_size=64,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"retriever = DenseRetrievalExactSearch(model, batch_size=64)\n",
|
||||
"evaluator = EvaluateRetrieval(retriever, score_function=\"cos_sim\")\n",
|
||||
"\n",
|
||||
"results = evaluator.retrieve(corpus, queries)\n",
|
||||
"ndcg, _map, recall, precision = evaluator.evaluate(\n",
|
||||
" qrels, results, [1, 3, 5, 10, 100]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"NDCG:\", ndcg)\n",
|
||||
"print(\"MAP:\", _map)\n",
|
||||
"print(\"Recall:\", recall)\n",
|
||||
"print(\"Precision:\", precision)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c4e643ca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test qwen2.5:1.5b"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "5ced1c25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"NDCG: {'NDCG@1': 0.0, 'NDCG@3': 0.0, 'NDCG@5': 0.0, 'NDCG@10': 0.0, 'NDCG@100': 0.0021}\n",
|
||||
"MAP: {'MAP@1': 0.0, 'MAP@3': 0.0, 'MAP@5': 0.0, 'MAP@10': 0.0, 'MAP@100': 0.00043}\n",
|
||||
"Recall: {'Recall@1': 0.0, 'Recall@3': 0.0, 'Recall@5': 0.0, 'Recall@10': 0.0, 'Recall@100': 0.01}\n",
|
||||
"Precision: {'P@1': 0.0, 'P@3': 0.0, 'P@5': 0.0, 'P@10': 0.0, 'P@100': 0.0001}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model_qwen2 = BEIROllamaEmbeddings(\n",
|
||||
" base_url=\"http://localhost:11434\",\n",
|
||||
" model=\"qwen2.5:1.5b\",\n",
|
||||
" batch_size=64,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"retriever_qwen_2 = DenseRetrievalExactSearch(model_qwen2, batch_size=64)\n",
|
||||
"evaluator_qwen_2 = EvaluateRetrieval(retriever_qwen_2, score_function=\"cos_sim\")\n",
|
||||
"\n",
|
||||
"results_qwen_2 = evaluator_qwen_2.retrieve(corpus, queries)\n",
|
||||
"ndcg_qwen_2, _map_qwen_2, recall_qwen_2, precision_qwen_2 = evaluator_qwen_2.evaluate(\n",
|
||||
" qrels, results_qwen_2, [1, 3, 5, 10, 100]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"NDCG:\", ndcg_qwen_2)\n",
|
||||
"print(\"MAP:\", _map_qwen_2)\n",
|
||||
"print(\"Recall:\", recall_qwen_2)\n",
|
||||
"print(\"Precision:\", precision_qwen_2)"
|
||||
]
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "1db7d110",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Resultados guardados en /home/acano/PycharmProjects/assistance-engine/data/interim/beir_cosqa_results.json\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results_data = {\n",
|
||||
" \"qwen3-0.6B-emb:latest\": {\n",
|
||||
" \"NDCG\": ndcg,\n",
|
||||
" \"MAP\": _map,\n",
|
||||
" \"Recall\": recall,\n",
|
||||
" \"Precision\": precision,\n",
|
||||
" },\n",
|
||||
" \"qwen2.5:1.5b\": {\n",
|
||||
" \"NDCG\": ndcg_qwen_2,\n",
|
||||
" \"MAP\": _map_qwen_2,\n",
|
||||
" \"Recall\": recall_qwen_2,\n",
|
||||
" \"Precision\": precision_qwen_2,\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
" \n",
|
||||
"output_file = \"/home/acano/PycharmProjects/assistance-engine/data/interim/beir_cosqa_results.json\"\n",
|
||||
"with open(output_file, \"w\") as f:\n",
|
||||
" json.dump(results_data, f, indent=2)\n",
|
||||
" \n",
|
||||
"print(f\"Resultados guardados en {output_file}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e4f8d78b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
>>>>>>> 4b5352d93cf89b7562895b550fb5bd62160586c5
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "assistance-engine",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.13"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
|
|
@ -0,0 +1,289 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "096e6224",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4b0853e9",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/tmp/ipykernel_2931729/1845255288.py:4: DeprecationWarning: Importing SemanticSimilarity from 'ragas.metrics' is deprecated and will be removed in v1.0. Please use 'ragas.metrics.collections' instead. Example: from ragas.metrics.collections import SemanticSimilarity\n",
|
||||
" from ragas.metrics import SemanticSimilarity\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# ...existing code...\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from ragas import EvaluationDataset, evaluate\n",
|
||||
"from ragas.metrics import SemanticSimilarity\n",
|
||||
"from langchain_community.embeddings import OllamaEmbeddings\n",
|
||||
"import asyncio\n",
|
||||
"from typing import Sequence\n",
|
||||
"from ragas.embeddings.base import BaseRagasEmbedding\n",
|
||||
"import os\n",
|
||||
"from transformers import AutoConfig\n",
|
||||
"import nltk\n",
|
||||
"# ...existing code..."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "6bfe1ca0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"nltk.download(\"punkt\", quiet=True)\n",
|
||||
"\n",
|
||||
"ES_URL = os.getenv(\"ELASTICSEARCH_LOCAL_URL\")\n",
|
||||
"ES_INDEX_NAME = os.getenv(\"ELASTICSEARCH_INDEX\")\n",
|
||||
"HF_EMBEDDING_MODEL_NAME = os.getenv(\"HF_EMBEDDING_MODEL_NAME\")\n",
|
||||
"BASE_URL = os.getenv(\"LLM_BASE_LOCAL_URL\")\n",
|
||||
"MODEL_NAME = os.getenv(\"OLLAMA_MODEL_NAME\")\n",
|
||||
"\n",
|
||||
"config = AutoConfig.from_pretrained(HF_EMBEDDING_MODEL_NAME)\n",
|
||||
"embedding_dim = config.hidden_size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "ea41ce0f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/tmp/ipykernel_2931729/256987240.py:1: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the `langchain-ollama package and should be used instead. To use it run `pip install -U `langchain-ollama` and import as `from `langchain_ollama import OllamaEmbeddings``.\n",
|
||||
" embeddings = OllamaEmbeddings(base_url=BASE_URL, model=MODEL_NAME)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"embeddings = OllamaEmbeddings(base_url=BASE_URL, model=MODEL_NAME)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8eee9390",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Similitud Aleatoria"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d7b150e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datasets import load_dataset\n",
|
||||
"from ragas import EvaluationDataset\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _normalize_answer(answer_value: object) -> str:\n",
|
||||
" \"\"\"\n",
|
||||
" Normalize answer values to a single string.\n",
|
||||
" \"\"\"\n",
|
||||
" if isinstance(answer_value, dict):\n",
|
||||
" text_value = answer_value.get(\"text\")\n",
|
||||
" if isinstance(text_value, list):\n",
|
||||
" return str(text_value[0]) if text_value else \"\"\n",
|
||||
" if text_value is not None:\n",
|
||||
" return str(text_value)\n",
|
||||
"\n",
|
||||
" if isinstance(answer_value, list):\n",
|
||||
" return str(answer_value[0]) if answer_value else \"\"\n",
|
||||
"\n",
|
||||
" return str(answer_value)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _first_existing_key(candidates: list[str], keys: set[str]) -> str | None:\n",
|
||||
" \"\"\"\n",
|
||||
" Return the first key present in keys from candidates.\n",
|
||||
" \"\"\"\n",
|
||||
" for candidate in candidates:\n",
|
||||
" if candidate in keys:\n",
|
||||
" return candidate\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"ds = load_dataset(\"sentence-transformers/natural-questions\")\n",
|
||||
"train_ds = ds[\"train\"]\n",
|
||||
"\n",
|
||||
"max_questions = min(100, len(train_ds))\n",
|
||||
"train_ds = train_ds.select(range(max_questions))\n",
|
||||
"\n",
|
||||
"available_keys = set(train_ds.column_names)\n",
|
||||
"reference_key = _first_existing_key(\n",
|
||||
" [\"question\", \"query\", \"text\", \"input\"], available_keys\n",
|
||||
")\n",
|
||||
"response_key = _first_existing_key(\n",
|
||||
" [\"answer\", \"answers\", \"response\", \"output\"], available_keys\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if reference_key is None or response_key is None:\n",
|
||||
" raise KeyError(\n",
|
||||
" f\"Expected question/answer-like columns not found. \"\n",
|
||||
" f\"Available columns: {train_ds.column_names}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"rows = []\n",
|
||||
"for row in train_ds:\n",
|
||||
" rows.append(\n",
|
||||
" {\n",
|
||||
" \"reference\": str(row[reference_key]),\n",
|
||||
" \"response\": _normalize_answer(row[response_key]),\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_ds = EvaluationDataset.from_list(rows)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "753aab30",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"DatasetDict({\n",
|
||||
" train: Dataset({\n",
|
||||
" features: ['query', 'answer'],\n",
|
||||
" num_rows: 100231\n",
|
||||
" })\n",
|
||||
"})\n",
|
||||
"['query', 'answer']\n",
|
||||
"{'query': 'when did richmond last play in a preliminary final', 'answer': \"Richmond Football Club Richmond began 2017 with 5 straight wins, a feat it had not achieved since 1995. A series of close losses hampered the Tigers throughout the middle of the season, including a 5-point loss to the Western Bulldogs, 2-point loss to Fremantle, and a 3-point loss to the Giants. Richmond ended the season strongly with convincing victories over Fremantle and St Kilda in the final two rounds, elevating the club to 3rd on the ladder. Richmond's first final of the season against the Cats at the MCG attracted a record qualifying final crowd of 95,028; the Tigers won by 51 points. Having advanced to the first preliminary finals for the first time since 2001, Richmond defeated Greater Western Sydney by 36 points in front of a crowd of 94,258 to progress to the Grand Final against Adelaide, their first Grand Final appearance since 1982. The attendance was 100,021, the largest crowd to a grand final since 1986. The Crows led at quarter time and led by as many as 13, but the Tigers took over the game as it progressed and scored seven straight goals at one point. They eventually would win by 48 points – 16.12 (108) to Adelaide's 8.12 (60) – to end their 37-year flag drought.[22] Dustin Martin also became the first player to win a Premiership medal, the Brownlow Medal and the Norm Smith Medal in the same season, while Damien Hardwick was named AFL Coaches Association Coach of the Year. Richmond's jump from 13th to premiers also marked the biggest jump from one AFL season to the next.\"}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(ds)\n",
|
||||
"print(ds[\"train\"].column_names)\n",
|
||||
"print(ds[\"train\"][0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "6c3d4235",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ...existing code...\n",
|
||||
"class OllamaRagasEmbeddingAdapter(BaseRagasEmbedding):\n",
|
||||
" \"\"\"Adaptador de LangChain Ollama a la API moderna de embeddings en Ragas.\"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, base_url: str, model_name: str) -> None:\n",
|
||||
" self._client = OllamaEmbeddings(base_url=base_url, model=model_name)\n",
|
||||
"\n",
|
||||
" def embed_text(self, text: str) -> list[float]:\n",
|
||||
" return self._client.embed_query(text)\n",
|
||||
"\n",
|
||||
" async def aembed_text(self, text: str) -> list[float]:\n",
|
||||
" return await asyncio.to_thread(self.embed_text, text)\n",
|
||||
"\n",
|
||||
" def embed_query(self, text: str) -> list[float]:\n",
|
||||
" return self.embed_text(text)\n",
|
||||
"\n",
|
||||
" def embed_documents(self, texts: Sequence[str]) -> list[list[float]]:\n",
|
||||
" return self._client.embed_documents(list(texts))\n",
|
||||
"\n",
|
||||
" async def aembed_query(self, text: str) -> list[float]:\n",
|
||||
" return await self.aembed_text(text)\n",
|
||||
"\n",
|
||||
" async def aembed_documents(\n",
|
||||
" self, texts: Sequence[str]\n",
|
||||
" ) -> list[list[float]]:\n",
|
||||
" return await asyncio.to_thread(self.embed_documents, texts)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if not BASE_URL or not MODEL_NAME:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Faltan variables de entorno: LLM_BASE_LOCAL_URL u OLLAMA_MODEL_NAME.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"embeddings = OllamaRagasEmbeddingAdapter(\n",
|
||||
" base_url=BASE_URL,\n",
|
||||
" model_name=MODEL_NAME,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"semantic_sim = SemanticSimilarity()\n",
|
||||
"# ...existing code..."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "54aacf01",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "6a4b6e91c71d4849922f36d45f3e9f7f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Evaluating: 0%| | 0/100231 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# ...existing code...\n",
|
||||
"result = evaluate(\n",
|
||||
" dataset=eval_ds,\n",
|
||||
" metrics=[semantic_sim],\n",
|
||||
" embeddings=embeddings,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(result)\n",
|
||||
"# ...existing code...\n",
|
||||
"# ...existing code..."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "assistance-engine",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Loading…
Reference in New Issue