Merge branch 'mrh-online-dev' of github.com:BRUNIX-AI/assistance-engine into mrh-online-dev

scratches/pseco/evaluation/embeddings/n00 first Analysis.ipynb

@@ -0,0 +1,116 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "096e6224",
+   "metadata": {},
+   "source": [
+    "# Libraries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b1970d59",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core import download_loader\n",
+    "from ragas.testset.evolutions import simple, reasoning, multi_context\n",
+    "from ragas.testset.generator import TestsetGenerator\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from ragas.embeddings import OpenAIEmbeddings\n",
+    "import openai\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6bfe1ca0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import re\n",
+    "import uuid\n",
+    "from pathlib import Path\n",
+    "from typing import Any, Protocol\n",
+    "from datasets import load_dataset\n",
+    "from langchain_core.documents import Document\n",
+    "from langchain_elasticsearch import ElasticsearchStore\n",
+    "import torch\n",
+    "import torch.nn.functional as F\n",
+    "from loguru import logger\n",
+    "from langchain_ollama import OllamaEmbeddings\n",
+    "from ragas.metrics.collections import SemanticSimilarity  \n",
+    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
+    "from elasticsearch import Elasticsearch\n",
+    "from langchain_elasticsearch import ElasticsearchStore\n",
+    "import nltk\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "nltk.download(\"punkt\", quiet=True)\n",
+    "\n",
+    "ES_URL = os.getenv(\"ELASTICSEARCH_LOCAL_URL\")\n",
+    "ES_INDEX_NAME = os.getenv(\"ELASTICSEARCH_INDEX\")\n",
+    "HF_EMBEDDING_MODEL_NAME = os.getenv(\"HF_EMBEDDING_MODEL_NAME\")\n",
+    "BASE_URL = os.getenv(\"LLM_BASE_LOCAL_URL\")\n",
+    "MODEL_NAME = os.getenv(\"OLLAMA_MODEL_NAME\")\n",
+    "\n",
+    "config = AutoConfig.from_pretrained(HF_EMBEDDING_MODEL_NAME)\n",
+    "embedding_dim = config.hidden_size"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea41ce0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings = OllamaEmbeddings(base_url=BASE_URL, model=MODEL_NAME)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8eee9390",
+   "metadata": {},
+   "source": [
+    "# Similitud Aleatoria"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d7b150e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = load_dataset(\"sentence-transformers/natural-questions\")\n",
+    "\n",
+    "metric = SemanticSimilarity(embeddings=embeddings)  \n",
+    "result = await metric.ascore(reference=pregunta, response=respuesta)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "assistance-engine",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}