diff --git a/scratches/pseco/evaluation/embeddings/n00 first Analysis.ipynb b/scratches/pseco/evaluation/embeddings/n00 first Analysis.ipynb new file mode 100644 index 0000000..6769fe1 --- /dev/null +++ b/scratches/pseco/evaluation/embeddings/n00 first Analysis.ipynb @@ -0,0 +1,116 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "096e6224", + "metadata": {}, + "source": [ + "# Libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1970d59", + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.core import download_loader\n", + "from ragas.testset.evolutions import simple, reasoning, multi_context\n", + "from ragas.testset.generator import TestsetGenerator\n", + "from langchain_openai import ChatOpenAI\n", + "from ragas.embeddings import OpenAIEmbeddings\n", + "import openai\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bfe1ca0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "import uuid\n", + "from pathlib import Path\n", + "from typing import Any, Protocol\n", + "from datasets import load_dataset\n", + "from langchain_core.documents import Document\n", + "from langchain_elasticsearch import ElasticsearchStore\n", + "import torch\n", + "import torch.nn.functional as F\n", + "from loguru import logger\n", + "from langchain_ollama import OllamaEmbeddings\n", + "from ragas.metrics.collections import SemanticSimilarity \n", + "from transformers import AutoTokenizer, AutoModel, AutoConfig\n", + "from elasticsearch import Elasticsearch\n", + "from langchain_elasticsearch import ElasticsearchStore\n", + "import nltk\n", + "from nltk.tokenize import sent_tokenize\n", + "nltk.download(\"punkt\", quiet=True)\n", + "\n", + "ES_URL = os.getenv(\"ELASTICSEARCH_LOCAL_URL\")\n", + "ES_INDEX_NAME = os.getenv(\"ELASTICSEARCH_INDEX\")\n", + "HF_EMBEDDING_MODEL_NAME = os.getenv(\"HF_EMBEDDING_MODEL_NAME\")\n", + "BASE_URL = os.getenv(\"LLM_BASE_LOCAL_URL\")\n", + "MODEL_NAME = os.getenv(\"OLLAMA_MODEL_NAME\")\n", + "\n", + "config = AutoConfig.from_pretrained(HF_EMBEDDING_MODEL_NAME)\n", + "embedding_dim = config.hidden_size" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea41ce0f", + "metadata": {}, + "outputs": [], + "source": [ + "embeddings = OllamaEmbeddings(base_url=BASE_URL, model=MODEL_NAME)" + ] + }, + { + "cell_type": "markdown", + "id": "8eee9390", + "metadata": {}, + "source": [ + "# Similitud Aleatoria" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7b150e5", + "metadata": {}, + "outputs": [], + "source": [ + "ds = load_dataset(\"sentence-transformers/natural-questions\")\n", + "\n", + "metric = SemanticSimilarity(embeddings=embeddings) \n", + "result = await metric.ascore(reference=pregunta, response=respuesta)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "assistance-engine", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}