{ "cells": [ { "cell_type": "code", "execution_count": 5, "id": "0a8abbfa", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import os\n", "import re\n", "import uuid\n", "from dataclasses import dataclass\n", "from pathlib import Path\n", "from typing import Any, Dict, List, Optional, Tuple\n", "# from bnf import grammar\n", "import nltk\n", "from elasticsearch import Elasticsearch\n", "from langchain_core.documents import Document\n", "from langchain_elasticsearch import ElasticsearchStore\n", "from langchain_ollama import OllamaEmbeddings\n", "from lark import Lark, Token, Transformer, Tree\n", "from transformers import AutoConfig\n", "\n", "from src.config import settings\n", "\n", "nltk.download(\"punkt\", quiet=True)" ] }, { "cell_type": "code", "execution_count": 6, "id": "5c9d292b", "metadata": {}, "outputs": [], "source": [ "config = AutoConfig.from_pretrained(settings.hf_emb_model_name)\n", "embedding_dim = config.hidden_size" ] }, { "cell_type": "code", "execution_count": 9, "id": "d2009c2b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "qwen3.5:2b\n" ] } ], "source": [ "print(settings.ollama_model_name)" ] }, { "cell_type": "markdown", "id": "baa779f3", "metadata": {}, "source": [ "# Functions" ] }, { "cell_type": "code", "execution_count": 3, "id": "26927d0c", "metadata": {}, "outputs": [], "source": [ "def bnf_to_lark(bnf_text):\n", " text = re.sub(r\"<([^>]+)>\", r\"\\1\", bnf_text) # remove <>\n", " text = text.replace(\"::=\", \":\")\n", " return text" ] }, { "cell_type": "code", "execution_count": 4, "id": "89be8bf6", "metadata": {}, "outputs": [], "source": [ "@dataclass\n", "class Chunk:\n", " text: str\n", " kind: str\n", " metadata: Dict[str, Any]\n", "\n", "def _span(node: Tree) -> Optional[Tuple[int, int]]:\n", " m = node.meta\n", " s = getattr(m, \"start_pos\", None)\n", " e = getattr(m, \"end_pos\", None)\n", " if s is None or e is None:\n", " return None\n", " return s, e\n", "\n", "def _iter_trees(t: Tree):\n", " yield t\n", " for c in t.children:\n", " if isinstance(c, Tree):\n", " yield from _iter_trees(c)\n", "\n", "def _cmd_name(line: str) -> Optional[str]:\n", " m = re.match(r\"^\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*\\(\", line)\n", " return m.group(1) if m else None\n", "\n", "def chunk_atomic_lines(code: str) -> List[Chunk]:\n", " tree = parser.parse(code)\n", " chunks: List[Chunk] = []\n", "\n", " for node in _iter_trees(tree):\n", " if node.data == \"stmt_line\":\n", " sp = _span(node)\n", " if not sp:\n", " continue\n", " s, e = sp\n", " text = code[s:e].strip()\n", " if not text:\n", " continue\n", "\n", " chunks.append(\n", " Chunk(\n", " text=text,\n", " kind=\"line\",\n", " metadata={\n", " \"granularity\": \"atomic\",\n", " \"command\": _cmd_name(text)\n", " }\n", " )\n", " )\n", " return chunks\n", "\n", "def chunk_blocks(code: str) -> List[Chunk]:\n", " tree = parser.parse(code)\n", " chunks: List[Chunk] = []\n", "\n", " for node in _iter_trees(tree):\n", " if node.data in (\"if_block\", \"loop_block\", \"try_block\", \"go_async_block\", \"function_block\"):\n", " sp = _span(node)\n", " if not sp:\n", " continue\n", " s, e = sp\n", " text = code[s:e].strip()\n", " if not text:\n", " continue\n", "\n", " chunks.append(\n", " Chunk(\n", " text=text,\n", " kind=node.data,\n", " metadata={\"granularity\": \"block\"}\n", " )\n", " )\n", " return chunks\n", "\n", "def chunk_avap_code(code: str) -> List[Chunk]:\n", " # Keep original offsets: do NOT lstrip. Grammar already accepts leading _NL.\n", " blocks = chunk_blocks(code)\n", " lines = chunk_atomic_lines(code)\n", " return blocks + lines" ] }, { "cell_type": "markdown", "id": "23a92e13", "metadata": {}, "source": [ "# BNF " ] }, { "cell_type": "code", "execution_count": 5, "id": "26ad9c81", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "line 9 : syntax error at or before | = |\n", "syntax error at end of file (missing ; ?)\n" ] }, { "data": { "text/plain": [ "({}, None)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grammar_ = (DATA_DIR / \"raw\" / \"code\" / \"BNF_v1.txt\").read_text(\n", " encoding=\"utf-8\"\n", ")\n", "grammar(grammar_)" ] }, { "cell_type": "code", "execution_count": null, "id": "7cdf69c4", "metadata": {}, "outputs": [], "source": [ "parser = Lark(grammar=grammar, parser=\"lalr\", propagate_positions=True)" ] }, { "cell_type": "code", "execution_count": 18, "id": "19253100", "metadata": {}, "outputs": [ { "ename": "UnexpectedToken", "evalue": "Unexpected token Token('COMMAND', '(base, 1000)') at line 2, column 11.\nExpected one of: \n\t* EQUAL\nPrevious tokens: [Token('NAME', 'addVar')]\n", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mUnexpectedCharacters\u001b[39m Traceback (most recent call last)", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/lexer.py:689\u001b[39m, in \u001b[36mContextualLexer.lex\u001b[39m\u001b[34m(self, lexer_state, parser_state)\u001b[39m\n\u001b[32m 688\u001b[39m lexer = \u001b[38;5;28mself\u001b[39m.lexers[parser_state.position]\n\u001b[32m--> \u001b[39m\u001b[32m689\u001b[39m \u001b[38;5;28;01myield\u001b[39;00m \u001b[43mlexer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mnext_token\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlexer_state\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparser_state\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 690\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mEOFError\u001b[39;00m:\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/lexer.py:622\u001b[39m, in \u001b[36mBasicLexer.next_token\u001b[39m\u001b[34m(self, lex_state, parser_state)\u001b[39m\n\u001b[32m 621\u001b[39m allowed = {\u001b[33m\"\u001b[39m\u001b[33m\u001b[39m\u001b[33m\"\u001b[39m}\n\u001b[32m--> \u001b[39m\u001b[32m622\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m UnexpectedCharacters(lex_state.text.text, line_ctr.char_pos, line_ctr.line, line_ctr.column,\n\u001b[32m 623\u001b[39m allowed=allowed, token_history=lex_state.last_token \u001b[38;5;129;01mand\u001b[39;00m [lex_state.last_token],\n\u001b[32m 624\u001b[39m state=parser_state, terminals_by_name=\u001b[38;5;28mself\u001b[39m.terminals_by_name)\n\u001b[32m 626\u001b[39m value, type_ = res\n", "\u001b[31mUnexpectedCharacters\u001b[39m: No terminal matches '(' in the current parser context, at line 2 col 11\n\n addVar(base, 1000)\n ^\nExpected one of: \n\t* RPAR\n\t* EQUAL\n\nPrevious tokens: Token('NAME', 'addVar')\n", "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[31mUnexpectedToken\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[18]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 1\u001b[39m code = \u001b[33m\"\"\"\u001b[39m\n\u001b[32m 2\u001b[39m \u001b[33m addVar(base, 1000)\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[33m addVar(copia, $base)\u001b[39m\n\u001b[32m 4\u001b[39m \u001b[33m addResult(copia)\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[33m\"\"\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m tree = \u001b[43mparser\u001b[49m\u001b[43m.\u001b[49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcode\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/lark.py:677\u001b[39m, in \u001b[36mLark.parse\u001b[39m\u001b[34m(self, text, start, on_error)\u001b[39m\n\u001b[32m 675\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m on_error \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.options.parser != \u001b[33m'\u001b[39m\u001b[33mlalr\u001b[39m\u001b[33m'\u001b[39m:\n\u001b[32m 676\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mThe on_error option is only implemented for the LALR(1) parser.\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m--> \u001b[39m\u001b[32m677\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mparser\u001b[49m\u001b[43m.\u001b[49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mon_error\u001b[49m\u001b[43m=\u001b[49m\u001b[43mon_error\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/parser_frontends.py:131\u001b[39m, in \u001b[36mParsingFrontend.parse\u001b[39m\u001b[34m(self, text, start, on_error)\u001b[39m\n\u001b[32m 129\u001b[39m kw = {} \u001b[38;5;28;01mif\u001b[39;00m on_error \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m {\u001b[33m'\u001b[39m\u001b[33mon_error\u001b[39m\u001b[33m'\u001b[39m: on_error}\n\u001b[32m 130\u001b[39m stream = \u001b[38;5;28mself\u001b[39m._make_lexer_thread(text)\n\u001b[32m--> \u001b[39m\u001b[32m131\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mparser\u001b[49m\u001b[43m.\u001b[49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchosen_start\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/parsers/lalr_parser.py:42\u001b[39m, in \u001b[36mLALR_Parser.parse\u001b[39m\u001b[34m(self, lexer, start, on_error)\u001b[39m\n\u001b[32m 40\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mparse\u001b[39m(\u001b[38;5;28mself\u001b[39m, lexer, start, on_error=\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m 41\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m42\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mparser\u001b[49m\u001b[43m.\u001b[49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 43\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m UnexpectedInput \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 44\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m on_error \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/parsers/lalr_parser.py:88\u001b[39m, in \u001b[36m_Parser.parse\u001b[39m\u001b[34m(self, lexer, start, value_stack, state_stack, start_interactive)\u001b[39m\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m start_interactive:\n\u001b[32m 87\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m InteractiveParser(\u001b[38;5;28mself\u001b[39m, parser_state, parser_state.lexer)\n\u001b[32m---> \u001b[39m\u001b[32m88\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mparse_from_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparser_state\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/parsers/lalr_parser.py:111\u001b[39m, in \u001b[36m_Parser.parse_from_state\u001b[39m\u001b[34m(self, state, last_token)\u001b[39m\n\u001b[32m 109\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mNameError\u001b[39;00m:\n\u001b[32m 110\u001b[39m \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m111\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[32m 112\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 113\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.debug:\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/parsers/lalr_parser.py:100\u001b[39m, in \u001b[36m_Parser.parse_from_state\u001b[39m\u001b[34m(self, state, last_token)\u001b[39m\n\u001b[32m 98\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 99\u001b[39m token = last_token\n\u001b[32m--> \u001b[39m\u001b[32m100\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlexer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlex\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 101\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01massert\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\n\u001b[32m 102\u001b[39m \u001b[43m \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfeed_token\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[36mFile \u001b[39m\u001b[32m~/VsCodeProjects/assistance-engine/.venv/lib/python3.12/site-packages/lark/lexer.py:698\u001b[39m, in \u001b[36mContextualLexer.lex\u001b[39m\u001b[34m(self, lexer_state, parser_state)\u001b[39m\n\u001b[32m 696\u001b[39m last_token = lexer_state.last_token \u001b[38;5;66;03m# Save last_token. Calling root_lexer.next_token will change this to the wrong token\u001b[39;00m\n\u001b[32m 697\u001b[39m token = \u001b[38;5;28mself\u001b[39m.root_lexer.next_token(lexer_state, parser_state)\n\u001b[32m--> \u001b[39m\u001b[32m698\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m UnexpectedToken(token, e.allowed, state=parser_state, token_history=[last_token], terminals_by_name=\u001b[38;5;28mself\u001b[39m.root_lexer.terminals_by_name)\n\u001b[32m 699\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m UnexpectedCharacters:\n\u001b[32m 700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m e\n", "\u001b[31mUnexpectedToken\u001b[39m: Unexpected token Token('COMMAND', '(base, 1000)') at line 2, column 11.\nExpected one of: \n\t* EQUAL\nPrevious tokens: [Token('NAME', 'addVar')]\n" ] } ], "source": [ "code = \"\"\"\n", " addVar(base, 1000)\n", " addVar(copia, $base)\n", " addResult(copia)\n", "\"\"\"\n", "tree = parser.parse(code)" ] }, { "cell_type": "code", "execution_count": 19, "id": "04bf9223", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'tree' is not defined", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[19]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mtree\u001b[49m.pretty()\n", "\u001b[31mNameError\u001b[39m: name 'tree' is not defined" ] } ], "source": [ "tree.pretty()" ] }, { "cell_type": "code", "execution_count": 35, "id": "b2999a98", "metadata": {}, "outputs": [], "source": [ "chunks = chunk_avap_code(code)\n", "\n", "for c in chunks:\n", " print(\"----\")\n", " print(\"TYPE:\", c.kind)\n", " print(\"TEXT:\\n\", c.text)\n", " print(\"META:\", c.metadata)" ] }, { "cell_type": "markdown", "id": "77f6c552", "metadata": {}, "source": [ "## Elastic Search" ] }, { "cell_type": "code", "execution_count": 51, "id": "09ce3e29", "metadata": {}, "outputs": [], "source": [ "es = Elasticsearch(\n", " ELASTICSEARCH_URL,\n", " request_timeout=120,\n", " max_retries=5,\n", " retry_on_timeout=True,\n", ")" ] }, { "cell_type": "code", "execution_count": 52, "id": "d575c386", "metadata": {}, "outputs": [], "source": [ "if es.indices.exists(index=ELASTICSEARCH_CODE_INDEX):\n", " es.indices.delete(index=ELASTICSEARCH_CODE_INDEX)" ] }, { "cell_type": "code", "execution_count": 56, "id": "40ea0af8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "avap-code\n", "avap-docs-test\n" ] } ], "source": [ "for index in es.indices.get(index=\"*\"):\n", " print(index)" ] }, { "cell_type": "code", "execution_count": 54, "id": "4e091b39", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "OllamaEmbeddings(model='qwen3-0.6B-emb:latest', validate_model_on_init=False, base_url='http://localhost:11434', client_kwargs={}, async_client_kwargs={}, sync_client_kwargs={}, mirostat=None, mirostat_eta=None, mirostat_tau=None, num_ctx=None, num_gpu=None, keep_alive=None, num_thread=None, repeat_last_n=None, repeat_penalty=None, temperature=None, stop=None, tfs_z=None, top_k=None, top_p=None)" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "embeddings = OllamaEmbeddings(base_url=OLLAMA_LOCAL_URL, model=OLLAMA_EMB_MODEL_NAME)\n", "embeddings" ] }, { "cell_type": "code", "execution_count": 55, "id": "5aff21c0", "metadata": {}, "outputs": [], "source": [ "# index into Elasticsearch\n", "db = ElasticsearchStore.from_documents(\n", " code_chunks,\n", " embeddings,\n", " client=es,\n", " index_name=ELASTICSEARCH_CODE_INDEX,\n", " distance_strategy=\"COSINE\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "74c0a377", "metadata": {}, "outputs": [], "source": [ "response = es.search(\n", " index=ELASTICSEARCH_CODE_INDEX,\n", " body={\n", " \"query\": {\"match_all\": {}},\n", " \"size\": 10 \n", " }\n", ")\n", "\n", "for hit in response[\"hits\"][\"hits\"]:\n", " print(\"ID:\", hit[\"_id\"])\n", " print(\"Source:\", hit[\"_source\"])\n", " print(\"-\" * 40)" ] }, { "cell_type": "markdown", "id": "d823650e", "metadata": {}, "source": [ "# Retrive" ] }, { "cell_type": "code", "execution_count": null, "id": "5732a27d", "metadata": {}, "outputs": [], "source": [ "base_retriever = db.as_retriever(\n", " search_type=\"similarity\",\n", " search_kwargs={\"k\": 5}\n", " ) \n", "\n", "docs = base_retriever.invoke(\"What reserved words does AVAP have?\")\n", "docs" ] }, { "cell_type": "code", "execution_count": null, "id": "8706506f", "metadata": {}, "outputs": [], "source": [ "embeddings = OllamaEmbeddings(base_url=OLLAMA_URL, model=OLLAMA_EMB_MODEL_NAME)\n", "\n", "vector_store = ElasticsearchStore(\n", " client=es,\n", " index_name=ELASTICSEARCH_DOCS_INDEX,\n", " embedding=embeddings,\n", " query_field=\"text\",\n", " vector_query_field=\"vector\",\n", ")\n", "\n", "results = vector_store.similarity_search_with_score(\n", " query=\"What data types does AVAP have?\",\n", " k=50\n", ")\n", "\n", "results" ] } ], "metadata": { "kernelspec": { "display_name": "assistance-engine", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.11" } }, "nbformat": 4, "nbformat_minor": 5 }