diff --git a/output/evaluation_bge-m3-latest_avap-docs-test-v4-bge_20260406_114819.json b/output/evaluation_bge-m3-latest_avap-docs-test-v4-bge_20260406_114819.json new file mode 100644 index 0000000..f1ab503 --- /dev/null +++ b/output/evaluation_bge-m3-latest_avap-docs-test-v4-bge_20260406_114819.json @@ -0,0 +1,373 @@ +{ + "generated_at": "2026-04-06T11:48:19.164260", + "embedding_model": "bge-m3:latest", + "index_name": "avap-docs-test-v4-bge", + "result": { + "status": "ok", + "questions_evaluated": 50, + "elapsed_seconds": 427.5, + "judge_model": "claude-sonnet-4-20250514", + "index": "avap-docs-test-v4-bge", + "category_filter": "all", + "scores": { + "faithfulness": 0.4091, + "answer_relevancy": 0.8033, + "context_recall": 0.4084, + "context_precision": 0.3582 + }, + "global_score": 0.4948, + "verdict": "INSUFFICIENT", + "details": [ + { + "id": "GD-R-001", + "category": "RETRIEVAL", + "question": "What is AVAP and what is it designed for?", + "answer_preview": "AVAP, or Advanced Virtual Application Protocol, is a new proprietary language. It appears to be used in the context of an application that involves adding variables, parameters, and results. The language seems to have specific syntax and structure, such as using `addVar`, `addParam`, and `addResult`", + "n_chunks": 5 + }, + { + "id": "GD-R-002", + "category": "RETRIEVAL", + "question": "How does the if() conditional block work in AVAP? How are blocks closed?", + "answer_preview": "In AVAP, a conditional block is enclosed using curly braces `{}`. The `if` statement checks a condition and executes code within an `if` block only if the condition evaluates to true.\n\nBlocks in AVAP are not explicitly closed with `end()` as they are in some other programming languages. Instead, the", + "n_chunks": 5 + }, + { + "id": "GD-R-003", + "category": "RETRIEVAL", + "question": "How does AVAP handle external HTTP calls? What commands are available and how is timeout managed?", + "answer_preview": "AVAP handles external HTTP calls through the `RequestGet` function, which sends a GET request to the specified URL. The response from this call is stored in the variable `respuesta`.\n\nTimeout management in AVAP is handled by the `try...except` block. If an exception occurs during the execution of th", + "n_chunks": 5 + }, + { + "id": "GD-R-004", + "category": "RETRIEVAL", + "question": "How do functions work in AVAP? What is the scope of variables inside a function?", + "answer_preview": "In AVAP, functions are used to encapsulate code that performs specific tasks and can be reused throughout the program. Functions allow for modularity and maintainability by breaking down complex logic into smaller, more manageable pieces.\n\nThe scope of variables within a function in AVAP is determin", + "n_chunks": 5 + }, + { + "id": "GD-R-005", + "category": "RETRIEVAL", + "question": "What are the three types of variable scopes in AVAP and what are their visibility rules?", + "answer_preview": "In AVAP, there are three main types of variable scopes: local, global, and instance. The visibility rules for these scopes are as follows:\n\n- **Local Variables**: These variables are scoped to a specific function or block where they are declared. They can only be accessed within the same scope in wh", + "n_chunks": 5 + }, + { + "id": "GD-R-006", + "category": "RETRIEVAL", + "question": "How does concurrency work in AVAP? What are goroutines and how are they launched?", + "answer_preview": "Concurrency is a fundamental concept in AVAP that allows multiple tasks to be executed simultaneously. In AVAP, this is achieved through the use of goroutines.\n\nGoroutines are lightweight threads of execution that can run concurrently with other goroutines or within the same goroutine. They are mana", + "n_chunks": 5 + }, + { + "id": "GD-R-007", + "category": "RETRIEVAL", + "question": "What is the addParam command and how does it capture HTTP request parameters?", + "answer_preview": "The `addParam` command in this context is used to add key-value pairs to an HTTP request. It captures the specified parameters, which can include headers like \"lang\" or \"client_id\", as well as query string elements such as \"sells\". The parameters are added to the request using the `addParam` functio", + "n_chunks": 5 + }, + { + "id": "GD-R-008", + "category": "RETRIEVAL", + "question": "How does the startLoop / endLoop construct work in AVAP?", + "answer_preview": "The `startLoop` and `endLoop` constructs are used to iterate over a sequence of items or variables. In this context, they appear to be part of a loop structure that iterates through an index variable `i`, which is incremented by 1 each time the loop runs.\n\nHere's how it works:\n\n- The `startLoop(i, 1", + "n_chunks": 5 + }, + { + "id": "GD-R-009", + "category": "RETRIEVAL", + "question": "What is the addResult command and how does it build the HTTP response?", + "answer_preview": "The `addResult` command in this context is used to store a result variable within the function's return value. It takes two parameters: the name of the result variable (`msg`, `error_trace`, etc.) and the actual value that should be stored.\n\nIn the provided code, it appears to be building an HTTP re", + "n_chunks": 5 + }, + { + "id": "GD-R-010", + "category": "RETRIEVAL", + "question": "How does error handling work in AVAP with try() and exception()?", + "answer_preview": "In the provided context, error handling is implemented using the `try()` function followed by an `exception(e)` block. This structure allows for the execution of a potentially risky operation (indicated by the `try()` part) that may raise exceptions (`e`). The `exception(e)` block then captures and ", + "n_chunks": 5 + }, + { + "id": "GD-R-011", + "category": "RETRIEVAL", + "question": "What is the replace() command in AVAP and how is it used?", + "answer_preview": "The `replace()` command in AVAP is used to perform a string replacement operation. It takes three parameters: the first parameter is the original string, the second parameter specifies what part of the string should be replaced (using a regular expression), and the third parameter is the new value t", + "n_chunks": 5 + }, + { + "id": "GD-R-012", + "category": "RETRIEVAL", + "question": "What are the reserved keywords in AVAP that cannot be used as identifiers?", + "answer_preview": "In AVAP, there are no explicitly mentioned reserved keywords that cannot be used as identifiers. The provided context does not contain any information about reserved keywords or their restrictions on use as identifiers. Therefore, based solely on the given context:\n\nAnswer: There are no specific res", + "n_chunks": 2 + }, + { + "id": "GD-R-013", + "category": "RETRIEVAL", + "question": "How does AVAP handle string formatting and concatenation?", + "answer_preview": "AVAP uses the `addVar` function to dynamically generate strings. For example, in the first snippet:\n\n```plaintext\naddVar(mensaje, \"Hola mundo desde AVAP\")\n```\n\nThis line adds a variable named `mensaje` with the value `\"Hola mundo desde AVAP\"`. The `addResult` function then outputs this string.\n\nIn a", + "n_chunks": 5 + }, + { + "id": "GD-R-014", + "category": "RETRIEVAL", + "question": "How does the encodeSHA256 command work in AVAP?", + "answer_preview": "The `encodeSHA256` command in AVAP is used to generate a SHA-256 hash of a payload data string. This function takes two parameters: the payload data and a checksum, which are combined using the SHA-256 algorithm to produce a fixed-size output.\n\nHere's how it works:\n\n1. The `encodeSHA256` command acc", + "n_chunks": 5 + }, + { + "id": "GD-R-015", + "category": "RETRIEVAL", + "question": "How does AVAP handle date and time operations?", + "answer_preview": "AVAP handles date and time operations using the `getDateTime` function. This function takes three parameters: a string for the language (defaulting to English), an integer representing the number of days to add or subtract, and a string specifying the timezone. The function returns the current date ", + "n_chunks": 5 + }, + { + "id": "GD-R-016", + "category": "RETRIEVAL", + "question": "What is the AddvariableToJSON command and how is it used to build JSON objects?", + "answer_preview": "The `AddvariableToJSON` command in this context is a function that takes three parameters: an identifier (`clave`), a variable name, and a value. It appends the given value to the specified variable within a JSON object.\n\nFor example:\n- The first line of code uses `AddvariableToJSON(clave, \"cliente_", + "n_chunks": 5 + }, + { + "id": "GD-R-017", + "category": "RETRIEVAL", + "question": "How does the getListLen command work and what is it used for?", + "answer_preview": "The getListLen function calculates the length of a given list. In this context, it appears to be used with two lists:\n\n1. `mi_lista` in the first snippet.\n2. `registros` in the second snippet.\n\nIt returns the number of elements in these lists. This could be useful for determining how many items are ", + "n_chunks": 5 + }, + { + "id": "GD-R-018", + "category": "RETRIEVAL", + "question": "How does the randomString command work in AVAP?", + "answer_preview": "The `randomString` command generates a random string of characters based on the specified parameters. In this case, it creates a 32-character long string consisting of uppercase letters and digits.\n\nHere is the relevant code snippet from the context:\n\n```plaintext\n[2] randomString(\"[A-Z]\\d\", 32, tok", + "n_chunks": 5 + }, + { + "id": "GD-R-019", + "category": "RETRIEVAL", + "question": "What is the $ dereference operator in AVAP and when is it used?", + "answer_preview": "The $ dereference operator in AVAP is used to access elements of an array or list. It is typically followed by a variable name enclosed in square brackets, indicating that the value should be retrieved from the corresponding element in the array.\n\nFor example:\n```avap\n$[arrayName[index]]\n```\nThis ex", + "n_chunks": 5 + }, + { + "id": "GD-R-020", + "category": "RETRIEVAL", + "question": "How does AVAP handle ORM database operations? What commands are available?", + "answer_preview": "AVAP uses the `ormCheckTable` function to check if a table exists in the database. If the table does not exist, it creates the table using the `ormCreateTable` command.\n\nThe available commands for ORM database operations include:\n- `ormCheckTable(tabla_pruebas, resultado_comprobacion)`: Checks if a ", + "n_chunks": 5 + }, + { + "id": "GD-C-001", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'name' parameter and returns a personalized greeting.", + "answer_preview": "```avap\nregisterEndpoint(\"/greet\",\"GET\",[],\"HELLO_WORLD\",main,result)\n\naddVar(name, \"Alberto\")\nresult = \"Hello,\" + name \naddResult(result)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-002", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'password' parameter, generates a SHA-256 hash, and returns it.", + "answer_preview": "```avap\nfunction generateSha256Hash(password) {\n addVar(hash, encodeSHA256(password))\n return(hash)\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-003", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that loops from 1 to 5, builds a JSON object with each index as a key, and returns it.", + "answer_preview": "```avap\nstartLoop(i, 1, 5)\n item = \"item_%s\" % i\n AddvariableToJSON(item,'valor_generado',mi_json)\nendLoop()\naddResult(mi_json)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-004", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that validates if a 'role' parameter belongs to a list of allowed roles and returns the access result.", + "answer_preview": "```avap\nacceso = False\n\nif(None, None, \"r == 'admin' or r == 'editor' or r == 'root'\")\n acceso = True\nend()\n\naddResult(acceso)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-005", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that makes a GET request to an external API and handles connection errors.", + "answer_preview": "```avap\ntry()\n addVar(_status, 500)\n addVar(error_msg, \"Error de conexión\")\n addResult(error_msg)\nend()\n\naddVar(url, \"https://api.example.com/data\")\naddVar(headers, {\"Content-Type\": \"application/json\"})\naddVar(body, \"{}\")\n\nif(!url || !headers || !body) {\n addVar(_status, 400)\n addVar(", + "n_chunks": 5 + }, + { + "id": "GD-C-006", + "category": "CODE_GENERATION", + "question": "Write an AVAP function that takes two numbers and returns their sum, then call it and return the result.", + "answer_preview": "```avap\nfunction suma(a, b){\n total = a + b\n return(total)\n}\nresultado = suma(10, 20)\naddResult(resultado)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-007", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'subtotal' parameter, computes 21% VAT, and returns the total.", + "answer_preview": "```avap\n[1] subtotal = readParam(\"subtotal\")\niva = subtotal * 0.21\ntotal = subtotal + iva\naddResult(total)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-008", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads an 'api_key' parameter and returns status 403 if it is null.", + "answer_preview": "```avap\naddParam(\"api_key\", key)\nif(key == None, None, \"==\")\n addVar(_status, 403)\n addVar(error, \"Acceso denegado: falta API KEY\")\n addResult(error)\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-009", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that generates a 32-character random alphanumeric token and returns it.", + "answer_preview": "```avap\ntoken = randomString(\"[A-Z]\\d\", 32, token_seguridad)\nreturn(token)\n```\n\nEvidence:\n- \"[1] randomString(\"[A-Z]\\d\", 32, token_seguridad)\"\n(Generated a 32-character random alphanumeric token)", + "n_chunks": 5 + }, + { + "id": "GD-C-010", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'lang' parameter and returns 'Hola' if it is 'es' or 'Hello' if it is 'en'.", + "answer_preview": "```avap\nifParam(\"lang\", l)\n if(l, \"es\", \"=\")\n addVar(msg, \"Hola\")\n end()\nend()\n\naddResult(msg)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-011", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that checks if a database table exists and creates it if it does not.", + "answer_preview": "```avap\nif(not db.exists(\"database_table\"))\n createTable(\"database_table\", \"column1\", \"type1\", \"nullable\")\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-012", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that gets the current UTC timestamp and adds 24 hours to compute an expiration time.", + "answer_preview": "```avap\ngetDateTime(\"\", 86400, \"UTC\", expira)\naddResult(expira)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-013", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that receives a new password parameter, validates it is not equal to the old password, and returns a confirmation.", + "answer_preview": "```avap\naddParam(\"password\", pass_nueva)\npass_antigua = \"password\"\nif(pass_nueva != pass_antigua)\n addVar(cambio, \"Contraseña actualizada\")\nend()\naddResult(cambio)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-014", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a list parameter and returns its element count.", + "answer_preview": "```avap\nfunction getListLen(param_name, total){\n addParam(\"param_name\", param_name)\n addResult(total)\n}\ngetListLen(\"data_list\", cantidad)\naddResult(cantidad)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-015", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses a validation function to check a token parameter and returns the authorization result.", + "answer_preview": "```avap\nfunction es_valido(token){\n response = False\n if(token, \"SECRET\", \"=\")\n response = True\n end()\n return(response)\n}\n\nautorizado = es_valido(\"SECRET\")\naddResult(autorizado)\n\nrandomString(\"[A-Z]\\d\", 32, token_seguridad)\naddResult(token_seguridad)\n\nfunction suma(a, b){\n tot", + "n_chunks": 5 + }, + { + "id": "GD-C-016", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that returns two values in the HTTP response: a status code 200 and a message 'Success'.", + "answer_preview": "```avap\naddVar(code, 200)\naddVar(status, \"Success\")\naddResult(code)\naddResult(status)\n\n// Example of adding more variables or results\naddVar(mensaje, \"Hola mundo desde AVAP\")\naddResult(mensaje)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-017", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'saldo' parameter and returns True if it is greater than zero, False otherwise.", + "answer_preview": "```avap\nifParam(\"sal_par\", saldo)\nif(saldo > 0)\n permitir = True\nelse()\n permitir = False\nend()\naddResult(permitir)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-018", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that converts a Unix timestamp parameter to a human-readable date in dd/mm/yyyy format.", + "answer_preview": "```avap\nstampToDatetime(1708726162, \"%d/%m/%Y\", 0, fecha_human)\naddResult(fecha_human)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-019", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that replaces all spaces in a string parameter with hyphens and returns the result.", + "answer_preview": "```avap\nfunction replaceSpacesWithHyphens(inputString) {\n let modifiedString = inputString.replace(/ /g, '-');\n return modifiedString;\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-020", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses try/exception to execute a raw SQL query and return status 500 on database errors.", + "answer_preview": "```avap\ntry()\n ormDirect(\"UPDATE table_inexistente SET a=1\", res)\nexception(e)\n addVar(_status, 500)\n addVar(error_msg, \"Error de base de datos\")\n addResult(error_msg)\nend()\n\naddVar(code, 200)\naddVar(status, \"Success\")\naddResult(code)\naddResult(status)\n\naddParam(\"api_key\", key)\nif(key, N", + "n_chunks": 5 + }, + { + "id": "GD-V-001", + "category": "CONVERSATIONAL", + "question": "Can you summarize what you just explained about AVAP scopes in fewer words?", + "answer_preview": "AVAP is a new proprietary language that uses variables to store and manipulate data. It includes functions like `addVar` for adding variables, `result` for storing results, and loops with `startLoop`. The context provided shows examples of how these functions are used, such as in the \"HELLO_WORLD\" e", + "n_chunks": 5 + }, + { + "id": "GD-V-002", + "category": "CONVERSATIONAL", + "question": "You mentioned that addResult builds the JSON response — can you clarify how multiple addResult calls work together?", + "answer_preview": "Certainly! The `addResult` function is used to build and return a JSON response. Each call to `addResult` adds an element to the JSON structure being constructed.\n\nFor example, in context [1], `addParam(\"client_id\", id_interno)` and `addResult(id_interno)` are part of a larger process where `id_inte", + "n_chunks": 5 + }, + { + "id": "GD-V-003", + "category": "CONVERSATIONAL", + "question": "What is the difference between addVar and a plain assignment like x = 10 in AVAP?", + "answer_preview": "In AVAP, `addVar` is used to define variables that can be referenced later within the program. It allows for dynamic variable creation based on input or predefined values.\n\nA plain assignment like `x = 10` does not create a new variable; it simply assigns an initial value to an existing variable nam", + "n_chunks": 5 + }, + { + "id": "GD-V-004", + "category": "CONVERSATIONAL", + "question": "Can you explain again the difference between the two modes of the if() command?", + "answer_preview": "The `if()` command in AVAP is used to execute a block of code based on a condition. The syntax for the `if()` command can be represented as follows:\n\n```avap\n[1] addParam(\"lang\", l)\nif(l, \"es\", \"=\")\n addVar(msg, \"Hola\")\nend()\naddResult(msg)\n\n[2] encontrado = False\nstartLoop(i, 1, 10)\n if(i, 5,", + "n_chunks": 5 + }, + { + "id": "GD-V-005", + "category": "CONVERSATIONAL", + "question": "What happens if an error occurs in AVAP without a try block?", + "answer_preview": "If an error occurs in AVAP without a try block, the following actions are taken:\n\n- An exception is raised.\n- The status code is set to 500 (Internal Server Error).\n- An error message indicating \"Error de base de datos\" is added to the result.\n\nThis indicates that there was an unexpected issue with ", + "n_chunks": 5 + }, + { + "id": "GD-V-006", + "category": "CONVERSATIONAL", + "question": "Can you explain again how the timeout in RequestGet works?", + "answer_preview": "The `RequestGet` function in AVAP is used to make a request to an API endpoint. The timeout parameter allows you to specify how long the function should wait before timing out if it hasn't received a response from the server within that period.\n\nIn this case, the `timeout` parameter is set to 0, whi", + "n_chunks": 5 + }, + { + "id": "GD-V-007", + "category": "CONVERSATIONAL", + "question": "Can I iterate over a list of items in AVAP instead of a numeric range?", + "answer_preview": "Yes, you can iterate over a list of items in AVAP. The provided code snippet shows an example where the loop variable `i` is used to access each item in the list by name. This demonstrates that lists are iterable and can be accessed using names rather than indices.\n\n```avap\nstartLoop(i, 1, 10)\n i", + "n_chunks": 5 + }, + { + "id": "GD-V-008", + "category": "CONVERSATIONAL", + "question": "What is the difference between RequestGet and RequestPost in practice?", + "answer_preview": "The `RequestGet` function retrieves data from a specified URL, while the `RequestPost` function sends data to a specified URL. In this context, both functions are used to interact with an API endpoint.\n\n- **RequestGet**: This function is typically used when you want to retrieve information or data f", + "n_chunks": 5 + }, + { + "id": "GD-V-009", + "category": "CONVERSATIONAL", + "question": "Goroutines cannot access Main Local Scope — can you give a practical example of why that matters?", + "answer_preview": "In Go programming language, goroutines are lightweight threads of execution that run concurrently with the main thread. They do not have direct access to the global or local scope variables declared in the main function. This is because each goroutine runs within its own independent stack frame and ", + "n_chunks": 5 + }, + { + "id": "GD-V-010", + "category": "CONVERSATIONAL", + "question": "What format does encodeSHA256 return its output in?", + "answer_preview": "The `encodeSHA256` function returns its output as a string.", + "n_chunks": 5 + } + ] + } +} \ No newline at end of file diff --git a/output/evaluation_bge-m3-latest_avap-docs-test-v4_20260406_113404.json b/output/evaluation_bge-m3-latest_avap-docs-test-v4_20260406_113404.json new file mode 100644 index 0000000..645bbcc --- /dev/null +++ b/output/evaluation_bge-m3-latest_avap-docs-test-v4_20260406_113404.json @@ -0,0 +1,373 @@ +{ + "generated_at": "2026-04-06T11:34:04.601590", + "embedding_model": "bge-m3:latest", + "index_name": "avap-docs-test-v4", + "result": { + "status": "ok", + "questions_evaluated": 50, + "elapsed_seconds": 621.9, + "judge_model": "claude-sonnet-4-20250514", + "index": "avap-docs-test-v4", + "category_filter": "all", + "scores": { + "faithfulness": 0.4938, + "answer_relevancy": 0.8273, + "context_recall": 0.5066, + "context_precision": 0.4674 + }, + "global_score": 0.5738, + "verdict": "INSUFFICIENT", + "details": [ + { + "id": "GD-R-001", + "category": "RETRIEVAL", + "question": "What is AVAP and what is it designed for?", + "answer_preview": "AVAP, or Advanced Virtual Application Protocol, is a new proprietary language. It is designed to be used in the context of virtual applications, likely for communication protocols or software development environments that require advanced features such as type casting, slicing, and lexical analysis.", + "n_chunks": 5 + }, + { + "id": "GD-R-002", + "category": "RETRIEVAL", + "question": "How does the if() conditional block work in AVAP? How are blocks closed?", + "answer_preview": "In AVAP, the `if()` conditional block evaluates a logical or comparison expression. It requires an explicit closing statement using the command `end()`.\n\nThe `if()` function supports two modes of invocation:\n1. **Modo 1 (Structured Comparison):** `if(variable, value, \"operator\")` — compares variable", + "n_chunks": 5 + }, + { + "id": "GD-R-003", + "category": "RETRIEVAL", + "question": "How does AVAP handle external HTTP calls? What commands are available and how is timeout managed?", + "answer_preview": "AVAP handles external HTTP calls using the `RequestPost` and `RequestGet` methods. The `timeout` parameter, which specifies a time limit in milliseconds for each request, ensures that requests do not hang due to network latency.\n\n- **`RequestPost(url, querystring, headers, body, destino, timeout)`**", + "n_chunks": 5 + }, + { + "id": "GD-R-004", + "category": "RETRIEVAL", + "question": "How do functions work in AVAP? What is the scope of variables inside a function?", + "answer_preview": "In AVAP, each invocation of a function creates an independent **function scope**. This scope contains parameters of the function, variables created within the function, and intermediate results.\n\nThe variable `x` defined in the example function `ejemplo()` does not exist outside this function becaus", + "n_chunks": 5 + }, + { + "id": "GD-R-005", + "category": "RETRIEVAL", + "question": "What are the three types of variable scopes in AVAP and what are their visibility rules?", + "answer_preview": "In AVAP, there are three types of variable scopes: Global Scope, Main Local Scope, and Function Scope. The visibility rule for these scopes is as follows:\n\n- **Global Scope**: Variables declared at the top level or outside any function can be accessed from anywhere in the program.\n \n- **Main Local ", + "n_chunks": 5 + }, + { + "id": "GD-R-006", + "category": "RETRIEVAL", + "question": "How does concurrency work in AVAP? What are goroutines and how are they launched?", + "answer_preview": "Concurrency in AVAP is managed through the use of goroutines, which are lightweight threads that allow for efficient handling of multiple tasks concurrently. Goroutines are launched using the `go` keyword within a function or method.\n\nHere's an example of launching a goroutine:\n\n```avap\nfunc main() ", + "n_chunks": 5 + }, + { + "id": "GD-R-007", + "category": "RETRIEVAL", + "question": "What is the addParam command and how does it capture HTTP request parameters?", + "answer_preview": "The `addParam` command in AVAP inspects incoming HTTP requests in a hierarchical order: first from the URL (Query arguments), then from the JSON body, and finally from the form data. If the requested parameter does not exist, the variable destination is initialized as `None`.\n\nThis ensures that para", + "n_chunks": 5 + }, + { + "id": "GD-R-008", + "category": "RETRIEVAL", + "question": "How does the startLoop / endLoop construct work in AVAP?", + "answer_preview": "The `startLoop` and `endLoop` constructs are used to iterate over a sequence of items, such as an array or list, within the AVAP programming language. The purpose is to perform actions on each item in the sequence.\n\nIn the provided context:\n\n- The first snippet:\n ```avap\n [1] encontrado = False\n ", + "n_chunks": 5 + }, + { + "id": "GD-R-009", + "category": "RETRIEVAL", + "question": "What is the addResult command and how does it build the HTTP response?", + "answer_preview": "The `addResult` command in AVAP is used to add a result to an array or list. It takes one parameter, which is expected to be an object containing information about the result.\n\nTo generate an HTTP response using this command, you would typically pass an object that includes details such as the statu", + "n_chunks": 5 + }, + { + "id": "GD-R-010", + "category": "RETRIEVAL", + "question": "How does error handling work in AVAP with try() and exception()?", + "answer_preview": "In AVAP, error handling is managed using the `try()` block to attempt a specific operation or query. If an exception occurs during this process, it can be caught by the `exception(e)` statement.\n\nFor example:\n```avap\n[1] try()\n ormDirect(\"UPDATE table_inexistente SET a=1\", res)\nexception(e)\n a", + "n_chunks": 5 + }, + { + "id": "GD-R-011", + "category": "RETRIEVAL", + "question": "What is the replace() command in AVAP and how is it used?", + "answer_preview": "The `replace()` command in AVAP is not explicitly mentioned in the provided context. The context only lists a few operators like `and`, `or`, `not`, `in`, `is` and tokens such as `IDENTIFIER`, `INTEGER`, `FLOAT`, etc., but does not include any commands or functions that are commonly used for string ", + "n_chunks": 5 + }, + { + "id": "GD-R-012", + "category": "RETRIEVAL", + "question": "What are the reserved keywords in AVAP that cannot be used as identifiers?", + "answer_preview": "In AVAP, the following keywords are considered reserved and cannot be used as identifiers:\n\n- `and`\n- `or`\n- `not`\n- `in`\n- `is`\n\nThese keywords have specific meanings within the language and should not be used for naming variables or other identifiers.", + "n_chunks": 5 + }, + { + "id": "GD-R-013", + "category": "RETRIEVAL", + "question": "How does AVAP handle string formatting and concatenation?", + "answer_preview": "AVAP handles string formatting and concatenation using backticks (`) to enclose the expression that contains the formatted string. This allows for complex expressions involving multiple operations, comparisons, and access to data structures.\n\nFor example:\n```avap\nif(None, None, `user.name + \" is a f", + "n_chunks": 5 + }, + { + "id": "GD-R-014", + "category": "RETRIEVAL", + "question": "How does the encodeSHA256 command work in AVAP?", + "answer_preview": "The `encodeSHA256` command in AVAP is used to generate a SHA-256 hash of the provided payload data. This function takes the input string or binary data and returns its SHA-256 representation, which serves as a unique identifier for the data.\n\n```avap\nencodeSHA256(\"payload_data\", checksum)\n```\n\nThe `", + "n_chunks": 5 + }, + { + "id": "GD-R-015", + "category": "RETRIEVAL", + "question": "How does AVAP handle date and time operations?", + "answer_preview": "AVAP handles date and time operations using the `datetime` module in Python. This allows for easy manipulation of dates and times, including formatting, parsing, and arithmetic with other datetime objects.\n\nFor example:\n```python\nfrom datetime import datetime\n\n# Create a datetime object\ndate = datet", + "n_chunks": 5 + }, + { + "id": "GD-R-016", + "category": "RETRIEVAL", + "question": "What is the AddvariableToJSON command and how is it used to build JSON objects?", + "answer_preview": "The `AddVariableToJSON` command in AVAP allows you to dynamically add a new property to an existing JSON object. This function takes three parameters:\n\n1. **`clave`:** The key (property name) of the new property.\n2. **`valor`:** The value that will be added as the new property.\n3. **`json_destino`:*", + "n_chunks": 5 + }, + { + "id": "GD-R-017", + "category": "RETRIEVAL", + "question": "How does the getListLen command work and what is it used for?", + "answer_preview": "The `getListLen` command works by returning the length of a list. It takes no parameters and simply returns an integer representing the number of elements in the specified list.\n\nIt is typically used when you need to know how many items are present in a list, such as:\n\n```python\nmy_list = [1, 2, 3, ", + "n_chunks": 5 + }, + { + "id": "GD-R-018", + "category": "RETRIEVAL", + "question": "How does the randomString command work in AVAP?", + "answer_preview": "The `randomString` command generates a string of characters based on the specified parameters. In this case, it creates a 32-character string composed of uppercase letters and digits.\n\n```plaintext\n[1] randomString(\"[A-Z]\\d\", 32, token_seguridad)\naddResult(token_seguridad)\n```\n\nThe `randomString` co", + "n_chunks": 5 + }, + { + "id": "GD-R-019", + "category": "RETRIEVAL", + "question": "What is the $ dereference operator in AVAP and when is it used?", + "answer_preview": "The $ dereference operator in AVAP allows you to access the value of a variable by prefixing it with the dollar sign ($). This operator is typically used within expressions or functions where you need to directly reference the current state or content of a variable without needing to perform any ope", + "n_chunks": 5 + }, + { + "id": "GD-R-020", + "category": "RETRIEVAL", + "question": "How does AVAP handle ORM database operations? What commands are available?", + "answer_preview": "AVAP provides several commands for interacting with databases through the ORM (Object-Relational Mapping) system. The following commands and functionalities are available:\n\n1. **ormDirect(sentencia, destino)**: Executes a raw SQL command directly.\n2. **ormCheckTable(tableName, varTarget)**: Checks i", + "n_chunks": 5 + }, + { + "id": "GD-C-001", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'name' parameter and returns a personalized greeting.", + "answer_preview": "```avap\ninclude\n\nimport\n\nregisterEndpoint(\"/greet\", \"GET\", [], \"HELLO_WORLD\", main, result)\n\nresult = \"Hello,\" + name \naddResult(result)\n```\n\nIn this script, the `main` function is responsible for reading the 'name' parameter and returning a personalized greeting. The `result` variable holds the per", + "n_chunks": 5 + }, + { + "id": "GD-C-002", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'password' parameter, generates a SHA-256 hash, and returns it.", + "answer_preview": "```avap\nfunction generateSHA256Hash(passwordParam)\n total = encodeSHA256(passwordParam, \"new_hash\")\n return(total)\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-003", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that loops from 1 to 5, builds a JSON object with each index as a key, and returns it.", + "answer_preview": "```avap\n// StartLoop(0, 1, 6)\nstartLoop(index, 1, 6) {\n // Build the JSON object\n var json = {\"index\": index};\n \n // Return the built JSON object\n return json;\n}\nendLoop()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-004", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that validates if a 'role' parameter belongs to a list of allowed roles and returns the access result.", + "answer_preview": "```avap\naddParam(\"rol\", r)\nif(r, [\"admin\", \"editor\", \"root\"], \"in\")\n acceso = True\nelse\n acceso = False\nend()\naddResult(acceso)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-005", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that makes a GET request to an external API and handles connection errors.", + "answer_preview": "```avap\ntry() {\n ormDirect(\"GET /api/data\", res)\nexception(e) {\n addVar(_status, 500)\n addResult(\"Error al conectar con la API\")\nend()\n```\n\nThis script uses the `ormDirect` function to make a GET request to an external API. If there is a connection error (e.g., due to network issues), it wi", + "n_chunks": 5 + }, + { + "id": "GD-C-006", + "category": "CODE_GENERATION", + "question": "Write an AVAP function that takes two numbers and returns their sum, then call it and return the result.", + "answer_preview": "```avap\nfunction suma(a, b){\n total = a + b\n return(total)\n}\nresultado = suma(10, 20)\naddResult(resultado)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-007", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'subtotal' parameter, computes 21% VAT, and returns the total.", + "answer_preview": "```avap\nsubtotal = readParam(\"subtotal\")\niva = subtotal * 0.21\ntotal = subtotal + iva\nreturnResult(total)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-008", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads an 'api_key' parameter and returns status 403 if it is null.", + "answer_preview": "```avap\naddParam(\"api_key\", key)\nif(key, None, \"==\")\n addVar(_status, 403)\n addVar(error, \"Acceso denegado: falta API KEY\")\n addResult(error)\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-009", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that generates a 32-character random alphanumeric token and returns it.", + "answer_preview": "```avap\nfunction generateRandomToken() {\n let token = \"\";\n for (let i = 0; i < 32; i++) {\n const charType = Math.floor(Math.random() * 65) + 48;\n if (charType === 49) { // Digit\n token += String.fromCharCode(charType);\n } else { // Alphanumeric\n let r", + "n_chunks": 5 + }, + { + "id": "GD-C-010", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'lang' parameter and returns 'Hola' if it is 'es' or 'Hello' if it is 'en'.", + "answer_preview": "```avap\naddParam(\"lang\", \"l\")\nif(l, \"es\", \"=\")\n addVar(msg, \"Hola\")\nend()\nelse\n addVar(msg, \"Hello\")\nend()\naddResult(msg)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-011", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that checks if a database table exists and creates it if it does not.", + "answer_preview": "```avap\nif(table_exists(\"my_database.my_table\"), False, \"create_table\") {\n create_table(\"my_database.my_table\");\n} else {\n print(\"Table already exists.\");\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-012", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that gets the current UTC timestamp and adds 24 hours to compute an expiration time.", + "answer_preview": "```avap\ngetDateTime(\"\", 86400, \"UTC\", expira)\naddResult(expira)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-013", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that receives a new password parameter, validates it is not equal to the old password, and returns a confirmation.", + "answer_preview": "```avap\naddParam(password, pass_nueva)\npass_antigua = \"password\"\nif(pass_nueva != pass_antigua)\n addVar(cambio, \"Contraseña actualizada\")\nend()\naddResult(cambio)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-014", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a list parameter and returns its element count.", + "answer_preview": "```avap\ngetListLen(myList, len)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-015", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses a validation function to check a token parameter and returns the authorization result.", + "answer_preview": "```avap\nfunction es_valido(token) {\n response = False\n if (token == \"SECRET\") {\n response = True\n }\n return(response)\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-016", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that returns two values in the HTTP response: a status code 200 and a message 'Success'.", + "answer_preview": "```avap\naddVar(code, 200)\naddVar(status, \"Success\")\naddResult(code)\naddResult(status)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-017", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'saldo' parameter and returns True if it is greater than zero, False otherwise.", + "answer_preview": "```avap\nif(sal_par > 0, True, False)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-018", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that converts a Unix timestamp parameter to a human-readable date in dd/mm/yyyy format.", + "answer_preview": "```avap\nstampToDatetime(epoch, \"dd/MM/yyyy\", timedelta, destino)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-019", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that replaces all spaces in a string parameter with hyphens and returns the result.", + "answer_preview": "```avap\nreplaceSpacesWithHyphens(inputString)\n{\n var output = \"\";\n for (var i = 0; i < inputString.length; i++) {\n if (inputString[i] === \" \") {\n output += \"-\";\n } else {\n output += inputString[i];\n }\n }\n return output;\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-020", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses try/exception to execute a raw SQL query and return status 500 on database errors.", + "answer_preview": "```avap\ntry()\n ormDirect(\"UPDATE table_inexistente SET a=1\", res)\nexcept(e)\n addVar(_status, 500)\n addResult(\"Error de base de datos\")\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-V-001", + "category": "CONVERSATIONAL", + "question": "Can you summarize what you just explained about AVAP scopes in fewer words?", + "answer_preview": "AVAP uses three types of scopes: Global, Main Local, and Function. The Main Local scope is not visible within functions, and variables must be declared or exist in the visible scopes to avoid an execution error. Each function invocation creates a separate scope.", + "n_chunks": 5 + }, + { + "id": "GD-V-002", + "category": "CONVERSATIONAL", + "question": "You mentioned that addResult builds the JSON response — can you clarify how multiple addResult calls work together?", + "answer_preview": "Certainly! The `addResult` function is used to register variables that will form part of the final JSON body of the HTTP response. When multiple `addResult` calls are made, they each contribute their respective variable names and values to the overall JSON structure.\n\nFor example, if you have two `a", + "n_chunks": 5 + }, + { + "id": "GD-V-003", + "category": "CONVERSATIONAL", + "question": "What is the difference between addVar and a plain assignment like x = 10 in AVAP?", + "answer_preview": "In AVAP, `addVar(copia, $original)` is used to add a variable named `$original` to another variable named `copia`. This operation modifies the value of `copia`.\n\nOn the other hand, an assignment statement like `x = 10` simply assigns the integer value 10 to the variable `x`, without modifying any ex", + "n_chunks": 5 + }, + { + "id": "GD-V-004", + "category": "CONVERSATIONAL", + "question": "Can you explain again the difference between the two modes of the if() command?", + "answer_preview": "Certainly! The `if()` command in AVAP has two distinct modes, each with its own specific use cases and requirements:\n\n### Mode 1: Comparación Estructurada (Atómica)\nThis mode is used for direct comparisons between two simple values. It requires the following:\n- **Argument 1:** A simple identifier or", + "n_chunks": 5 + }, + { + "id": "GD-V-005", + "category": "CONVERSATIONAL", + "question": "What happens if an error occurs in AVAP without a try block?", + "answer_preview": "If an error occurs in AVAP without a `try` block, the program will not handle the exception and may continue executing or crash depending on how it is designed. The lack of a `try`-`except` structure means that any exceptions raised within the code will propagate to the next line of execution, poten", + "n_chunks": 5 + }, + { + "id": "GD-V-006", + "category": "CONVERSATIONAL", + "question": "Can you explain again how the timeout in RequestGet works?", + "answer_preview": "The `timeout` parameter in the `RequestGet` function is used to handle situations where a request might take longer than expected due to network latency. If the request exceeds the specified timeout, the variable `destino` receives the value `None`.\n\nThis ensures that if there's an issue with the co", + "n_chunks": 5 + }, + { + "id": "GD-V-007", + "category": "CONVERSATIONAL", + "question": "Can I iterate over a list of items in AVAP instead of a numeric range?", + "answer_preview": "Yes, you can iterate over a list of items in AVAP using the `itemFromList` command. This allows you to access elements by index within a list without needing to know the length beforehand.\n\nFor example:\n```avap\n// 1. Obtain the length of the list\ngetListLen(myList, len)\n\n// 2. Iterate with indexed c", + "n_chunks": 5 + }, + { + "id": "GD-V-008", + "category": "CONVERSATIONAL", + "question": "What is the difference between RequestGet and RequestPost in practice?", + "answer_preview": "In practice, `RequestPost` and `RequestGet` are used to interact with a server using HTTP methods POST and GET respectively. The main differences are:\n\n- **POST Method**: This method is typically used when you want to send data that needs to be processed on the server side. For example, it can be us", + "n_chunks": 5 + }, + { + "id": "GD-V-009", + "category": "CONVERSATIONAL", + "question": "Goroutines cannot access Main Local Scope — can you give a practical example of why that matters?", + "answer_preview": "Certainly! The concept of Goroutines in Go (which is similar to AVAP's Function Scope) refers to the ability for goroutines to run concurrently with other parts of the program. This means they do not have direct access to variables declared within the Main Local Scope.\n\nHere’s a practical example:\n\n", + "n_chunks": 5 + }, + { + "id": "GD-V-010", + "category": "CONVERSATIONAL", + "question": "What format does encodeSHA256 return its output in?", + "answer_preview": "The `encodeSHA256` function returns its output as a string of 64 characters hexadecimally. Specifically, it produces a digest that is 64 characters long and consists of hexadecimal digits. This ensures the highest possible resistance to cryptographic attacks compared to MD5, which has a shorter outp", + "n_chunks": 5 + } + ] + } +} \ No newline at end of file diff --git a/output/evaluation_bge-m3-latest_avap-knowledge-v2-bge_20260406_114111.json b/output/evaluation_bge-m3-latest_avap-knowledge-v2-bge_20260406_114111.json new file mode 100644 index 0000000..ccbdab2 --- /dev/null +++ b/output/evaluation_bge-m3-latest_avap-knowledge-v2-bge_20260406_114111.json @@ -0,0 +1,373 @@ +{ + "generated_at": "2026-04-06T11:41:11.688950", + "embedding_model": "bge-m3:latest", + "index_name": "avap-knowledge-v2-bge", + "result": { + "status": "ok", + "questions_evaluated": 50, + "elapsed_seconds": 427.0, + "judge_model": "claude-sonnet-4-20250514", + "index": "avap-knowledge-v2-bge", + "category_filter": "all", + "scores": { + "faithfulness": 0.5429, + "answer_relevancy": 0.8532, + "context_recall": 0.4989, + "context_precision": 0.5695 + }, + "global_score": 0.6161, + "verdict": "ACCEPTABLE", + "details": [ + { + "id": "GD-R-001", + "category": "RETRIEVAL", + "question": "What is AVAP and what is it designed for?", + "answer_preview": "AVAP (Advanced Virtual API Programming) is an Advanced Virtual API Programming language that is Turing Complete. It is designed to be used in the orquestation of microservices and I/O operations, with a focus on security, concurrency, determinism, and efficient processing of HTTP transactions, data ", + "n_chunks": 5 + }, + { + "id": "GD-R-002", + "category": "RETRIEVAL", + "question": "How does the if() conditional block work in AVAP? How are blocks closed?", + "answer_preview": "In AVAP, the `if()` conditional block evaluates a logical or comparison expression. It requires an explicit closing statement using the command `end()`.\n\nBlocks within the `if()` function are closed with the following structure:\n```plaintext\n\n *\n\n```\n\nThis is distinct from othe", + "n_chunks": 5 + }, + { + "id": "GD-R-003", + "category": "RETRIEVAL", + "question": "How does AVAP handle external HTTP calls? What commands are available and how is timeout managed?", + "answer_preview": "AVAP handles external HTTP calls through the use of the `RequestPost` and `RequestGet` functions. These functions execute POST and GET requests, respectively, storing the response in a destination variable.\n\nTimeout management in AVAP involves using the `timeout` parameter to specify the amount of t", + "n_chunks": 5 + }, + { + "id": "GD-R-004", + "category": "RETRIEVAL", + "question": "How do functions work in AVAP? What is the scope of variables inside a function?", + "answer_preview": "In AVAP, functions are enclosed within a recital of memory. When entering a function, AVAP creates a new dictionary of local variables isolated from the global context.\n\nVariables defined in the **Function Scope** can be accessed by other parts of the program or by other functions that call this one", + "n_chunks": 5 + }, + { + "id": "GD-R-005", + "category": "RETRIEVAL", + "question": "What are the three types of variable scopes in AVAP and what are their visibility rules?", + "answer_preview": "In AVAP, there are three types of variable scopes:\n\n1. **Global Scope**: Variables declared at the top level or outside any function.\n2. **Main Local Scope**: Variables declared within the `main` function.\n3. **Function Scope**: Variables declared inside a function.\n\nEach type has specific visibilit", + "n_chunks": 5 + }, + { + "id": "GD-R-006", + "category": "RETRIEVAL", + "question": "How does concurrency work in AVAP? What are goroutines and how are they launched?", + "answer_preview": "Concurrency in AVAP is managed through the concept of **goroutines**. Goroutines are lightweight threads that can be created using the `go` keyword, similar to creating a function call.\n\n```avap\ngo funcion()\n```\n\nGoroutines follow the same scope rules as normal functions:\n\n- They can access the **Gl", + "n_chunks": 5 + }, + { + "id": "GD-R-007", + "category": "RETRIEVAL", + "question": "What is the addParam command and how does it capture HTTP request parameters?", + "answer_preview": "The `addParam` command inspects a given HTTP request in an ordered, strict hierarchy: first in the URL (Query arguments), then in the JSON body, and finally in the form data. If the requested parameter does not exist, the variable destination is initialized as `None`.\n\nFor example:\n```plaintext\naddP", + "n_chunks": 5 + }, + { + "id": "GD-R-008", + "category": "RETRIEVAL", + "question": "How does the startLoop / endLoop construct work in AVAP?", + "answer_preview": "The `startLoop` and `endLoop` constructs are used to iterate over a range of indices, similar to Python's `range` function. The loop variable is incremented by 1 at each iteration.\n\n```plaintext\nstartLoop(idx, 0, 2)\n actual = registros[int(idx)]\nendLoop()\n```\n\nThis code snippet iterates over the ", + "n_chunks": 5 + }, + { + "id": "GD-R-009", + "category": "RETRIEVAL", + "question": "What is the addResult command and how does it build the HTTP response?", + "answer_preview": "The `addResult` command in AVAP is used to register variables that will form part of the JSON body of the final response. It allows for specifying both the status code directly (`_status = 404`) or through a function call (`addVar(_status, 401)`).\n\nIn the provided context, `addResult(acceso)` is cal", + "n_chunks": 5 + }, + { + "id": "GD-R-010", + "category": "RETRIEVAL", + "question": "How does error handling work in AVAP with try() and exception()?", + "answer_preview": "In AVAP, error handling is managed using the `try()` function. When an error occurs within a `try` block, it triggers the execution of the following code:\n\n```python\nexception(e)\n addVar(_status,500)\n addResult(\"Error de base de datos\")\n```\n\nThis sequence of actions happens when an exception i", + "n_chunks": 5 + }, + { + "id": "GD-R-011", + "category": "RETRIEVAL", + "question": "What is the replace() command in AVAP and how is it used?", + "answer_preview": "The `replace()` command in AVAP is a function that replaces occurrences of a specified substring with another substring within a string. It is commonly used to modify strings by replacing certain characters or patterns.\n\nIn the provided context, the `replace()` command is used as follows:\n\n1. The fi", + "n_chunks": 5 + }, + { + "id": "GD-R-012", + "category": "RETRIEVAL", + "question": "What are the reserved keywords in AVAP that cannot be used as identifiers?", + "answer_preview": "In AVAP, the following keywords are considered reserved and cannot be used as identifiers:\n\n- `_status`", + "n_chunks": 5 + }, + { + "id": "GD-R-013", + "category": "RETRIEVAL", + "question": "How does AVAP handle string formatting and concatenation?", + "answer_preview": "AVAP supports string formatting through the use of placeholders in expressions. For example, you can format strings by using curly braces `{}` with variables or expressions inside them:\n\n```plaintext\n\"Hello, {}! Your age is {} years old.\"\n```\n\nWhen this expression is evaluated, it will replace `{}` ", + "n_chunks": 5 + }, + { + "id": "GD-R-014", + "category": "RETRIEVAL", + "question": "How does the encodeSHA256 command work in AVAP?", + "answer_preview": "The `encodeSHA256` command in AVAP is used to generate a SHA-256 hash of a given payload data. This function takes two parameters: the payload data and a checksum, which are combined using the SHA-256 algorithm to produce a fixed-size output.\n\nHere's how it works:\n\n1. The `encodeSHA256` command acce", + "n_chunks": 5 + }, + { + "id": "GD-R-015", + "category": "RETRIEVAL", + "question": "How does AVAP handle date and time operations?", + "answer_preview": "AVAP provides three commands to cover all possible conversions between time representations. These commands support calendar formats in Python's `strftime` notation and calculations with `TimeDelta` expressed in seconds (positive for adding, negative for subtracting).\n\nThe context also mentions that", + "n_chunks": 5 + }, + { + "id": "GD-R-016", + "category": "RETRIEVAL", + "question": "What is the AddvariableToJSON command and how is it used to build JSON objects?", + "answer_preview": "The `AddvariableToJSON` command in this context is used to add variables to a JSON object. Specifically, it takes two parameters: `clave`, which specifies the key for the variable, and `datos_cliente`, which contains the data that will be added as a value under the specified key.\n\nFor example:\n```pl", + "n_chunks": 5 + }, + { + "id": "GD-R-017", + "category": "RETRIEVAL", + "question": "How does the getListLen command work and what is it used for?", + "answer_preview": "The getListLen command counts the number of elements in a list or string. It can be used to determine the length of any collection, such as a list of items or a string of characters.\n\nFor example:\n- The first command `getListLen(mi_lista, cantidad)` calculates the length of the list named `mi_lista`", + "n_chunks": 5 + }, + { + "id": "GD-R-018", + "category": "RETRIEVAL", + "question": "How does the randomString command work in AVAP?", + "answer_preview": "The `randomString` command generates a string of characters based on the specified parameters. Here's how it works:\n\n- `[1] randomString(\"[A-Z]\\d\", 32, token_seguridad)`: This command generates a 32-character string consisting only of uppercase letters (A-Z) and digits (0-9). The output is assigned ", + "n_chunks": 5 + }, + { + "id": "GD-R-019", + "category": "RETRIEVAL", + "question": "What is the $ dereference operator in AVAP and when is it used?", + "answer_preview": "The $ dereference operator in AVAP indicates that the value of a variable should be extracted from the table of symbols. This operator is utilized by the native command `addVar(copia, $original)`, where `$` signifies that the original variable's value should be retrieved.\n\nIn this context, it seems ", + "n_chunks": 5 + }, + { + "id": "GD-R-020", + "category": "RETRIEVAL", + "question": "How does AVAP handle ORM database operations? What commands are available?", + "answer_preview": "AVAP provides several commands for handling ORM database operations, including:\n\n- **ormCheckTable(tableName, varTarget)**: Checks the existence of a table in the connected database.\n- **ormCreateTable(fields, fieldsType, tableName, varTarget)**: DDL command to create tables.\n- **ormAccessSelect(fie", + "n_chunks": 5 + }, + { + "id": "GD-C-001", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'name' parameter and returns a personalized greeting.", + "answer_preview": "```avap\n/// personalizes a greeting based on the provided name\n\nfunction greet(name) {\n /// creates a personalized greeting message\n return \"Hello, \" + name + \"! How are you today?\";\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-002", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'password' parameter, generates a SHA-256 hash, and returns it.", + "answer_preview": "```avap\nreadParam(\"password\")\ngenerateSHA256Hash(password)\nreturn hashed_password\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-003", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that loops from 1 to 5, builds a JSON object with each index as a key, and returns it.", + "answer_preview": "```avap\nx = 10\ny = 20\n\n# Build the JSON object\njson_obj = {}\n\nfor i in range(1, 6):\n json_obj[i] = x + y\n\nreturn json_obj\n```\n\nThis script initializes two variables `x` and `y`, then builds a JSON object with each index from 1 to 5 as keys. The values are the sum of `x` and `y`.", + "n_chunks": 5 + }, + { + "id": "GD-C-004", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that validates if a 'role' parameter belongs to a list of allowed roles and returns the access result.", + "answer_preview": "```avap\n// Define the allowed roles as a list\nlet allowedRoles = [\"admin\", \"editor\", \"root\"];\n\n// Check if the provided role is in the allowed roles list\nif (allowedRoles.includes(rol)) {\n acceso = True;\n} else {\n acceso = False;\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-005", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that makes a GET request to an external API and handles connection errors.", + "answer_preview": "```avap\ntry()\n // Make the GET request to the external API\n var res = httpGet(\"https://example.com/api/resource\", {}, \"GET\")\nexception(e)\n addVar(_status, 500)\n addResult(\"Error al conectar con la API externa.\")\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-006", + "category": "CODE_GENERATION", + "question": "Write an AVAP function that takes two numbers and returns their sum, then call it and return the result.", + "answer_preview": "```plaintext\nfunction suma(a, b){\n total = a + b\n return(total)\n}\n\nresult = suma(5, 3)\nreturn(result)\n```\n\nThe `suma` function in AVAP adds two numbers together and returns their sum. When called with arguments `5` and `3`, it will return `8`.", + "n_chunks": 5 + }, + { + "id": "GD-C-007", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'subtotal' parameter, computes 21% VAT, and returns the total.", + "answer_preview": "```avap\nfunction computeTotal(subtotal){\n iva = subtotal * 0.21\n total = subtotal + iva\n return(total)\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-008", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads an 'api_key' parameter and returns status 403 if it is null.", + "answer_preview": "```avap\n/// checks for the presence of the api_key parameter\nif(api_key, None, \"==\")\n /// sets the status to 403\n addVar(_status, 403)\n /// sets an error message\n addVar(error, \"Acceso denegado: falta API KEY\")\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-009", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that generates a 32-character random alphanumeric token and returns it.", + "answer_preview": "```avap\ntoken = randomString(\"[A-Z]\\d\", 32, token_seguridad)\nreturn(token)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-010", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'lang' parameter and returns 'Hola' if it is 'es' or 'Hello' if it is 'en'.", + "answer_preview": "```avap\nif(l, \"es\", \"=\")\n addVar(msg, \"Hola\")\nend()\n\nif(l, \"en\", \"=\")\n addVar(msg, \"Hello\")\nend()\naddResult(msg)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-011", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that checks if a database table exists and creates it if it does not.", + "answer_preview": "```avap\nvarTarget = ormCheckTable(\"my_database\", \"my_table\")\nif varTarget == 0:\n # Table does not exist, create it\n varTarget = ormCreateTable([\"column1\", \"column2\"], [\"type1\", \"type2\"], \"my_database.my_table\", varTarget)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-012", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that gets the current UTC timestamp and adds 24 hours to compute an expiration time.", + "answer_preview": "```avap\ngetDateTime(\"\", 86400, \"UTC\", expira)\naddResult(expira)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-013", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that receives a new password parameter, validates it is not equal to the old password, and returns a confirmation.", + "answer_preview": "```avap\nconfirmPassword(newPass: string) {\n if (newPass !== pass_antigua) {\n return \"Confirmation: Password change successful.\";\n } else {\n return \"Error: New password cannot be the same as the old one.\";\n }\n}\n```\n\nEvidence:\n- \"\"\n(only quotes you actu", + "n_chunks": 5 + }, + { + "id": "GD-C-014", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a list parameter and returns its element count.", + "answer_preview": "```avap\n// 1. Obtain the length of the list\ngetListLen(myList, len)\n\n// 2. Return the element count\nlen\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-015", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses a validation function to check a token parameter and returns the authorization result.", + "answer_preview": "```avap\n// Validate the token using the es_valido function\nlet validatedToken = es_valido(\"SECRET\");\n\n// Check if the token is authorized\nif (validatedToken) {\n // Token is valid, proceed with further operations\n} else {\n // Token is not authorized, handle accordingly\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-016", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that returns two values in the HTTP response: a status code 200 and a message 'Success'.", + "answer_preview": "```avap\naddVar(code, 200)\naddVar(status, \"Success\")\naddResult(code)\naddResult(status)\n\n// Additional function to return a message\nfunction es_valido(token){\n response = False\n if(token, \"SECRET\", \"=\")\n response = True\n end()\n return(response)\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-017", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'saldo' parameter and returns True if it is greater than zero, False otherwise.", + "answer_preview": "```avap\n/// checks if the saldo is greater than zero\nif(saldo, 0, \">\")\n permitir = True\nelse()\n permitir = False\nend()\n\npermitir\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-018", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that converts a Unix timestamp parameter to a human-readable date in dd/mm/yyyy format.", + "answer_preview": "```avap\nstampToDatetime(1708726162, \"%d/%m/%Y\", 0, fecha_human)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-019", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that replaces all spaces in a string parameter with hyphens and returns the result.", + "answer_preview": "```avap\nreplace(\"spaces\", \" \", \"-\")\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-020", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses try/exception to execute a raw SQL query and return status 500 on database errors.", + "answer_preview": "```avap\ntry()\n ormDirect(\"UPDATE table_inexistente SET a=1\", res)\nexception(e)\n addVar(_status, 500)\n addResult(\"Error de base de datos\")\nend\n```", + "n_chunks": 5 + }, + { + "id": "GD-V-001", + "category": "CONVERSATIONAL", + "question": "Can you summarize what you just explained about AVAP scopes in fewer words?", + "answer_preview": "AVAP uses three types of scopes: Global Scope, Main Local Scope, and Function Scope. Variables are resolved from the closest to the farthest scope, with the Main Local Scope not visible within functions. If a variable isn't found in these scopes, it results in an execution error.", + "n_chunks": 5 + }, + { + "id": "GD-V-002", + "category": "CONVERSATIONAL", + "question": "You mentioned that addResult builds the JSON response — can you clarify how multiple addResult calls work together?", + "answer_preview": "Certainly! The `addResult` function is responsible for constructing and adding variables to the final JSON response. Each call to `addResult(variable)` adds a new variable to the JSON structure.\n\nFor example, if you have two `addResult` calls:\n1. `addResult(hashed_password)`\n2. `addResult(secure_tok", + "n_chunks": 5 + }, + { + "id": "GD-V-003", + "category": "CONVERSATIONAL", + "question": "What is the difference between addVar and a plain assignment like x = 10 in AVAP?", + "answer_preview": "In AVAP, `addVar` allows for direct variable assignment using the `=` operator, which provides flexibility while maintaining strict context control. This means that when you use `addVar(copia, $original)`, it assigns the value of `$original` to a variable named \"copia\". The key difference is that `a", + "n_chunks": 5 + }, + { + "id": "GD-V-004", + "category": "CONVERSATIONAL", + "question": "Can you explain again the difference between the two modes of the if() command?", + "answer_preview": "The `if()` command in AVAP has two different modes for its invocations:\n\n1. **Modo 1 (Comparación estructurada):** This mode is used when comparing two simple values directly, such as variables or literals. The syntax is:\n ```avap\n if(átomo_1, átomo_2, \"operador\")\n ```\n - **Argumentos 1 y 2:", + "n_chunks": 5 + }, + { + "id": "GD-V-005", + "category": "CONVERSATIONAL", + "question": "What happens if an error occurs in AVAP without a try block?", + "answer_preview": "If an error occurs in AVAP without a try block, the variable destination receives `None`. This is indicated by the comment: \"* **`RequestPost(url, querystring, headers, body, destino, timeout)`**: Ejecuta un POST almacenando la respuesta en `destino`. * Si se supera, la variable destino recibe `None", + "n_chunks": 5 + }, + { + "id": "GD-V-006", + "category": "CONVERSATIONAL", + "question": "Can you explain again how the timeout in RequestGet works?", + "answer_preview": "The `timeout` parameter in the `RequestGet` function is used to control the maximum amount of time that the request will wait for a response before it cancels the operation and returns `None`. This helps prevent potential issues where the server might take longer than expected to respond, leading to", + "n_chunks": 5 + }, + { + "id": "GD-V-007", + "category": "CONVERSATIONAL", + "question": "Can I iterate over a list of items in AVAP instead of a numeric range?", + "answer_preview": "Yes, you can iterate over a list of items in AVAP by using the `itemFromList` command. This allows you to access elements within the list without needing to know their indices beforehand.\n\nFor example:\n```avap\n// 1. Obtain the length of the list\ngetListLen(myList, len)\n\n// 2. Iterate with index-cont", + "n_chunks": 5 + }, + { + "id": "GD-V-008", + "category": "CONVERSATIONAL", + "question": "What is the difference between RequestGet and RequestPost in practice?", + "answer_preview": "In practice, **RequestGet** and **RequestPost** are used to interact with a web service or API by sending HTTP requests. The main differences between them are:\n\n- **RequestGet**: This method sends an HTTP GET request without including any body content (such as JSON or form data). It is typically use", + "n_chunks": 5 + }, + { + "id": "GD-V-009", + "category": "CONVERSATIONAL", + "question": "Goroutines cannot access Main Local Scope — can you give a practical example of why that matters?", + "answer_preview": "Certainly! The reason goroutines cannot directly access the `Main Local Scope` is crucial for managing resources and avoiding unintended side effects. Let's consider an example to illustrate this point:\n\n```python\nx = 10\ny = 20\n\n# This variable x is in the Main Local Scope, which means it can be acc", + "n_chunks": 5 + }, + { + "id": "GD-V-010", + "category": "CONVERSATIONAL", + "question": "What format does encodeSHA256 return its output in?", + "answer_preview": "The `encodeSHA256` function returns its output in the form of a hexadecimal digest. Specifically, it produces a 64-character hex-digest as specified by SHA-256. This ensures that the data is securely hashed and can be easily verified for integrity.", + "n_chunks": 5 + } + ] + } +} \ No newline at end of file diff --git a/output/evaluation_bge-m3-latest_avap-knowledge-v2-qwen_20260406_112342.json b/output/evaluation_bge-m3-latest_avap-knowledge-v2-qwen_20260406_112342.json new file mode 100644 index 0000000..66397d4 --- /dev/null +++ b/output/evaluation_bge-m3-latest_avap-knowledge-v2-qwen_20260406_112342.json @@ -0,0 +1,373 @@ +{ + "generated_at": "2026-04-06T11:23:42.728423", + "embedding_model": "bge-m3:latest", + "index_name": "avap-knowledge-v2-qwen", + "result": { + "status": "ok", + "questions_evaluated": 50, + "elapsed_seconds": 422.4, + "judge_model": "claude-sonnet-4-20250514", + "index": "avap-knowledge-v2-qwen", + "category_filter": "all", + "scores": { + "faithfulness": 0.4153, + "answer_relevancy": 0.8179, + "context_recall": 0.4377, + "context_precision": 0.4563 + }, + "global_score": 0.5318, + "verdict": "INSUFFICIENT", + "details": [ + { + "id": "GD-R-001", + "category": "RETRIEVAL", + "question": "What is AVAP and what is it designed for?", + "answer_preview": "AVAP is a new proprietary language. The provided context does not contain specific information about the purpose or design of AVAP. Therefore, I don't have enough information in the given context to answer that question.", + "n_chunks": 5 + }, + { + "id": "GD-R-002", + "category": "RETRIEVAL", + "question": "How does the if() conditional block work in AVAP? How are blocks closed?", + "answer_preview": "In AVAP, the `if()` conditional block evaluates a logical or comparison expression. The structure of an `if()` statement includes:\n\n1. **Modo 1 (Comparación estructurada):** `if(variable, value, comparison_operator)` — This mode compares the variable and value using the specified string operator (e.", + "n_chunks": 5 + }, + { + "id": "GD-R-003", + "category": "RETRIEVAL", + "question": "How does AVAP handle external HTTP calls? What commands are available and how is timeout managed?", + "answer_preview": "AVAP handles external HTTP calls through the `RequestPost` and `RequestGet` functions. These functions allow for making POST and GET requests, respectively, to an external URL with optional query parameters, headers, and body content.\n\nTimeout management in AVAP is handled by the `timeout` parameter", + "n_chunks": 5 + }, + { + "id": "GD-R-004", + "category": "RETRIEVAL", + "question": "How do functions work in AVAP? What is the scope of variables inside a function?", + "answer_preview": "In AVAP, functions operate within three scopes: Global Scope, Main Local Scope, and Function Scope. Variables defined outside these scopes are not visible within them. If a variable does not exist in any of the scopes, it results in an error during execution.\n\n- **Global Scope**: Variables declared ", + "n_chunks": 5 + }, + { + "id": "GD-R-005", + "category": "RETRIEVAL", + "question": "What are the three types of variable scopes in AVAP and what are their visibility rules?", + "answer_preview": "In AVAP, there are three types of variable scopes: Global Scope, Main Local Scope, and Function Scope. Each type has specific rules regarding visibility:\n\n1. **Global Scope**: Variables declared at this level can be accessed throughout the entire program.\n2. **Main Local Scope**: These variables are", + "n_chunks": 5 + }, + { + "id": "GD-R-006", + "category": "RETRIEVAL", + "question": "How does concurrency work in AVAP? What are goroutines and how are they launched?", + "answer_preview": "Concurrency in AVAP is achieved through the use of goroutines, which are lightweight threads that can be created using the `go` keyword. Goroutines allow for concurrent execution of multiple tasks without requiring explicit thread management.\n\nGoroutines follow the same scope rules as regular functi", + "n_chunks": 5 + }, + { + "id": "GD-R-007", + "category": "RETRIEVAL", + "question": "What is the addParam command and how does it capture HTTP request parameters?", + "answer_preview": "The `addParam(parametro, destino)` command inspects a given HTTP request in an ordered hierarchy: first in the URL (Query arguments), then in the JSON body, and finally in the form data. If the requested parameter does not exist, the variable destination is initialized as `None`.\n\nThis mechanism ens", + "n_chunks": 5 + }, + { + "id": "GD-R-008", + "category": "RETRIEVAL", + "question": "How does the startLoop / endLoop construct work in AVAP?", + "answer_preview": "The `startLoop` and `endLoop` constructs are used to iterate over a sequence of elements. In this context, it appears that the loop is iterating from 1 to 10 (inclusive) and checking if the element at index 5 equals \"==\". If it does, then the variable \"encontrado\" is set to True and the loop exits e", + "n_chunks": 5 + }, + { + "id": "GD-R-009", + "category": "RETRIEVAL", + "question": "What is the addResult command and how does it build the HTTP response?", + "answer_preview": "The `addResult` command in AVAP is used to register which variables will form part of the final JSON body of the response. It allows for specifying the status code of the output, either through direct assignment (`_status = 404`) or by using a more flexible method with `addVar(_status, 401)`.\n\nThe c", + "n_chunks": 5 + }, + { + "id": "GD-R-010", + "category": "RETRIEVAL", + "question": "How does error handling work in AVAP with try() and exception()?", + "answer_preview": "In AVAP, error handling is managed using the `try()` function to attempt a specific operation or query. If an exception occurs during this process, it can be caught by the `exception(e)` statement.\n\nFor example:\n```avap\n[1] try()\n ormDirect(\"UPDATE table_inexistente SET a=1\", res)\nexception(e)\n ", + "n_chunks": 5 + }, + { + "id": "GD-R-011", + "category": "RETRIEVAL", + "question": "What is the replace() command in AVAP and how is it used?", + "answer_preview": "The `replace()` command in AVAP is a function used to modify strings by replacing occurrences of a specified substring with another string. It takes three parameters: the first parameter specifies the original string, the second parameter is the substring to be replaced, and the third parameter is t", + "n_chunks": 5 + }, + { + "id": "GD-R-012", + "category": "RETRIEVAL", + "question": "What are the reserved keywords in AVAP that cannot be used as identifiers?", + "answer_preview": "In AVAP, the following keywords are considered reserved and cannot be used as identifiers:\n\n- _status\n\nThese keywords have specific meanings within the language and should not be used for variable names or other purposes.", + "n_chunks": 5 + }, + { + "id": "GD-R-013", + "category": "RETRIEVAL", + "question": "How does AVAP handle string formatting and concatenation?", + "answer_preview": "AVAP supports string formatting through the use of placeholders in expressions. For example, you can format strings by using curly braces `{}` with variables or expressions inside them.\n\n```regex\n{variable}\n```\n\nThis allows for dynamic string manipulation where parts of a string are replaced with va", + "n_chunks": 5 + }, + { + "id": "GD-R-014", + "category": "RETRIEVAL", + "question": "How does the encodeSHA256 command work in AVAP?", + "answer_preview": "The `encodeSHA256` command in AVAP is used to generate a SHA-256 hash of a given payload data. This function takes two parameters: the payload data and a checksum, which are combined using the SHA-256 algorithm to produce a fixed-size output.\n\nHere's how it works:\n\n1. The `encodeSHA256` command acce", + "n_chunks": 5 + }, + { + "id": "GD-R-015", + "category": "RETRIEVAL", + "question": "How does AVAP handle date and time operations?", + "answer_preview": "AVAP handles date and time operations through the use of a specific syntax for representing dates and times. The context provided includes a regular expression definition for strings containing quoted values, which is used to parse and validate input data.\n\nFor example:\n- A string like `\"2023-10-05\"", + "n_chunks": 5 + }, + { + "id": "GD-R-016", + "category": "RETRIEVAL", + "question": "What is the AddvariableToJSON command and how is it used to build JSON objects?", + "answer_preview": "The AddvariableToJSON command takes three parameters: a key (clave), an array of values (datos_cliente), and a final JSON object (mi_json_final). It adds each value from datos_cliente into mi_json_final under the specified key. This process builds up a complex JSON structure by iterating through the", + "n_chunks": 5 + }, + { + "id": "GD-R-017", + "category": "RETRIEVAL", + "question": "How does the getListLen command work and what is it used for?", + "answer_preview": "The getListLen command in AVAP returns the length of a list or array. It takes two parameters: \"mi_lista\" (the name of the list) and \"cantidad\" (the desired length). The function then calculates the number of elements in the specified list.\n\nFor example, if you call getListLen(mi_lista, cantidad), i", + "n_chunks": 5 + }, + { + "id": "GD-R-018", + "category": "RETRIEVAL", + "question": "How does the randomString command work in AVAP?", + "answer_preview": "The `randomString` command generates a string of characters based on the specified parameters. In this case, it creates a 32-character string consisting only of uppercase letters and digits.\n\n- The first parameter `[A-Z]\\d`, where `[A-Z]` represents any uppercase letter (a-z) and `\\d` represents any", + "n_chunks": 5 + }, + { + "id": "GD-R-019", + "category": "RETRIEVAL", + "question": "What is the $ dereference operator in AVAP and when is it used?", + "answer_preview": "The `$` dereference operator in AVAP is used to access elements of an array or object. It allows you to retrieve a specific element by providing its index.\n\nFor example, if `copia` is an array and `$original` is the original value that was copied into `copia`, then `addVar(copia, $original)` would a", + "n_chunks": 5 + }, + { + "id": "GD-R-020", + "category": "RETRIEVAL", + "question": "How does AVAP handle ORM database operations? What commands are available?", + "answer_preview": "AVAP handles ORM database operations through the `` command. This command specifies whether the adapter is an ORM of a third-party proxy, determined by passing a UUID as an argument.\n\nThe set of commands available in AVAP includes integrations for manipulating complex types ", + "n_chunks": 5 + }, + { + "id": "GD-C-001", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'name' parameter and returns a personalized greeting.", + "answer_preview": "```avap\nif name:\n greeting = \"Hello, \" + name + \". Welcome!\"\nelse:\n greeting = \"Hello, user. Welcome!\"\n\nreturn greeting\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-002", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'password' parameter, generates a SHA-256 hash, and returns it.", + "answer_preview": "```avap\naddParam(\"password\", password)\nencodeSHA256(password, hashed_password)\nreturn hashed_password\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-003", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that loops from 1 to 5, builds a JSON object with each index as a key, and returns it.", + "answer_preview": "```avap\n{\n \"script\": {\n \"name\": \"LoopAndBuildJSON\",\n \"parameters\": [],\n \"body\": [\n \"0\",\n \"1\",\n \"2\",\n \"3\",\n \"4\"\n ],\n \"returnType\": \"json_object\",\n \"function\": \"buildJsonObject\"\n }\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-004", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that validates if a 'role' parameter belongs to a list of allowed roles and returns the access result.", + "answer_preview": "```avap\nif(rol, [\"admin\", \"editor\", \"root\"], \"in\")\n acceso = True\nelse\n acceso = False\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-005", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that makes a GET request to an external API and handles connection errors.", + "answer_preview": "```avap\ntry()\n // Make a GET request to the external API\n var res = httpGet(\"https://example.com/api/data\", \"GET\")\nexception(e)\n addVar(_status, 500)\n addResult(\"Error al conectar con la API externa\")\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-006", + "category": "CODE_GENERATION", + "question": "Write an AVAP function that takes two numbers and returns their sum, then call it and return the result.", + "answer_preview": "```AVAP\nfunction suma(a, b){\n total = a + b\n return(total)\n}\n\nresult = suma(5, 7)\nreturn(result)\n```\n\nThe `suma` function adds two integers `a` and `b`, returning their sum. When called with arguments `5` and `7`, it returns the result of `12`.", + "n_chunks": 5 + }, + { + "id": "GD-C-007", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'subtotal' parameter, computes 21% VAT, and returns the total.", + "answer_preview": "```avap\nsubtotal = 150.50\niva = subtotal * 0.21\ntotal = subtotal + iva\n\nreturn(total)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-008", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads an 'api_key' parameter and returns status 403 if it is null.", + "answer_preview": "```avap\naddParam(\"api_key\", key)\nif(key, None, \"==\")\n addVar(_status, 403)\n addVar(error, \"Acceso denegado: falta API KEY\")\n addResult(error)\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-009", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that generates a 32-character random alphanumeric token and returns it.", + "answer_preview": "```avap\nrandomString(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\", 32, \"token\")\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-010", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'lang' parameter and returns 'Hola' if it is 'es' or 'Hello' if it is 'en'.", + "answer_preview": "```avap\nif(l, \"es\", \"=\")\n addVar(msg, \"Hola\")\nend()\n\nif(l, \"en\", \"=\")\n addVar(msg, \"Hello\")\nend()\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-011", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that checks if a database table exists and creates it if it does not.", + "answer_preview": "```avap\nif (ormCheckTable(\"my_table\", \"varTarget\")) {\n // Table already exists, do nothing\n} else {\n ormCreateTable([\"column1\", \"column2\"], [\"type1\", \"type2\"], \"my_table\", \"varTarget\");\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-012", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that gets the current UTC timestamp and adds 24 hours to compute an expiration time.", + "answer_preview": "```regex\nINTEGER ::= [0-9]+\n\ngetDateTime(\"\", 86400, \"UTC\", expira)\naddResult(expira)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-013", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that receives a new password parameter, validates it is not equal to the old password, and returns a confirmation.", + "answer_preview": "```avap\nvalidatePassword(new_password) {\n if (new_password != old_password) {\n return \"Confirmation: Password validation successful.\"\n } else {\n return \"Confirmation: Old password detected.\"\n }\n}\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-014", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a list parameter and returns its element count.", + "answer_preview": "```avap\ngetListLen(fuente, destino)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-015", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses a validation function to check a token parameter and returns the authorization result.", + "answer_preview": "```regex\nIDENTIFIER ::= [a-zA-Z_][a-zA-Z0-9_]*\nINTEGER ::= [0-9]+\nFLOAT ::= [0-9]+\\.[0-9]*([eE][-+]?[0-9]+)?\nSTRING ::= \\\"[^\\\"]*\\\"|\\'[^\\']*\\'\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-016", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that returns two values in the HTTP response: a status code 200 and a message 'Success'.", + "answer_preview": "```avap\naddVar(code, 200)\naddVar(status, \"Success\")\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-017", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that reads a 'saldo' parameter and returns True if it is greater than zero, False otherwise.", + "answer_preview": "```avap\nTrue\nFalse\nNone\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-018", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that converts a Unix timestamp parameter to a human-readable date in dd/mm/yyyy format.", + "answer_preview": "```avap\nstampToDatetime(epoch, \"yyyy-mm-dd\", timedelta, destino)\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-019", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that replaces all spaces in a string parameter with hyphens and returns the result.", + "answer_preview": "```avap\nreplace(\"spaces\", \" \", \"-\")\n```", + "n_chunks": 5 + }, + { + "id": "GD-C-020", + "category": "CODE_GENERATION", + "question": "Write an AVAP script that uses try/exception to execute a raw SQL query and return status 500 on database errors.", + "answer_preview": "```avap\ntry()\n ormDirect(\"UPDATE table_inexistente SET a=1\", res)\nexcept(e)\n addVar(_status, 500)\n addResult(\"Error de base de datos\")\nend\n\naddVar(code, 200)\naddVar(status, \"Success\")\naddResult(code)\naddResult(status)\n```", + "n_chunks": 5 + }, + { + "id": "GD-V-001", + "category": "CONVERSATIONAL", + "question": "Can you summarize what you just explained about AVAP scopes in fewer words?", + "answer_preview": "AVAP uses three types of scopes: Global Scope, Main Local Scope, and Function Scope. Each scope has strict visibility rules. Variables are resolved from the main local scope to global scope, with errors occurring if a variable is not found in visible scopes or within the function itself.", + "n_chunks": 5 + }, + { + "id": "GD-V-002", + "category": "CONVERSATIONAL", + "question": "You mentioned that addResult builds the JSON response — can you clarify how multiple addResult calls work together?", + "answer_preview": "Certainly! The `addResult` function is used to append results to a JSON response. Each call to `addResult` adds one piece of information to the JSON structure being built.\n\nFor example, in context [2], we have:\n\n```plaintext\n[2] randomString(\"[A-Z]\\d\", 32, token_seguridad)\naddResult(token_seguridad)", + "n_chunks": 5 + }, + { + "id": "GD-V-003", + "category": "CONVERSATIONAL", + "question": "What is the difference between addVar and a plain assignment like x = 10 in AVAP?", + "answer_preview": "In AVAP, `addVar` is used to assign values to system variables. Specifically:\n\n- **addVar**:\n - Takes two arguments: a variable name (``) and an expression value.\n - This function is designed for assigning values to predefined system variables such as `_status`.\n \n- **Plain Assignment", + "n_chunks": 5 + }, + { + "id": "GD-V-004", + "category": "CONVERSATIONAL", + "question": "Can you explain again the difference between the two modes of the if() command?", + "answer_preview": "The if() command in AVAP has two different modes for its invocations:\n\n1. **Modo 1 (Comparación estructurada):** This mode is used to compare two simple values directly, such as strings or numbers. The syntax is `if(átomo_1, átomo_2, \"operador\")`, where:\n - `átomo_1` and `átomo_2` are either simpl", + "n_chunks": 5 + }, + { + "id": "GD-V-005", + "category": "CONVERSATIONAL", + "question": "What happens if an error occurs in AVAP without a try block?", + "answer_preview": "If an error occurs in AVAP without a corresponding try block, the program will not handle it gracefully. The provided code snippet shows that when an exception is thrown (e.g., due to a database access failure), the `exception` statement is called. This results in setting `_status` to 403 and adding", + "n_chunks": 5 + }, + { + "id": "GD-V-006", + "category": "CONVERSATIONAL", + "question": "Can you explain again how the timeout in RequestGet works?", + "answer_preview": "The `timeout` parameter in the `RequestGet` function is used to control the maximum amount of time that the request will wait for a response before it cancels the operation and returns `None`. This helps prevent potential issues where the server might take longer than expected to respond, causing th", + "n_chunks": 5 + }, + { + "id": "GD-V-007", + "category": "CONVERSATIONAL", + "question": "Can I iterate over a list of items in AVAP instead of a numeric range?", + "answer_preview": "Yes, you can iterate over a list of items in AVAP by using the `while` loop structure provided. The code snippet shows how to achieve this:\n\n```avap\n// 1. Obtener longitud de la lista\ngetListLen(myList, len)\n\n// 2. Iterar con índice controlado\ni = 0\nwhile (i < len) {\n itemFromList(myList, i, curr", + "n_chunks": 5 + }, + { + "id": "GD-V-008", + "category": "CONVERSATIONAL", + "question": "What is the difference between RequestGet and RequestPost in practice?", + "answer_preview": "In practice, `RequestGet` and `RequestPost` are used to interact with a server using HTTP methods. The main differences between them are:\n\n- **Method Type**: \n - `RequestGet`: This method is typically used for GET requests, which retrieve data from the server without sending any additional informat", + "n_chunks": 5 + }, + { + "id": "GD-V-009", + "category": "CONVERSATIONAL", + "question": "Goroutines cannot access Main Local Scope — can you give a practical example of why that matters?", + "answer_preview": "Certainly! When using goroutines in Go (which is based on the AVAP language), it's important to understand how they interact with the main program and its local variables. Here’s an example scenario where accessing the `Main Local Scope` would be problematic:\n\n```go\npackage main\n\nimport (\n\t\"fmt\"\n)\n\n", + "n_chunks": 5 + }, + { + "id": "GD-V-010", + "category": "CONVERSATIONAL", + "question": "What format does encodeSHA256 return its output in?", + "answer_preview": "The `encodeSHA256` function returns its output as a checksum.", + "n_chunks": 5 + } + ] + } +} \ No newline at end of file diff --git a/scratches/pseco/agent/n00 Run Evaluation.ipynb b/scratches/pseco/agent/n00 Run Evaluation.ipynb index 2c3b615..5715fd6 100644 --- a/scratches/pseco/agent/n00 Run Evaluation.ipynb +++ b/scratches/pseco/agent/n00 Run Evaluation.ipynb @@ -45,12 +45,12 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "2e2ceb6f", "metadata": {}, "outputs": [], "source": [ - "emb = [\"bge-m3:latest\", \"bge-m3:latest\"]\n", + "emb = [\"harrier-oss-v1:0.6b\"]\n", "index = [\"avap-knowledge-v2-qwen\", \"avap-docs-test-v4\",\"avap-knowledge-v2-bge\", \"avap-docs-test-v4-bge\"]" ] }, @@ -109,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "ad8d341e", "metadata": {}, "outputs": [ @@ -141,8 +141,163 @@ "name": "stderr", "output_type": "stream", "text": [ - "Exception raised in Job[45]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n" + "Exception raised in Job[45]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n", + "Exception raised in Job[97]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n", + "Exception raised in Job[117]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n", + "Exception raised in Job[121]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n", + "Exception raised in Job[137]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n", + "Exception raised in Job[153]: ResponseError(failed to encode response: json: unsupported value: NaN (status code: 500))\n" ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved: /home/pseco/VsCodeProjects/assistance-engine/output/evaluation_bge-m3-latest_avap-knowledge-v2-qwen_20260406_112342.json\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[eval] embed_query fails: failed to encode response: json: unsupported value: NaN (status code: 500)\n", + "[eval] embed_query fails: failed to encode response: json: unsupported value: NaN (status code: 500)\n", + "[eval] embed_query fails: failed to encode response: json: unsupported value: NaN (status code: 500)\n", + "[eval] embed_query fails: failed to encode response: json: unsupported value: NaN (status code: 500)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "554aa243310546fc99980133ad1db56e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Evaluating: 0%| | 0/200 [00:00