Merge branch 'mrh-online-dev' of github.com:BRUNIX-AI/assistance-engine into mrh-online-dev
This commit is contained in:
pseco
commit
80cdbcc38e
|
|
@ -0,0 +1,228 @@
|
||||||
|
start: program
|
||||||
|
|
||||||
|
program: separator* line_or_comment (separator+ line_or_comment)* separator*
|
||||||
|
|
||||||
|
?line_or_comment: simple_stmt comment?
|
||||||
|
| compound_stmt
|
||||||
|
| comment
|
||||||
|
| BLOCK_COMMENT
|
||||||
|
|
||||||
|
?separator: EOL+
|
||||||
|
|
||||||
|
comment: DOC_COMMENT | LINE_COMMENT
|
||||||
|
|
||||||
|
EOL: /\r?\n/
|
||||||
|
|
||||||
|
DOC_COMMENT.2: /\/\/\/[^\r\n]*/
|
||||||
|
LINE_COMMENT.1: /\/\/[^\r\n]*/
|
||||||
|
BLOCK_COMMENT: /\/\*[\s\S]*?\*\//
|
||||||
|
|
||||||
|
?simple_stmt: assignment
|
||||||
|
| return_stmt
|
||||||
|
| system_command
|
||||||
|
| io_command
|
||||||
|
| async_command
|
||||||
|
| connector_cmd
|
||||||
|
| db_command
|
||||||
|
| http_command
|
||||||
|
| util_command
|
||||||
|
| modularity_cmd
|
||||||
|
| call_stmt
|
||||||
|
|
||||||
|
?compound_stmt: function_decl
|
||||||
|
| if_stmt
|
||||||
|
| loop_stmt
|
||||||
|
| try_stmt
|
||||||
|
|
||||||
|
assignment: identifier "=" expression
|
||||||
|
|
||||||
|
call_stmt: identifier "(" argument_list? ")"
|
||||||
|
| identifier "=" identifier "." identifier "(" argument_list? ")"
|
||||||
|
| identifier "." identifier "(" argument_list? ")"
|
||||||
|
|
||||||
|
system_command: register_cmd
|
||||||
|
| addvar_cmd
|
||||||
|
|
||||||
|
register_cmd: "registerEndpoint" "(" stringliteral "," stringliteral "," list_display "," stringliteral "," identifier "," identifier ")"
|
||||||
|
|
||||||
|
addvar_cmd: "addVar" "(" addvar_arg "," addvar_arg ")"
|
||||||
|
|
||||||
|
addvar_arg: identifier
|
||||||
|
| literal
|
||||||
|
| "$" identifier
|
||||||
|
|
||||||
|
identifier: IDENTIFIER
|
||||||
|
|
||||||
|
system_variable: "_status"
|
||||||
|
|
||||||
|
io_command: addparam_cmd
|
||||||
|
| getlistlen_cmd
|
||||||
|
| addresult_cmd
|
||||||
|
| getparamlist_cmd
|
||||||
|
|
||||||
|
addparam_cmd: "addParam" "(" stringliteral "," identifier ")"
|
||||||
|
getlistlen_cmd: "getListLen" "(" identifier "," identifier ")"
|
||||||
|
getparamlist_cmd: "getQueryParamList" "(" stringliteral "," identifier ")"
|
||||||
|
addresult_cmd: "addResult" "(" identifier ")"
|
||||||
|
|
||||||
|
if_stmt: "if" "(" if_condition ")" separator block ("else" "(" ")" separator block)? "end" "(" ")"
|
||||||
|
|
||||||
|
if_condition: if_atom "," if_atom "," stringliteral
|
||||||
|
| "None" "," "None" "," stringliteral
|
||||||
|
|
||||||
|
if_atom: identifier
|
||||||
|
| literal
|
||||||
|
|
||||||
|
loop_stmt: "startLoop" "(" identifier "," expression "," expression ")" separator block "endLoop" "(" ")"
|
||||||
|
|
||||||
|
try_stmt: "try" "(" ")" separator block "exception" "(" identifier ")" separator block "end" "(" ")"
|
||||||
|
|
||||||
|
block: separator* line_or_comment (separator+ line_or_comment)* separator*
|
||||||
|
|
||||||
|
async_command: go_stmt
|
||||||
|
| gather_stmt
|
||||||
|
|
||||||
|
go_stmt: identifier "=" "go" identifier "(" argument_list? ")"
|
||||||
|
gather_stmt: identifier "=" "gather" "(" identifier ("," expression)? ")"
|
||||||
|
|
||||||
|
connector_cmd: connector_instantiation
|
||||||
|
|
||||||
|
connector_instantiation: identifier "=" "avapConnector" "(" stringliteral ")"
|
||||||
|
|
||||||
|
http_command: req_post_cmd
|
||||||
|
| req_get_cmd
|
||||||
|
|
||||||
|
req_post_cmd: "RequestPost" "(" expression "," expression "," expression "," expression "," identifier "," expression ")"
|
||||||
|
req_get_cmd: "RequestGet" "(" expression "," expression "," expression "," identifier "," expression ")"
|
||||||
|
|
||||||
|
db_command: orm_direct
|
||||||
|
| orm_check
|
||||||
|
| orm_create
|
||||||
|
| orm_select
|
||||||
|
| orm_insert
|
||||||
|
| orm_update
|
||||||
|
|
||||||
|
orm_direct: "ormDirect" "(" expression "," identifier ")"
|
||||||
|
orm_check: "ormCheckTable" "(" expression "," identifier ")"
|
||||||
|
orm_create: "ormCreateTable" "(" expression "," expression "," expression "," identifier ")"
|
||||||
|
|
||||||
|
orm_select: "ormAccessSelect" "(" orm_fields "," expression ("," expression)? "," identifier ")"
|
||||||
|
|
||||||
|
orm_fields: "*"
|
||||||
|
| expression
|
||||||
|
|
||||||
|
orm_insert: "ormAccessInsert" "(" expression "," expression "," identifier ")"
|
||||||
|
orm_update: "ormAccessUpdate" "(" expression "," expression "," expression "," expression "," identifier ")"
|
||||||
|
|
||||||
|
util_command: json_list_cmd
|
||||||
|
| crypto_cmd
|
||||||
|
| regex_cmd
|
||||||
|
| datetime_cmd
|
||||||
|
| stamp_cmd
|
||||||
|
| string_cmd
|
||||||
|
| replace_cmd
|
||||||
|
|
||||||
|
json_list_cmd: "variableToList" "(" expression "," identifier ")"
|
||||||
|
| "itemFromList" "(" identifier "," expression "," identifier ")"
|
||||||
|
| "variableFromJSON" "(" identifier "," expression "," identifier ")"
|
||||||
|
| "AddVariableToJSON" "(" expression "," expression "," identifier ")"
|
||||||
|
|
||||||
|
crypto_cmd: "encodeSHA256" "(" identifier_or_string "," identifier ")"
|
||||||
|
| "encodeMD5" "(" identifier_or_string "," identifier ")"
|
||||||
|
|
||||||
|
regex_cmd: "getRegex" "(" identifier "," stringliteral "," identifier ")"
|
||||||
|
|
||||||
|
datetime_cmd: "getDateTime" "(" stringliteral "," expression "," stringliteral "," identifier ")"
|
||||||
|
|
||||||
|
stamp_cmd: "stampToDatetime" "(" expression "," stringliteral "," expression "," identifier ")"
|
||||||
|
| "getTimeStamp" "(" stringliteral "," stringliteral "," expression "," identifier ")"
|
||||||
|
|
||||||
|
string_cmd: "randomString" "(" expression "," expression "," identifier ")"
|
||||||
|
|
||||||
|
replace_cmd: "replace" "(" identifier_or_string "," stringliteral "," stringliteral "," identifier ")"
|
||||||
|
|
||||||
|
function_decl: "function" identifier "(" param_list? ")" "{" separator block "}"
|
||||||
|
|
||||||
|
param_list: identifier ("," identifier)*
|
||||||
|
|
||||||
|
return_stmt: "return" "(" expression? ")"
|
||||||
|
|
||||||
|
modularity_cmd: include_stmt
|
||||||
|
| import_stmt
|
||||||
|
|
||||||
|
include_stmt: "include" stringliteral
|
||||||
|
import_stmt: "import" ("<" identifier ">" | stringliteral)
|
||||||
|
|
||||||
|
?expression: logical_or
|
||||||
|
|
||||||
|
?logical_or: logical_and ("or" logical_and)*
|
||||||
|
?logical_and: logical_not ("and" logical_not)*
|
||||||
|
|
||||||
|
?logical_not: "not" logical_not
|
||||||
|
| comparison
|
||||||
|
|
||||||
|
?comparison: arithmetic (comp_op arithmetic)*
|
||||||
|
|
||||||
|
comp_op: "==" | "!=" | "<" | ">" | "<=" | ">=" | "in" | "is"
|
||||||
|
|
||||||
|
?arithmetic: term (("+" | "-") term)*
|
||||||
|
?term: factor (("*" | "/" | "%") factor)*
|
||||||
|
|
||||||
|
?factor: ("+" | "-") factor
|
||||||
|
| power
|
||||||
|
|
||||||
|
?power: primary ("**" factor)?
|
||||||
|
|
||||||
|
?primary: atom postfix*
|
||||||
|
|
||||||
|
postfix: "." identifier
|
||||||
|
| "[" expression "]"
|
||||||
|
| "[" expression? ":" expression? (":" expression?)? "]"
|
||||||
|
| "(" argument_list? ")"
|
||||||
|
|
||||||
|
?atom: identifier
|
||||||
|
| "$" identifier
|
||||||
|
| literal
|
||||||
|
| "(" expression ")"
|
||||||
|
| list_display
|
||||||
|
| dict_display
|
||||||
|
|
||||||
|
list_display: "[" argument_list? "]"
|
||||||
|
| "[" expression "for" identifier "in" expression if_clause? "]"
|
||||||
|
|
||||||
|
if_clause: "if" expression
|
||||||
|
|
||||||
|
dict_display: "{" key_datum_list? "}"
|
||||||
|
|
||||||
|
key_datum_list: key_datum ("," key_datum)*
|
||||||
|
key_datum: expression ":" expression
|
||||||
|
|
||||||
|
argument_list: expression ("," expression)*
|
||||||
|
|
||||||
|
number: FLOATNUMBER
|
||||||
|
| INTEGER
|
||||||
|
|
||||||
|
literal: stringliteral
|
||||||
|
| number
|
||||||
|
| boolean
|
||||||
|
| "None"
|
||||||
|
|
||||||
|
boolean: "True" | "False"
|
||||||
|
|
||||||
|
INTEGER: /[0-9]+/
|
||||||
|
FLOATNUMBER: /(?:[0-9]+\.[0-9]*|\.[0-9]+)/
|
||||||
|
|
||||||
|
stringliteral: STRING_DOUBLE
|
||||||
|
| STRING_SINGLE
|
||||||
|
|
||||||
|
# STRING_DOUBLE: /"([^"\\]|\\["'\\ntr0])*"/
|
||||||
|
# STRING_SINGLE: /'([^'\\]|\\["'\\ntr0])*'/
|
||||||
|
STRING_DOUBLE: /"([^"\\]|\\.)*"/
|
||||||
|
STRING_SINGLE: /'([^'\\]|\\.)*'/
|
||||||
|
|
||||||
|
identifier_or_string: identifier
|
||||||
|
| stringliteral
|
||||||
|
|
||||||
|
IDENTIFIER: /[A-Za-z_][A-Za-z0-9_]*/
|
||||||
|
|
||||||
|
%ignore /[ \t]+/
|
||||||
60390
ingestion/chunks.json
60390
ingestion/chunks.json
File diff suppressed because one or more lines are too long
|
|
@ -14,12 +14,14 @@ dependencies = [
|
||||||
"langchain-elasticsearch>=1.0.0",
|
"langchain-elasticsearch>=1.0.0",
|
||||||
"langchain-huggingface>=1.2.0",
|
"langchain-huggingface>=1.2.0",
|
||||||
"langchain-ollama>=1.0.1",
|
"langchain-ollama>=1.0.1",
|
||||||
|
"llama-cpp-python>=0.3.16",
|
||||||
"loguru>=0.7.3",
|
"loguru>=0.7.3",
|
||||||
"nltk>=3.9.2",
|
"nltk>=3.9.2",
|
||||||
"numpy>=2.4.2",
|
"numpy>=2.4.2",
|
||||||
"pandas>=3.0.0",
|
"pandas>=3.0.0",
|
||||||
"python-dotenv>=1.2.1",
|
"python-dotenv>=1.2.1",
|
||||||
"rapidfuzz>=3.14.3",
|
"rapidfuzz>=3.14.3",
|
||||||
|
"tenacity>=9.1.4",
|
||||||
"tqdm>=4.67.3",
|
"tqdm>=4.67.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,609 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"id": "9d524159",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import pprint\n",
|
||||||
|
"\n",
|
||||||
|
"from llama_cpp.llama import Llama, LlamaGrammar\n",
|
||||||
|
"\n",
|
||||||
|
"from src.config import settings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"id": "330f1975",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"bnf = []\n",
|
||||||
|
"\n",
|
||||||
|
"for file_path in (settings.proj_root / \"ingestion/code/BNF/\").glob(\"*.txt\"):\n",
|
||||||
|
" with file_path.open(\"r\", encoding=\"utf-8\") as file:\n",
|
||||||
|
" bnf.append(file.read())\n",
|
||||||
|
"\n",
|
||||||
|
"bnf = \"\\n\".join(bnf)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"id": "68887173",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"grammar_string = r'''\n",
|
||||||
|
"root ::= program\n",
|
||||||
|
"\n",
|
||||||
|
"program ::= (line | block-comment)*\n",
|
||||||
|
"\n",
|
||||||
|
"line ::= statement-with-comment eol | trailing-comment eol | eol\n",
|
||||||
|
"statement-with-comment ::= statement trailing-comment? | statement?\n",
|
||||||
|
"trailing-comment ::= line-comment | doc-comment\n",
|
||||||
|
"\n",
|
||||||
|
"eol ::= \"\\r\\n\" | \"\\n\"\n",
|
||||||
|
"\n",
|
||||||
|
"doc-comment ::= \"///\" any-text\n",
|
||||||
|
"line-comment ::= \"//\" any-text\n",
|
||||||
|
"block-comment ::= \"/*\" block-comment-body \"*/\"\n",
|
||||||
|
"\n",
|
||||||
|
"any-text ::= any-text-char*\n",
|
||||||
|
"any-text-char ::= [^\\r\\n]\n",
|
||||||
|
"\n",
|
||||||
|
"block-comment-body ::= block-comment-part*\n",
|
||||||
|
"block-comment-part ::= [^*] | \"*\" [^/]\n",
|
||||||
|
"\n",
|
||||||
|
"statement ::= assignment | method-call-stmt | function-call-stmt | function-decl | return-stmt | system-command | io-command | control-flow | async-command | connector-cmd | db-command | http-command | util-command | modularity-cmd\n",
|
||||||
|
"\n",
|
||||||
|
"assignment ::= identifier \"=\" expression\n",
|
||||||
|
"\n",
|
||||||
|
"function-call-stmt ::= identifier \"(\" argument-list? \")\"\n",
|
||||||
|
"method-call-stmt ::= identifier \"=\" identifier \".\" identifier \"(\" argument-list? \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"system-command ::= register-cmd | addvar-cmd\n",
|
||||||
|
"register-cmd ::= \"registerEndpoint(\" stringliteral \",\" stringliteral \",\" list-display \",\" stringliteral \",\" identifier \",\" identifier \")\"\n",
|
||||||
|
"addvar-cmd ::= \"addVar(\" addvar-arg \",\" addvar-arg \")\"\n",
|
||||||
|
"addvar-arg ::= identifier | literal | \"$\" identifier\n",
|
||||||
|
"\n",
|
||||||
|
"system-variable ::= \"_status\"\n",
|
||||||
|
"\n",
|
||||||
|
"io-command ::= addparam-cmd | getlistlen-cmd | addresult-cmd | getparamlist-cmd\n",
|
||||||
|
"addparam-cmd ::= \"addParam(\" stringliteral \",\" identifier \")\"\n",
|
||||||
|
"getlistlen-cmd ::= \"getListLen(\" identifier \",\" identifier \")\"\n",
|
||||||
|
"getparamlist-cmd ::= \"getQueryParamList(\" stringliteral \",\" identifier \")\"\n",
|
||||||
|
"addresult-cmd ::= \"addResult(\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"control-flow ::= if-stmt | loop-stmt | try-stmt\n",
|
||||||
|
"\n",
|
||||||
|
"if-stmt ::= \"if(\" if-condition \")\" eol block else-clause? \"end()\" eol\n",
|
||||||
|
"else-clause ::= \"else()\" eol block\n",
|
||||||
|
"if-condition ::= if-atom \",\" if-atom \",\" stringliteral | \"None\" \",\" \"None\" \",\" stringliteral\n",
|
||||||
|
"if-atom ::= identifier | literal\n",
|
||||||
|
"\n",
|
||||||
|
"loop-stmt ::= \"startLoop(\" identifier \",\" expression \",\" expression \")\" eol block \"endLoop()\" eol\n",
|
||||||
|
"\n",
|
||||||
|
"try-stmt ::= \"try()\" eol block \"exception(\" identifier \")\" eol block \"end()\" eol\n",
|
||||||
|
"\n",
|
||||||
|
"block ::= line*\n",
|
||||||
|
"\n",
|
||||||
|
"async-command ::= go-stmt | gather-stmt\n",
|
||||||
|
"go-stmt ::= identifier \"=\" \"go\" identifier \"(\" argument-list? \")\"\n",
|
||||||
|
"gather-stmt ::= identifier \"=\" \"gather(\" identifier gather-tail? \")\"\n",
|
||||||
|
"gather-tail ::= \",\" expression\n",
|
||||||
|
"\n",
|
||||||
|
"connector-cmd ::= connector-instantiation | connector-method-call\n",
|
||||||
|
"connector-instantiation ::= identifier \"=\" \"avapConnector(\" stringliteral \")\"\n",
|
||||||
|
"connector-method-call ::= connector-method-assignment? identifier \".\" identifier \"(\" argument-list? \")\"\n",
|
||||||
|
"connector-method-assignment ::= identifier \"=\"\n",
|
||||||
|
"\n",
|
||||||
|
"http-command ::= req-post-cmd | req-get-cmd\n",
|
||||||
|
"req-post-cmd ::= \"RequestPost(\" expression \",\" expression \",\" expression \",\" expression \",\" identifier \",\" expression \")\"\n",
|
||||||
|
"req-get-cmd ::= \"RequestGet(\" expression \",\" expression \",\" expression \",\" identifier \",\" expression \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"db-command ::= orm-direct | orm-check | orm-create | orm-select | orm-insert | orm-update\n",
|
||||||
|
"orm-direct ::= \"ormDirect(\" expression \",\" identifier \")\"\n",
|
||||||
|
"orm-check ::= \"ormCheckTable(\" expression \",\" identifier \")\"\n",
|
||||||
|
"orm-create ::= \"ormCreateTable(\" expression \",\" expression \",\" expression \",\" identifier \")\"\n",
|
||||||
|
"orm-select ::= \"ormAccessSelect(\" orm-fields \",\" expression orm-select-tail? \",\" identifier \")\"\n",
|
||||||
|
"orm-select-tail ::= \",\" expression\n",
|
||||||
|
"orm-fields ::= \"*\" | expression\n",
|
||||||
|
"orm-insert ::= \"ormAccessInsert(\" expression \",\" expression \",\" identifier \")\"\n",
|
||||||
|
"orm-update ::= \"ormAccessUpdate(\" expression \",\" expression \",\" expression \",\" expression \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"util-command ::= json-list-cmd | crypto-cmd | regex-cmd | datetime-cmd | stamp-cmd | string-cmd | replace-cmd\n",
|
||||||
|
"\n",
|
||||||
|
"json-list-cmd ::= \"variableToList(\" expression \",\" identifier \")\" | \"itemFromList(\" identifier \",\" expression \",\" identifier \")\" | \"variableFromJSON(\" identifier \",\" expression \",\" identifier \")\" | \"AddVariableToJSON(\" expression \",\" expression \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"crypto-cmd ::= \"encodeSHA256(\" identifier-or-string \",\" identifier \")\" | \"encodeMD5(\" identifier-or-string \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"regex-cmd ::= \"getRegex(\" identifier \",\" stringliteral \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"datetime-cmd ::= \"getDateTime(\" stringliteral \",\" expression \",\" stringliteral \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"stamp-cmd ::= \"stampToDatetime(\" expression \",\" stringliteral \",\" expression \",\" identifier \")\" | \"getTimeStamp(\" stringliteral \",\" stringliteral \",\" expression \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"string-cmd ::= \"randomString(\" expression \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"replace-cmd ::= \"replace(\" identifier-or-string \",\" stringliteral \",\" stringliteral \",\" identifier \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"function-decl ::= \"function\" identifier \"(\" param-list? \")\" \"{\" eol block \"}\" eol\n",
|
||||||
|
"param-list ::= identifier (\",\" identifier)*\n",
|
||||||
|
"\n",
|
||||||
|
"return-stmt ::= \"return(\" expression? \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"modularity-cmd ::= include-stmt | import-stmt\n",
|
||||||
|
"include-stmt ::= \"include\" \" \" stringliteral\n",
|
||||||
|
"import-stmt ::= \"import\" \" \" (\"<\" identifier \">\" | stringliteral)\n",
|
||||||
|
"\n",
|
||||||
|
"expression ::= logical-or\n",
|
||||||
|
"\n",
|
||||||
|
"logical-or ::= logical-and (\"or\" logical-and)*\n",
|
||||||
|
"logical-and ::= logical-not (\"and\" logical-not)*\n",
|
||||||
|
"logical-not ::= \"not\" logical-not | comparison\n",
|
||||||
|
"\n",
|
||||||
|
"comparison ::= arithmetic (comp-op arithmetic)*\n",
|
||||||
|
"comp-op ::= \"==\" | \"!=\" | \"<=\" | \">=\" | \"<\" | \">\" | \"in\" | \"is\"\n",
|
||||||
|
"\n",
|
||||||
|
"arithmetic ::= term (add-op term)*\n",
|
||||||
|
"add-op ::= \"+\" | \"-\"\n",
|
||||||
|
"\n",
|
||||||
|
"term ::= factor (mul-op factor)*\n",
|
||||||
|
"mul-op ::= \"*\" | \"/\" | \"%\"\n",
|
||||||
|
"\n",
|
||||||
|
"factor ::= unary-op factor | power\n",
|
||||||
|
"unary-op ::= \"+\" | \"-\"\n",
|
||||||
|
"\n",
|
||||||
|
"power ::= primary power-tail?\n",
|
||||||
|
"power-tail ::= \"**\" factor\n",
|
||||||
|
"\n",
|
||||||
|
"primary ::= atom postfix-part*\n",
|
||||||
|
"postfix-part ::= \".\" identifier | \"[\" subscript \"]\" | \"(\" argument-list? \")\"\n",
|
||||||
|
"\n",
|
||||||
|
"subscript ::= expression | slice-expr\n",
|
||||||
|
"slice-expr ::= expression? \":\" expression? slice-step?\n",
|
||||||
|
"slice-step ::= \":\" expression?\n",
|
||||||
|
"\n",
|
||||||
|
"atom ::= identifier | \"$\" identifier | literal | \"(\" expression \")\" | list-display | dict-display\n",
|
||||||
|
"\n",
|
||||||
|
"list-display ::= \"[\" list-display-body? \"]\"\n",
|
||||||
|
"list-display-body ::= list-comprehension | argument-list\n",
|
||||||
|
"list-comprehension ::= expression \"for\" identifier \"in\" expression if-clause?\n",
|
||||||
|
"if-clause ::= \"if\" expression\n",
|
||||||
|
"\n",
|
||||||
|
"dict-display ::= \"{\" key-datum-list? \"}\"\n",
|
||||||
|
"key-datum-list ::= key-datum (\",\" key-datum)*\n",
|
||||||
|
"key-datum ::= expression \":\" expression\n",
|
||||||
|
"\n",
|
||||||
|
"argument-list ::= expression (\",\" expression)*\n",
|
||||||
|
"\n",
|
||||||
|
"literal ::= stringliteral | number | boolean | \"None\"\n",
|
||||||
|
"boolean ::= \"True\" | \"False\"\n",
|
||||||
|
"\n",
|
||||||
|
"number ::= floatnumber | integer\n",
|
||||||
|
"integer ::= digit+\n",
|
||||||
|
"floatnumber ::= digit+ \".\" digit* | \".\" digit+\n",
|
||||||
|
"digit ::= [0-9]\n",
|
||||||
|
"\n",
|
||||||
|
"stringliteral ::= \"\\\"\" text-double \"\\\"\" | \"'\" text-single \"'\"\n",
|
||||||
|
"escape-sequence ::= \"\\\\\" escape-char\n",
|
||||||
|
"escape-char ::= \"\\\"\" | \"'\" | \"\\\\\" | \"n\" | \"t\" | \"r\" | \"0\"\n",
|
||||||
|
"\n",
|
||||||
|
"text-double ::= (text-double-char | escape-sequence)*\n",
|
||||||
|
"text-single ::= (text-single-char | escape-sequence)*\n",
|
||||||
|
"\n",
|
||||||
|
"text-double-char ::= [^\"\\\\]\n",
|
||||||
|
"text-single-char ::= [^'\\\\]\n",
|
||||||
|
"\n",
|
||||||
|
"identifier-or-string ::= identifier | stringliteral\n",
|
||||||
|
"\n",
|
||||||
|
"identifier ::= ident-start ident-rest*\n",
|
||||||
|
"ident-start ::= [A-Za-z_]\n",
|
||||||
|
"ident-rest ::= [A-Za-z0-9_]\n",
|
||||||
|
"'''\n",
|
||||||
|
"grammar = LlamaGrammar.from_string(grammar_string)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"id": "e693a3fa",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"llama_model_loader: loaded meta data with 33 key-value pairs and 338 tensors from /home/acano/PycharmProjects/assistance-engine/data/models/qwen2.5-coder-1.5b-q8_0.gguf (version GGUF V3 (latest))\n",
|
||||||
|
"llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
|
||||||
|
"llama_model_loader: - kv 0: general.architecture str = qwen2\n",
|
||||||
|
"llama_model_loader: - kv 1: general.type str = model\n",
|
||||||
|
"llama_model_loader: - kv 2: general.name str = Qwen2.5 Coder 1.5B\n",
|
||||||
|
"llama_model_loader: - kv 3: general.basename str = Qwen2.5-Coder\n",
|
||||||
|
"llama_model_loader: - kv 4: general.size_label str = 1.5B\n",
|
||||||
|
"llama_model_loader: - kv 5: general.license str = apache-2.0\n",
|
||||||
|
"llama_model_loader: - kv 6: general.license.link str = https://huggingface.co/Qwen/Qwen2.5-C...\n",
|
||||||
|
"llama_model_loader: - kv 7: general.base_model.count u32 = 1\n",
|
||||||
|
"llama_model_loader: - kv 8: general.base_model.0.name str = Qwen2.5 1.5B\n",
|
||||||
|
"llama_model_loader: - kv 9: general.base_model.0.organization str = Qwen\n",
|
||||||
|
"llama_model_loader: - kv 10: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen2.5-1.5B\n",
|
||||||
|
"llama_model_loader: - kv 11: general.tags arr[str,5] = [\"code\", \"qwen\", \"qwen-coder\", \"codeq...\n",
|
||||||
|
"llama_model_loader: - kv 12: general.languages arr[str,1] = [\"en\"]\n",
|
||||||
|
"llama_model_loader: - kv 13: qwen2.block_count u32 = 28\n",
|
||||||
|
"llama_model_loader: - kv 14: qwen2.context_length u32 = 32768\n",
|
||||||
|
"llama_model_loader: - kv 15: qwen2.embedding_length u32 = 1536\n",
|
||||||
|
"llama_model_loader: - kv 16: qwen2.feed_forward_length u32 = 8960\n",
|
||||||
|
"llama_model_loader: - kv 17: qwen2.attention.head_count u32 = 12\n",
|
||||||
|
"llama_model_loader: - kv 18: qwen2.attention.head_count_kv u32 = 2\n",
|
||||||
|
"llama_model_loader: - kv 19: qwen2.rope.freq_base f32 = 1000000.000000\n",
|
||||||
|
"llama_model_loader: - kv 20: qwen2.attention.layer_norm_rms_epsilon f32 = 0.000001\n",
|
||||||
|
"llama_model_loader: - kv 21: general.file_type u32 = 7\n",
|
||||||
|
"llama_model_loader: - kv 22: tokenizer.ggml.model str = gpt2\n",
|
||||||
|
"llama_model_loader: - kv 23: tokenizer.ggml.pre str = qwen2\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"llama_model_loader: - kv 24: tokenizer.ggml.tokens arr[str,151936] = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
|
||||||
|
"llama_model_loader: - kv 25: tokenizer.ggml.token_type arr[i32,151936] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
|
||||||
|
"llama_model_loader: - kv 26: tokenizer.ggml.merges arr[str,151387] = [\"Ġ Ġ\", \"ĠĠ ĠĠ\", \"i n\", \"Ġ t\",...\n",
|
||||||
|
"llama_model_loader: - kv 27: tokenizer.ggml.eos_token_id u32 = 151643\n",
|
||||||
|
"llama_model_loader: - kv 28: tokenizer.ggml.padding_token_id u32 = 151643\n",
|
||||||
|
"llama_model_loader: - kv 29: tokenizer.ggml.bos_token_id u32 = 151643\n",
|
||||||
|
"llama_model_loader: - kv 30: tokenizer.ggml.add_bos_token bool = false\n",
|
||||||
|
"llama_model_loader: - kv 31: tokenizer.chat_template str = {%- if tools %}\\n {{- '<|im_start|>...\n",
|
||||||
|
"llama_model_loader: - kv 32: general.quantization_version u32 = 2\n",
|
||||||
|
"llama_model_loader: - type f32: 141 tensors\n",
|
||||||
|
"llama_model_loader: - type q8_0: 197 tensors\n",
|
||||||
|
"print_info: file format = GGUF V3 (latest)\n",
|
||||||
|
"print_info: file type = Q8_0\n",
|
||||||
|
"print_info: file size = 1.53 GiB (8.50 BPW) \n",
|
||||||
|
"init_tokenizer: initializing tokenizer for type 2\n",
|
||||||
|
"load: control token: 151660 '<|fim_middle|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151659 '<|fim_prefix|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151653 '<|vision_end|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151648 '<|box_start|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151646 '<|object_ref_start|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151649 '<|box_end|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151655 '<|image_pad|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151651 '<|quad_end|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151647 '<|object_ref_end|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151652 '<|vision_start|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151654 '<|vision_pad|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151656 '<|video_pad|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151644 '<|im_start|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151661 '<|fim_suffix|>' is not marked as EOG\n",
|
||||||
|
"load: control token: 151650 '<|quad_start|>' is not marked as EOG\n",
|
||||||
|
"load: printing all EOG tokens:\n",
|
||||||
|
"load: - 151643 ('<|endoftext|>')\n",
|
||||||
|
"load: - 151645 ('<|im_end|>')\n",
|
||||||
|
"load: - 151662 ('<|fim_pad|>')\n",
|
||||||
|
"load: - 151663 ('<|repo_name|>')\n",
|
||||||
|
"load: - 151664 ('<|file_sep|>')\n",
|
||||||
|
"load: special tokens cache size = 22\n",
|
||||||
|
"load: token to piece cache size = 0.9310 MB\n",
|
||||||
|
"print_info: arch = qwen2\n",
|
||||||
|
"print_info: vocab_only = 0\n",
|
||||||
|
"print_info: n_ctx_train = 32768\n",
|
||||||
|
"print_info: n_embd = 1536\n",
|
||||||
|
"print_info: n_layer = 28\n",
|
||||||
|
"print_info: n_head = 12\n",
|
||||||
|
"print_info: n_head_kv = 2\n",
|
||||||
|
"print_info: n_rot = 128\n",
|
||||||
|
"print_info: n_swa = 0\n",
|
||||||
|
"print_info: is_swa_any = 0\n",
|
||||||
|
"print_info: n_embd_head_k = 128\n",
|
||||||
|
"print_info: n_embd_head_v = 128\n",
|
||||||
|
"print_info: n_gqa = 6\n",
|
||||||
|
"print_info: n_embd_k_gqa = 256\n",
|
||||||
|
"print_info: n_embd_v_gqa = 256\n",
|
||||||
|
"print_info: f_norm_eps = 0.0e+00\n",
|
||||||
|
"print_info: f_norm_rms_eps = 1.0e-06\n",
|
||||||
|
"print_info: f_clamp_kqv = 0.0e+00\n",
|
||||||
|
"print_info: f_max_alibi_bias = 0.0e+00\n",
|
||||||
|
"print_info: f_logit_scale = 0.0e+00\n",
|
||||||
|
"print_info: f_attn_scale = 0.0e+00\n",
|
||||||
|
"print_info: n_ff = 8960\n",
|
||||||
|
"print_info: n_expert = 0\n",
|
||||||
|
"print_info: n_expert_used = 0\n",
|
||||||
|
"print_info: causal attn = 1\n",
|
||||||
|
"print_info: pooling type = -1\n",
|
||||||
|
"print_info: rope type = 2\n",
|
||||||
|
"print_info: rope scaling = linear\n",
|
||||||
|
"print_info: freq_base_train = 1000000.0\n",
|
||||||
|
"print_info: freq_scale_train = 1\n",
|
||||||
|
"print_info: n_ctx_orig_yarn = 32768\n",
|
||||||
|
"print_info: rope_finetuned = unknown\n",
|
||||||
|
"print_info: model type = 1.5B\n",
|
||||||
|
"print_info: model params = 1.54 B\n",
|
||||||
|
"print_info: general.name = Qwen2.5 Coder 1.5B\n",
|
||||||
|
"print_info: vocab type = BPE\n",
|
||||||
|
"print_info: n_vocab = 151936\n",
|
||||||
|
"print_info: n_merges = 151387\n",
|
||||||
|
"print_info: BOS token = 151643 '<|endoftext|>'\n",
|
||||||
|
"print_info: EOS token = 151643 '<|endoftext|>'\n",
|
||||||
|
"print_info: EOT token = 151645 '<|im_end|>'\n",
|
||||||
|
"print_info: PAD token = 151643 '<|endoftext|>'\n",
|
||||||
|
"print_info: LF token = 198 'Ċ'\n",
|
||||||
|
"print_info: FIM PRE token = 151659 '<|fim_prefix|>'\n",
|
||||||
|
"print_info: FIM SUF token = 151661 '<|fim_suffix|>'\n",
|
||||||
|
"print_info: FIM MID token = 151660 '<|fim_middle|>'\n",
|
||||||
|
"print_info: FIM PAD token = 151662 '<|fim_pad|>'\n",
|
||||||
|
"print_info: FIM REP token = 151663 '<|repo_name|>'\n",
|
||||||
|
"print_info: FIM SEP token = 151664 '<|file_sep|>'\n",
|
||||||
|
"print_info: EOG token = 151643 '<|endoftext|>'\n",
|
||||||
|
"print_info: EOG token = 151645 '<|im_end|>'\n",
|
||||||
|
"print_info: EOG token = 151662 '<|fim_pad|>'\n",
|
||||||
|
"print_info: EOG token = 151663 '<|repo_name|>'\n",
|
||||||
|
"print_info: EOG token = 151664 '<|file_sep|>'\n",
|
||||||
|
"print_info: max token length = 256\n",
|
||||||
|
"load_tensors: loading model tensors, this can take a while... (mmap = true)\n",
|
||||||
|
"load_tensors: layer 0 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 1 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 2 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 3 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 4 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 5 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 6 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 7 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 8 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 9 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 10 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 11 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 12 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 13 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 14 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 15 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 16 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 17 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 18 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 19 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 20 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 21 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 22 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 23 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 24 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 25 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 26 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 27 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: layer 28 assigned to device CPU, is_swa = 0\n",
|
||||||
|
"load_tensors: tensor 'token_embd.weight' (q8_0) (and 338 others) cannot be used with preferred buffer type CPU_REPACK, using CPU instead\n",
|
||||||
|
"load_tensors: CPU_Mapped model buffer size = 1564.62 MiB\n",
|
||||||
|
"......................................................................................\n",
|
||||||
|
"llama_context: constructing llama_context\n",
|
||||||
|
"llama_context: n_seq_max = 1\n",
|
||||||
|
"llama_context: n_ctx = 512\n",
|
||||||
|
"llama_context: n_ctx_per_seq = 512\n",
|
||||||
|
"llama_context: n_batch = 512\n",
|
||||||
|
"llama_context: n_ubatch = 512\n",
|
||||||
|
"llama_context: causal_attn = 1\n",
|
||||||
|
"llama_context: flash_attn = 0\n",
|
||||||
|
"llama_context: kv_unified = false\n",
|
||||||
|
"llama_context: freq_base = 1000000.0\n",
|
||||||
|
"llama_context: freq_scale = 1\n",
|
||||||
|
"llama_context: n_ctx_per_seq (512) < n_ctx_train (32768) -- the full capacity of the model will not be utilized\n",
|
||||||
|
"set_abort_callback: call\n",
|
||||||
|
"llama_context: CPU output buffer size = 0.58 MiB\n",
|
||||||
|
"create_memory: n_ctx = 512 (padded)\n",
|
||||||
|
"llama_kv_cache_unified: layer 0: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 1: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 2: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 3: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 4: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 5: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 6: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 7: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 8: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 9: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 10: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 11: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 12: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 13: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 14: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 15: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 16: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 17: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 18: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 19: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 20: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 21: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 22: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 23: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 24: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 25: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 26: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: layer 27: dev = CPU\n",
|
||||||
|
"llama_kv_cache_unified: CPU KV buffer size = 14.00 MiB\n",
|
||||||
|
"llama_kv_cache_unified: size = 14.00 MiB ( 512 cells, 28 layers, 1/1 seqs), K (f16): 7.00 MiB, V (f16): 7.00 MiB\n",
|
||||||
|
"llama_context: enumerating backends\n",
|
||||||
|
"llama_context: backend_ptrs.size() = 1\n",
|
||||||
|
"llama_context: max_nodes = 2704\n",
|
||||||
|
"llama_context: worst-case: n_tokens = 512, n_seqs = 1, n_outputs = 0\n",
|
||||||
|
"graph_reserve: reserving a graph for ubatch with n_tokens = 512, n_seqs = 1, n_outputs = 512\n",
|
||||||
|
"graph_reserve: reserving a graph for ubatch with n_tokens = 1, n_seqs = 1, n_outputs = 1\n",
|
||||||
|
"graph_reserve: reserving a graph for ubatch with n_tokens = 512, n_seqs = 1, n_outputs = 512\n",
|
||||||
|
"llama_context: CPU compute buffer size = 299.75 MiB\n",
|
||||||
|
"llama_context: graph nodes = 1070\n",
|
||||||
|
"llama_context: graph splits = 1\n",
|
||||||
|
"CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | \n",
|
||||||
|
"Model metadata: {'tokenizer.ggml.bos_token_id': '151643', 'general.file_type': '7', 'qwen2.attention.layer_norm_rms_epsilon': '0.000001', 'tokenizer.ggml.eos_token_id': '151643', 'qwen2.rope.freq_base': '1000000.000000', 'qwen2.attention.head_count': '12', 'general.quantization_version': '2', 'tokenizer.ggml.model': 'gpt2', 'qwen2.feed_forward_length': '8960', 'general.architecture': 'qwen2', 'tokenizer.ggml.padding_token_id': '151643', 'qwen2.embedding_length': '1536', 'general.basename': 'Qwen2.5-Coder', 'tokenizer.ggml.add_bos_token': 'false', 'general.base_model.0.organization': 'Qwen', 'tokenizer.ggml.pre': 'qwen2', 'general.name': 'Qwen2.5 Coder 1.5B', 'general.base_model.0.name': 'Qwen2.5 1.5B', 'qwen2.block_count': '28', 'general.type': 'model', 'general.size_label': '1.5B', 'tokenizer.chat_template': '{%- if tools %}\\n {{- \\'<|im_start|>system\\\\n\\' }}\\n {%- if messages[0][\\'role\\'] == \\'system\\' %}\\n {{- messages[0][\\'content\\'] }}\\n {%- else %}\\n {{- \\'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.\\' }}\\n {%- endif %}\\n {{- \"\\\\n\\\\n# Tools\\\\n\\\\nYou may call one or more functions to assist with the user query.\\\\n\\\\nYou are provided with function signatures within <tools></tools> XML tags:\\\\n<tools>\" }}\\n {%- for tool in tools %}\\n {{- \"\\\\n\" }}\\n {{- tool | tojson }}\\n {%- endfor %}\\n {{- \"\\\\n</tools>\\\\n\\\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\\\n<tool_call>\\\\n{\\\\\"name\\\\\": <function-name>, \\\\\"arguments\\\\\": <args-json-object>}\\\\n</tool_call><|im_end|>\\\\n\" }}\\n{%- else %}\\n {%- if messages[0][\\'role\\'] == \\'system\\' %}\\n {{- \\'<|im_start|>system\\\\n\\' + messages[0][\\'content\\'] + \\'<|im_end|>\\\\n\\' }}\\n {%- else %}\\n {{- \\'<|im_start|>system\\\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\\\n\\' }}\\n {%- endif %}\\n{%- endif %}\\n{%- for message in messages %}\\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\\n {{- \\'<|im_start|>\\' + message.role + \\'\\\\n\\' + message.content + \\'<|im_end|>\\' + \\'\\\\n\\' }}\\n {%- elif message.role == \"assistant\" %}\\n {{- \\'<|im_start|>\\' + message.role }}\\n {%- if message.content %}\\n {{- \\'\\\\n\\' + message.content }}\\n {%- endif %}\\n {%- for tool_call in message.tool_calls %}\\n {%- if tool_call.function is defined %}\\n {%- set tool_call = tool_call.function %}\\n {%- endif %}\\n {{- \\'\\\\n<tool_call>\\\\n{\"name\": \"\\' }}\\n {{- tool_call.name }}\\n {{- \\'\", \"arguments\": \\' }}\\n {{- tool_call.arguments | tojson }}\\n {{- \\'}\\\\n</tool_call>\\' }}\\n {%- endfor %}\\n {{- \\'<|im_end|>\\\\n\\' }}\\n {%- elif message.role == \"tool\" %}\\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\\n {{- \\'<|im_start|>user\\' }}\\n {%- endif %}\\n {{- \\'\\\\n<tool_response>\\\\n\\' }}\\n {{- message.content }}\\n {{- \\'\\\\n</tool_response>\\' }}\\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\\n {{- \\'<|im_end|>\\\\n\\' }}\\n {%- endif %}\\n {%- endif %}\\n{%- endfor %}\\n{%- if add_generation_prompt %}\\n {{- \\'<|im_start|>assistant\\\\n\\' }}\\n{%- endif %}\\n', 'qwen2.attention.head_count_kv': '2', 'general.license.link': 'https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B/blob/main/LICENSE', 'general.base_model.count': '1', 'general.license': 'apache-2.0', 'general.base_model.0.repo_url': 'https://huggingface.co/Qwen/Qwen2.5-1.5B', 'qwen2.context_length': '32768'}\n",
|
||||||
|
"Available chat formats from metadata: chat_template.default\n",
|
||||||
|
"Using gguf chat template: {%- if tools %}\n",
|
||||||
|
" {{- '<|im_start|>system\\n' }}\n",
|
||||||
|
" {%- if messages[0]['role'] == 'system' %}\n",
|
||||||
|
" {{- messages[0]['content'] }}\n",
|
||||||
|
" {%- else %}\n",
|
||||||
|
" {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
" {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n",
|
||||||
|
" {%- for tool in tools %}\n",
|
||||||
|
" {{- \"\\n\" }}\n",
|
||||||
|
" {{- tool | tojson }}\n",
|
||||||
|
" {%- endfor %}\n",
|
||||||
|
" {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n",
|
||||||
|
"{%- else %}\n",
|
||||||
|
" {%- if messages[0]['role'] == 'system' %}\n",
|
||||||
|
" {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n",
|
||||||
|
" {%- else %}\n",
|
||||||
|
" {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
"{%- endif %}\n",
|
||||||
|
"{%- for message in messages %}\n",
|
||||||
|
" {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n",
|
||||||
|
" {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n",
|
||||||
|
" {%- elif message.role == \"assistant\" %}\n",
|
||||||
|
" {{- '<|im_start|>' + message.role }}\n",
|
||||||
|
" {%- if message.content %}\n",
|
||||||
|
" {{- '\\n' + message.content }}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
" {%- for tool_call in message.tool_calls %}\n",
|
||||||
|
" {%- if tool_call.function is defined %}\n",
|
||||||
|
" {%- set tool_call = tool_call.function %}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
" {{- '\\n<tool_call>\\n{\"name\": \"' }}\n",
|
||||||
|
" {{- tool_call.name }}\n",
|
||||||
|
" {{- '\", \"arguments\": ' }}\n",
|
||||||
|
" {{- tool_call.arguments | tojson }}\n",
|
||||||
|
" {{- '}\\n</tool_call>' }}\n",
|
||||||
|
" {%- endfor %}\n",
|
||||||
|
" {{- '<|im_end|>\\n' }}\n",
|
||||||
|
" {%- elif message.role == \"tool\" %}\n",
|
||||||
|
" {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n",
|
||||||
|
" {{- '<|im_start|>user' }}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
" {{- '\\n<tool_response>\\n' }}\n",
|
||||||
|
" {{- message.content }}\n",
|
||||||
|
" {{- '\\n</tool_response>' }}\n",
|
||||||
|
" {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n",
|
||||||
|
" {{- '<|im_end|>\\n' }}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
" {%- endif %}\n",
|
||||||
|
"{%- endfor %}\n",
|
||||||
|
"{%- if add_generation_prompt %}\n",
|
||||||
|
" {{- '<|im_start|>assistant\\n' }}\n",
|
||||||
|
"{%- endif %}\n",
|
||||||
|
"\n",
|
||||||
|
"Using chat eos_token: <|endoftext|>\n",
|
||||||
|
"Using chat bos_token: <|endoftext|>\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# grammar = LlamaGrammar.from_string(bnf)\n",
|
||||||
|
"llm_model = Llama(str(settings.models_path / \"qwen2.5-coder-1.5b-q8_0.gguf\"))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"id": "aa66f897",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Llama.generate: 9 prefix-match hit, remaining 1 prompt tokens to eval\n",
|
||||||
|
"llama_perf_context_print: load time = 1762.25 ms\n",
|
||||||
|
"llama_perf_context_print: prompt eval time = 0.00 ms / 1 tokens ( 0.00 ms per token, inf tokens per second)\n",
|
||||||
|
"llama_perf_context_print: eval time = 64123.14 ms / 502 runs ( 127.74 ms per token, 7.83 tokens per second)\n",
|
||||||
|
"llama_perf_context_print: total time = 133698.82 ms / 503 tokens\n",
|
||||||
|
"llama_perf_context_print: graphs reused = 486\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"response = llm_model(\n",
|
||||||
|
" \"Create a simple hello world function in AVAP language\",\n",
|
||||||
|
" grammar=grammar, max_tokens=-1, temperature=0\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"id": "317b96ae",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"{'choices': [{'finish_reason': 'length',\n",
|
||||||
|
" 'index': 0,\n",
|
||||||
|
" 'logprobs': None,\n",
|
||||||
|
" 'text': '\\n'\n",
|
||||||
|
" 'Sure.HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltintextoutputfunctionalityinAVAPlanguage(assumingyouhaveAVAPinstalledandconfiguredonyoursystem)\\n'\n",
|
||||||
|
" '\\n'\n",
|
||||||
|
" 'HereisaprogramthatprintsouthestringhelloworldinAVAPlanguageusingthebuiltint'}],\n",
|
||||||
|
" 'created': 1773656986,\n",
|
||||||
|
" 'id': 'cmpl-3c382cd0-7254-4bbc-8e71-84f97f06006a',\n",
|
||||||
|
" 'model': '/home/acano/PycharmProjects/assistance-engine/data/models/qwen2.5-coder-1.5b-q8_0.gguf',\n",
|
||||||
|
" 'object': 'text_completion',\n",
|
||||||
|
" 'usage': {'completion_tokens': 502, 'prompt_tokens': 10, 'total_tokens': 512}}\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"pprint.pprint(response)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "b6aa106d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "assistance-engine",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.11.13"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
|
|
@ -4,6 +4,7 @@ from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from lark import Lark
|
||||||
from chonkie import (
|
from chonkie import (
|
||||||
Chunk,
|
Chunk,
|
||||||
FileFetcher,
|
FileFetcher,
|
||||||
|
|
@ -126,29 +127,48 @@ def process_documents(docs_path: list[Path]) -> list[dict[str, Chunk | dict[str,
|
||||||
List of dicts with "chunk" (Chunk object) and "metadata" (dict with file info)
|
List of dicts with "chunk" (Chunk object) and "metadata" (dict with file info)
|
||||||
"""
|
"""
|
||||||
processed_docs = []
|
processed_docs = []
|
||||||
|
specific_metadata = {}
|
||||||
custom_tokenizer = AutoTokenizer.from_pretrained(settings.hf_emb_model_name)
|
custom_tokenizer = AutoTokenizer.from_pretrained(settings.hf_emb_model_name)
|
||||||
|
|
||||||
chef_md = MarkdownChef(tokenizer=custom_tokenizer)
|
chef_md = MarkdownChef(tokenizer=custom_tokenizer)
|
||||||
chef_txt = TextChef()
|
chef_txt = TextChef()
|
||||||
chunker = TokenChunker(tokenizer=custom_tokenizer)
|
chunker = TokenChunker(tokenizer=custom_tokenizer)
|
||||||
|
with open(settings.proj_root / "docs/BNF/avap.lark") as grammar:
|
||||||
|
lark_parser = Lark(grammar=grammar, parser="lalr", propagate_positions=True, start="program")
|
||||||
|
|
||||||
for doc_path in docs_path:
|
for doc_path in docs_path:
|
||||||
doc_extension = doc_path.suffix.lower()
|
doc_extension = doc_path.suffix.lower()
|
||||||
filename = doc_path.name
|
|
||||||
|
|
||||||
if doc_extension == ".md":
|
if doc_extension == ".md":
|
||||||
processed_doc = chef_md.process(doc_path)
|
processed_doc = chef_md.process(doc_path)
|
||||||
fused_doc = _merge_markdown_document(processed_doc)
|
fused_doc = _merge_markdown_document(processed_doc)
|
||||||
chunked_doc = fused_doc.chunks
|
chunked_doc = fused_doc.chunks
|
||||||
|
specific_metadata = {
|
||||||
|
"file_type": "avap_docs",
|
||||||
|
"filename": doc_path.name
|
||||||
|
}
|
||||||
|
|
||||||
elif doc_extension == ".avap":
|
elif doc_extension == ".avap":
|
||||||
processed_doc = chef_txt.process(doc_path)
|
processed_doc = chef_txt.process(doc_path)
|
||||||
chunked_doc = chunker.chunk(processed_doc.content)
|
chunked_doc = chunker.chunk(processed_doc.content)
|
||||||
|
try:
|
||||||
|
ast = lark_parser.parse(processed_doc.content)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error parsing AVAP code in {doc_path.name}: {e}")
|
||||||
|
ast = None
|
||||||
|
specific_metadata = {
|
||||||
|
"file_type": "avap_code",
|
||||||
|
"filename": doc_path.name,
|
||||||
|
"AST": str(ast)
|
||||||
|
}
|
||||||
|
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for chunk in chunked_doc:
|
for chunk in chunked_doc:
|
||||||
processed_docs.append({
|
processed_docs.append({
|
||||||
"chunk": chunk,
|
"chunk": chunk,
|
||||||
"extra_metadata": {"file": filename}
|
"extra_metadata": {**specific_metadata}
|
||||||
})
|
})
|
||||||
|
|
||||||
return processed_docs
|
return processed_docs
|
||||||
|
|
|
||||||
22
uv.lock
22
uv.lock
|
|
@ -259,12 +259,14 @@ dependencies = [
|
||||||
{ name = "langchain-elasticsearch" },
|
{ name = "langchain-elasticsearch" },
|
||||||
{ name = "langchain-huggingface" },
|
{ name = "langchain-huggingface" },
|
||||||
{ name = "langchain-ollama" },
|
{ name = "langchain-ollama" },
|
||||||
|
{ name = "llama-cpp-python" },
|
||||||
{ name = "loguru" },
|
{ name = "loguru" },
|
||||||
{ name = "nltk" },
|
{ name = "nltk" },
|
||||||
{ name = "numpy" },
|
{ name = "numpy" },
|
||||||
{ name = "pandas" },
|
{ name = "pandas" },
|
||||||
{ name = "python-dotenv" },
|
{ name = "python-dotenv" },
|
||||||
{ name = "rapidfuzz" },
|
{ name = "rapidfuzz" },
|
||||||
|
{ name = "tenacity" },
|
||||||
{ name = "tqdm" },
|
{ name = "tqdm" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -298,12 +300,14 @@ requires-dist = [
|
||||||
{ name = "langchain-elasticsearch", specifier = ">=1.0.0" },
|
{ name = "langchain-elasticsearch", specifier = ">=1.0.0" },
|
||||||
{ name = "langchain-huggingface", specifier = ">=1.2.0" },
|
{ name = "langchain-huggingface", specifier = ">=1.2.0" },
|
||||||
{ name = "langchain-ollama", specifier = ">=1.0.1" },
|
{ name = "langchain-ollama", specifier = ">=1.0.1" },
|
||||||
|
{ name = "llama-cpp-python", specifier = ">=0.3.16" },
|
||||||
{ name = "loguru", specifier = ">=0.7.3" },
|
{ name = "loguru", specifier = ">=0.7.3" },
|
||||||
{ name = "nltk", specifier = ">=3.9.2" },
|
{ name = "nltk", specifier = ">=3.9.2" },
|
||||||
{ name = "numpy", specifier = ">=2.4.2" },
|
{ name = "numpy", specifier = ">=2.4.2" },
|
||||||
{ name = "pandas", specifier = ">=3.0.0" },
|
{ name = "pandas", specifier = ">=3.0.0" },
|
||||||
{ name = "python-dotenv", specifier = ">=1.2.1" },
|
{ name = "python-dotenv", specifier = ">=1.2.1" },
|
||||||
{ name = "rapidfuzz", specifier = ">=3.14.3" },
|
{ name = "rapidfuzz", specifier = ">=3.14.3" },
|
||||||
|
{ name = "tenacity", specifier = ">=9.1.4" },
|
||||||
{ name = "tqdm", specifier = ">=4.67.3" },
|
{ name = "tqdm", specifier = ">=4.67.3" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -2357,6 +2361,18 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/f2/24/8d99982f0aa9c1cd82073c6232b54a0dbe6797c7d63c0583a6c68ee3ddf2/litestar_htmx-0.5.0-py3-none-any.whl", hash = "sha256:92833aa47e0d0e868d2a7dbfab75261f124f4b83d4f9ad12b57b9a68f86c50e6", size = 9970, upload-time = "2025-06-11T21:19:44.465Z" },
|
{ url = "https://files.pythonhosted.org/packages/f2/24/8d99982f0aa9c1cd82073c6232b54a0dbe6797c7d63c0583a6c68ee3ddf2/litestar_htmx-0.5.0-py3-none-any.whl", hash = "sha256:92833aa47e0d0e868d2a7dbfab75261f124f4b83d4f9ad12b57b9a68f86c50e6", size = 9970, upload-time = "2025-06-11T21:19:44.465Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "llama-cpp-python"
|
||||||
|
version = "0.3.16"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "diskcache" },
|
||||||
|
{ name = "jinja2" },
|
||||||
|
{ name = "numpy" },
|
||||||
|
{ name = "typing-extensions" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/e4/b4/c8cd17629ced0b9644a71d399a91145aedef109c0333443bef015e45b704/llama_cpp_python-0.3.16.tar.gz", hash = "sha256:34ed0f9bd9431af045bb63d9324ae620ad0536653740e9bb163a2e1fcb973be6", size = 50688636, upload-time = "2025-08-15T04:58:29.212Z" }
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "loguru"
|
name = "loguru"
|
||||||
version = "0.7.3"
|
version = "0.7.3"
|
||||||
|
|
@ -5207,6 +5223,12 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/0f/8b/4b61d6e13f7108f36910df9ab4b58fd389cc2520d54d81b88660804aad99/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:418997cb02d0a0f1497cf6a09f63166f9f5df9f3e16c8a716ab76a72127c714f", size = 79423467, upload-time = "2026-02-10T21:44:48.711Z" },
|
{ url = "https://files.pythonhosted.org/packages/0f/8b/4b61d6e13f7108f36910df9ab4b58fd389cc2520d54d81b88660804aad99/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:418997cb02d0a0f1497cf6a09f63166f9f5df9f3e16c8a716ab76a72127c714f", size = 79423467, upload-time = "2026-02-10T21:44:48.711Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" },
|
{ url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" },
|
{ url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/36/ab/7b562f1808d3f65414cd80a4f7d4bb00979d9355616c034c171249e1a303/torch-2.10.0-3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ac5bdcbb074384c66fa160c15b1ead77839e3fe7ed117d667249afce0acabfac", size = 915518691, upload-time = "2026-03-11T14:15:43.147Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b3/7a/abada41517ce0011775f0f4eacc79659bc9bc6c361e6bfe6f7052a6b9363/torch-2.10.0-3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98c01b8bb5e3240426dcde1446eed6f40c778091c8544767ef1168fc663a05a6", size = 915622781, upload-time = "2026-03-11T14:17:11.354Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/ab/c6/4dfe238342ffdcec5aef1c96c457548762d33c40b45a1ab7033bb26d2ff2/torch-2.10.0-3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:80b1b5bfe38eb0e9f5ff09f206dcac0a87aadd084230d4a36eea5ec5232c115b", size = 915627275, upload-time = "2026-03-11T14:16:11.325Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/d8/f0/72bf18847f58f877a6a8acf60614b14935e2f156d942483af1ffc081aea0/torch-2.10.0-3-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:46b3574d93a2a8134b3f5475cfb98e2eb46771794c57015f6ad1fb795ec25e49", size = 915523474, upload-time = "2026-03-11T14:17:44.422Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f4/39/590742415c3030551944edc2ddc273ea1fdfe8ffb2780992e824f1ebee98/torch-2.10.0-3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b1d5e2aba4eb7f8e87fbe04f86442887f9167a35f092afe4c237dfcaaef6e328", size = 915632474, upload-time = "2026-03-11T14:15:13.666Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b6/8e/34949484f764dde5b222b7fe3fede43e4a6f0da9d7f8c370bb617d629ee2/torch-2.10.0-3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0228d20b06701c05a8f978357f657817a4a63984b0c90745def81c18aedfa591", size = 915523882, upload-time = "2026-03-11T14:14:46.311Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/78/89/f5554b13ebd71e05c0b002f95148033e730d3f7067f67423026cc9c69410/torch-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3282d9febd1e4e476630a099692b44fdc214ee9bf8ee5377732d9d9dfe5712e4", size = 145992610, upload-time = "2026-01-21T16:25:26.327Z" },
|
{ url = "https://files.pythonhosted.org/packages/78/89/f5554b13ebd71e05c0b002f95148033e730d3f7067f67423026cc9c69410/torch-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3282d9febd1e4e476630a099692b44fdc214ee9bf8ee5377732d9d9dfe5712e4", size = 145992610, upload-time = "2026-01-21T16:25:26.327Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/ae/30/a3a2120621bf9c17779b169fc17e3dc29b230c29d0f8222f499f5e159aa8/torch-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2f9edd8dbc99f62bc4dfb78af7bf89499bca3d753423ac1b4e06592e467b763", size = 915607863, upload-time = "2026-01-21T16:25:06.696Z" },
|
{ url = "https://files.pythonhosted.org/packages/ae/30/a3a2120621bf9c17779b169fc17e3dc29b230c29d0f8222f499f5e159aa8/torch-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2f9edd8dbc99f62bc4dfb78af7bf89499bca3d753423ac1b4e06592e467b763", size = 915607863, upload-time = "2026-01-21T16:25:06.696Z" },
|
||||||
{ url = "https://files.pythonhosted.org/packages/6f/3d/c87b33c5f260a2a8ad68da7147e105f05868c281c63d65ed85aa4da98c66/torch-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:29b7009dba4b7a1c960260fc8ac85022c784250af43af9fb0ebafc9883782ebd", size = 113723116, upload-time = "2026-01-21T16:25:21.916Z" },
|
{ url = "https://files.pythonhosted.org/packages/6f/3d/c87b33c5f260a2a8ad68da7147e105f05868c281c63d65ed85aa4da98c66/torch-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:29b7009dba4b7a1c960260fc8ac85022c784250af43af9fb0ebafc9883782ebd", size = 113723116, upload-time = "2026-01-21T16:25:21.916Z" },
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue