assistance-engine/scripts/pipelines/tasks/chunk.py

61 lines
2.2 KiB
Python

import os
import re
def replace_javascript_with_avap(text: str) -> str:
"""
Replace mentions of javascript language with avap in the text.
Handles code blocks, language identifiers, and references.
Args:
text: The text to process.
Returns:
The text with javascript references replaced with avap.
"""
# Replace ```javascript with ```avap
text = text.replace("```javascript", "```avap")
# Replace ```js with ```avap
text = text.replace("```js", "```avap")
# Replace common phrases (case-insensitive)
text = re.sub(r"\bjavascript\s+code\b", "avap code", text, flags=re.IGNORECASE)
text = re.sub(
r"\bjavascript\s+example\b", "avap example", text, flags=re.IGNORECASE
)
text = re.sub(r"\bjavascript\b(?!\s+file)", "avap", text, flags=re.IGNORECASE)
return text
def read_concat_files(folder_path: str, file_prefix: str, concatenate: bool = True) -> str | list[str]:
"""
Read and concatenate all files in a folder whose names start with a given prefix.
Replaces javascript language markers with avap.
Args:
folder_path: Path to the folder to search in.
file_prefix: The prefix that file names must start with.
concatenate: Whether to concatenate the contents of the files.
Returns:
A single string with the concatenated contents of all matching files,
with javascript markers replaced with avap, or a list of strings if concatenate is False.
"""
contents = []
for filename in sorted(os.listdir(folder_path)):
if filename.startswith(file_prefix):
file_path = os.path.join(folder_path, filename)
if os.path.isfile(file_path):
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
if content.strip():
print(f"Reading file: {filename}") # Skip empty files
contents.append(content)
if concatenate:
concatenated = "\n".join(contents)
return replace_javascript_with_avap(concatenated)
else:
return [replace_javascript_with_avap(content) for content in contents]