115 lines
3.9 KiB
Python
115 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_METADATA = ROOT / "plugins" / "1c" / "metadata" / "examples" / "metadata.example.json"
|
|
DEFAULT_BSL_MODULES = ROOT / "plugins" / "1c" / "metadata" / "examples" / "bsl-modules.example.json"
|
|
DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
|
|
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
|
|
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
|
|
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
|
|
DEFAULT_VECTOR_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_vector_index.sqlite"
|
|
|
|
|
|
def run(command: list[str]) -> None:
|
|
print(" ".join(command))
|
|
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
|
|
if result.returncode != 0:
|
|
raise SystemExit(result.returncode)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Build the 1C RAG knowledge base.")
|
|
parser.add_argument("--metadata", type=Path, default=DEFAULT_METADATA)
|
|
parser.add_argument("--bsl-modules", type=Path, help="Optional BSL module snapshot JSON.")
|
|
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
|
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
|
parser.add_argument("--vector-index", type=Path, default=DEFAULT_VECTOR_INDEX)
|
|
parser.add_argument("--skip-vector-index", action="store_true")
|
|
parser.add_argument(
|
|
"--include-example-bsl",
|
|
action="store_true",
|
|
help="Use the bundled BSL example when --bsl-modules is not provided.",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
metadata_output = args.source_dir / f"{args.metadata.stem}.metadata.md"
|
|
run([sys.executable, "scripts/validate_1c_metadata_snapshot.py", str(args.metadata)])
|
|
run(
|
|
[
|
|
sys.executable,
|
|
"scripts/convert_1c_metadata_to_rag.py",
|
|
"--input",
|
|
str(args.metadata),
|
|
"--output",
|
|
str(metadata_output),
|
|
]
|
|
)
|
|
|
|
bsl_modules = args.bsl_modules
|
|
if bsl_modules is None and args.include_example_bsl:
|
|
bsl_modules = DEFAULT_BSL_MODULES
|
|
if bsl_modules:
|
|
bsl_output = args.source_dir / f"{bsl_modules.stem}.bsl.md"
|
|
run([sys.executable, "scripts/validate_1c_bsl_modules.py", str(bsl_modules)])
|
|
run(
|
|
[
|
|
sys.executable,
|
|
"scripts/convert_1c_bsl_modules_to_rag.py",
|
|
"--input",
|
|
str(bsl_modules),
|
|
"--output",
|
|
str(bsl_output),
|
|
]
|
|
)
|
|
|
|
run([sys.executable, "scripts/validate_1c_rag_sources.py", "--source-dir", str(args.source_dir)])
|
|
run(
|
|
[
|
|
sys.executable,
|
|
"scripts/prepare_1c_rag_corpus.py",
|
|
"--source-dir",
|
|
str(args.source_dir),
|
|
"--output",
|
|
str(args.corpus),
|
|
"--manifest",
|
|
str(args.manifest),
|
|
]
|
|
)
|
|
run(
|
|
[
|
|
sys.executable,
|
|
"scripts/build_1c_rag_index.py",
|
|
"--corpus",
|
|
str(args.corpus),
|
|
"--output",
|
|
str(args.index),
|
|
]
|
|
)
|
|
if not args.skip_vector_index:
|
|
run(
|
|
[
|
|
sys.executable,
|
|
"scripts/build_1c_rag_vector_index.py",
|
|
"--corpus",
|
|
str(args.corpus),
|
|
"--output",
|
|
str(args.vector_index),
|
|
]
|
|
)
|
|
print(f"Built 1C knowledge base: {args.index}")
|
|
if not args.skip_vector_index:
|
|
print(f"Built 1C vector index: {args.vector_index}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|