36 lines
862 B
YAML
36 lines
862 B
YAML
id: qwen3-4b-1c-lora-v1
|
|
base_model_path: /models/base/qwen3-4b-instruct-2507
|
|
dataset_path: /workspace/plugins/1c/training/prepared/train.chat.jsonl
|
|
output_dir: /models/adapters/1c/qwen3-4b-1c-lora-v1
|
|
max_seq_length: 2048
|
|
train:
|
|
num_train_epochs: 3
|
|
per_device_train_batch_size: 1
|
|
gradient_accumulation_steps: 8
|
|
learning_rate: 0.0002
|
|
warmup_ratio: 0.03
|
|
logging_steps: 1
|
|
save_strategy: "no"
|
|
save_total_limit: 1
|
|
bf16: true
|
|
fp16: false
|
|
lora:
|
|
r: 16
|
|
lora_alpha: 32
|
|
lora_dropout: 0.05
|
|
bias: none
|
|
target_modules:
|
|
- q_proj
|
|
- k_proj
|
|
- v_proj
|
|
- o_proj
|
|
- gate_proj
|
|
- up_proj
|
|
- down_proj
|
|
quantization:
|
|
load_in_4bit: true
|
|
bnb_4bit_quant_type: nf4
|
|
bnb_4bit_use_double_quant: true
|
|
bnb_4bit_compute_dtype: bfloat16
|
|
notes: "Training config for the first 1C LoRA adapter. Requires complete base model and GPU host."
|