73 lines
3.2 KiB
JSON
73 lines
3.2 KiB
JSON
{
|
|
"trainer": "sft",
|
|
"experiment_name": "test-2",
|
|
"fine_tuning_image": "registry.aifrica.co.kr/cheetah/cheetahai-finetuning-autotrain-transformer:0.0.10-b5c98fb-4.57.6",
|
|
"model": {
|
|
"model_path": "/cheetah/input/model/groupuser/Llama-3.2-1B-Instruct",
|
|
"model_branch": "main",
|
|
"model_repository": "git@git.dev2.aifrica.co.kr:groupuser/Llama-3.2-1B-Instruct.git",
|
|
"type": "git"
|
|
},
|
|
"save_model": {
|
|
"model_repository": "git@git.dev2.aifrica.co.kr:groupuser/dbinc-Llama-3.2-1B-Instruct-finetuning.git",
|
|
"type": "git"
|
|
},
|
|
"dataset": {
|
|
"dataset_files": "*.jsonl",
|
|
"pre_processing": "ZnJvbSBkYXRhc2V0cyBpbXBvcnQgRGF0YXNldERpY3QKaW1wb3J0IGpzb24KCmRlZiBwcmVwcm9jZXNzKGV4YW1wbGUpOgogICAgdGFza2luZm8gPSBleGFtcGxlWyJ0YXNraW5mbyJdCiAgICBpZiBpc2luc3RhbmNlKHRhc2tpbmZvLCBzdHIpOgogICAgICAgIHRhc2tpbmZvID0ganNvbi5sb2Fkcyh0YXNraW5mbykKICAgIAogICAgc2VudGVuY2VzID0gdGFza2luZm8uZ2V0KCJzZW50ZW5jZXMiKSBvciBbXQogICAgY29udGV4dCA9ICIiLmpvaW4oc2VudGVuY2VzKS5zdHJpcCgpCiAgICBpbnB1dF90ZXh0ID0gKHRhc2tpbmZvLmdldCgiaW5wdXQiKSBvciAiIikuc3RyaXAoKQogICAgb3V0cHV0X3RleHQgPSAodGFza2luZm8uZ2V0KCJvdXRwdXQiKSBvciAiIikuc3RyaXAoKQogICAgCiAgICB1c2VyX2NvbnRlbnQgPSBmIuuLpOydjCDtjJDqsrDrrLjsnYQg7LC46rOg7ZWY7JesIOyniOusuOyXkCDri7XtlZjshLjsmpQuXG5cbntjb250ZXh0fVxuXG7sp4jrrLg6IHtpbnB1dF90ZXh0fSIKICAgIAogICAgbWVzc2FnZXMgPSBbCiAgICAgICAgeyJyb2xlIjogInVzZXIiLCAiY29udGVudCI6IHVzZXJfY29udGVudH0sCiAgICAgICAgeyJyb2xlIjogImFzc2lzdGFudCIsICJjb250ZW50Ijogb3V0cHV0X3RleHR9LAogICAgXQogICAgdGV4dCA9IHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKG1lc3NhZ2VzLCB0b2tlbml6ZT1GYWxzZSkKICAgIHJldHVybiB7InRleHQiOiB0ZXh0fQoKZGF0YXNldF9kaWN0ID0gRGF0YXNldERpY3QoewogICAgc3BsaXQ6IGRhdGFzZXRfZGljdFtzcGxpdF0ubWFwKAogICAgICAgIHByZXByb2Nlc3MsCiAgICAgICAgcmVtb3ZlX2NvbHVtbnM9ZGF0YXNldF9kaWN0W3NwbGl0XS5jb2x1bW5fbmFtZXMsCiAgICApCiAgICBmb3Igc3BsaXQgaW4gZGF0YXNldF9kaWN0Cn0p",
|
|
"dataset_repository": null,
|
|
"dataset_path": "/cheetah/input/dataset/dataset-dvjvydwm-zokjnzjp",
|
|
"dataset_branch": null,
|
|
"type": "volume"
|
|
},
|
|
"peft": {
|
|
"r": 16,
|
|
"lora_alpha": 32,
|
|
"bias": "none",
|
|
"merge_adapter": false,
|
|
"lora_dropout": 0.05,
|
|
"target_modules": "all-linear",
|
|
"tuner": "lora",
|
|
"task_type": "CAUSAL_LM",
|
|
"peft": true
|
|
},
|
|
"tokenizer_parameters": {
|
|
"padding_side": "right",
|
|
"add_eos_token": true,
|
|
"max_length": 2048
|
|
},
|
|
"train_model_parameters": {
|
|
"fp16": false,
|
|
"bf16": true,
|
|
"packing": false,
|
|
"optim": "adamw_torch",
|
|
"eval_steps": 500,
|
|
"save_steps": 500,
|
|
"report_to": "tensorboard",
|
|
"do_train": true,
|
|
"do_eval": true,
|
|
"per_device_train_batch_size": 2,
|
|
"per_device_eval_batch_size": 2,
|
|
"gradient_accumulation_steps": 4,
|
|
"gradient_checkpointing": true,
|
|
"torch_dtype": "bfloat16",
|
|
"use_flash_attention": false,
|
|
"dataset_text_field": "text",
|
|
"max_seq_length": 2048,
|
|
"num_train_epochs": 1,
|
|
"auto_find_batch_size": false,
|
|
"learning_rate": 0.0002,
|
|
"weight_decay": 0,
|
|
"max_grad_norm": 1,
|
|
"lr_scheduler_type": "linear",
|
|
"warmup_ratio": 0.1,
|
|
"eval_strategy": "epoch",
|
|
"save_strategy": "epoch",
|
|
"save_total_limit": 1,
|
|
"logging_strategy": "epoch",
|
|
"logging_steps": 10,
|
|
"output_dir": "/cheetah/fine-tuning/output",
|
|
"seed": 42
|
|
}
|
|
}
|