最优模型训练参数

[jiechuangu]

我用如下参数进行训练，lora相关参数与3.5、模型权重中huggingface提供的相同，在Spider official size 95M database上evaluate，execution accuracy为0.766，使用huggingface提供的权重execution accuracy为0.787，请问可以提供huggingface上公开模型的训练参数吗？
model_args***************************
ModelArguments(model_name_or_path='codellama/CodeLlama-13b-Instruct-hf', cache_dir=None, use_fast_tokenizer=False, use_auth_token=False, model_revision='main', padding_side='left', quantization_bit=None, quantization_type='nf4', double_quantization=True, rope_scaling=None, checkpoint_dir=None, plot_loss=True, hf_auth_token=None, compute_dtype=torch.bfloat16, model_max_length=2560, hf_hub_token=None, split_special_tokens=False)

data_args***************************
DataArguments(template='llama2', dataset='example_text2sql_train', dataset_dir='dbgpt_hub/data/', cutoff_len=1024, reserved_label_len=1, split='train', streaming=False, buffer_size=16384, mix_strategy='concat', interleave_probs=None, overwrite_cache=True, preprocessing_num_workers=None, max_source_length=2048, max_target_length=512, max_samples=None, eval_num_beams=None, ignore_pad_token_for_loss=True, system_prompt=None, val_size=0, predicted_input_filename='dbgpt_hub/data/example_text2sql_dev.json', predicted_out_filename='pred_sql.sql')

training_args***************************
Seq2SeqTrainingArguments(
_n_gpu=1,
adafactor=False,
adam_beta1=0.9,
adam_beta2=0.999,
adam_epsilon=1e-08,
auto_find_batch_size=False,
bf16=True,
bf16_full_eval=False,
data_seed=None,
dataloader_drop_last=False,
dataloader_num_workers=0,
dataloader_pin_memory=True,
ddp_backend=None,
ddp_broadcast_buffers=None,
ddp_bucket_cap_mb=None,
ddp_find_unused_parameters=False,
ddp_timeout=1800,
debug=[],
deepspeed=None,
disable_tqdm=False,
dispatch_batches=None,
do_eval=False,
do_predict=False,
do_train=True,
eval_accumulation_steps=None,
eval_delay=0,
eval_steps=None,
evaluation_strategy=no,
fp16=False,
fp16_backend=auto,
fp16_full_eval=False,
fp16_opt_level=O1,
fsdp=[],
fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},
fsdp_min_num_params=0,
fsdp_transformer_layer_cls_to_wrap=None,
full_determinism=False,
generation_config=None,
generation_max_length=None,
generation_num_beams=None,
gradient_accumulation_steps=16,
gradient_checkpointing=False,
greater_is_better=None,
group_by_length=False,
half_precision_backend=auto,
hub_always_push=False,
hub_model_id=None,
hub_private_repo=False,
hub_strategy=every_save,
hub_token=<HUB_TOKEN>,
ignore_data_skip=False,
include_inputs_for_metrics=False,
include_tokens_per_second=False,
jit_mode_eval=False,
label_names=None,
label_smoothing_factor=0.0,
learning_rate=0.0002,
length_column_name=length,
load_best_model_at_end=False,
local_rank=0,
log_level=passive,
log_level_replica=warning,
log_on_each_node=True,
logging_dir=dbgpt_hub/output/adapter/CodeLlama-13b-sql-lora-test/runs/Apr15_08-36-28_cni-2288H-V5,
logging_first_step=False,
logging_nan_inf_filter=True,
logging_steps=50,
logging_strategy=steps,
lr_scheduler_type=cosine_with_restarts,
max_grad_norm=1.0,
max_steps=-1,
metric_for_best_model=None,
mp_parameters=,
no_cuda=False,
num_train_epochs=8.0,
optim=adamw_torch,
optim_args=None,
output_dir=dbgpt_hub/output/adapter/CodeLlama-13b-sql-lora-test,
overwrite_output_dir=True,
past_index=-1,
per_device_eval_batch_size=8,
per_device_train_batch_size=1,
predict_with_generate=False,
prediction_loss_only=False,
push_to_hub=False,
push_to_hub_model_id=None,
push_to_hub_organization=None,
push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
ray_scope=last,
remove_unused_columns=True,
report_to=['wandb'],
resume_from_checkpoint=None,
run_name=dbgpt_hub/output/adapter/CodeLlama-13b-sql-lora-test,
save_on_each_node=False,
save_safetensors=False,
save_steps=2000,
save_strategy=steps,
save_total_limit=None,
seed=42,
sharded_ddp=[],
skip_memory_metrics=True,
sortish_sampler=False,
tf32=None,
torch_compile=False,
torch_compile_backend=None,
torch_compile_mode=None,
torchdynamo=None,
tpu_metrics_debug=False,
tpu_num_cores=None,
use_cpu=False,
use_ipex=False,
use_legacy_prediction_loop=False,
use_mps_device=False,
warmup_ratio=0.0,
warmup_steps=0,
weight_decay=0.0,
)

finetuning_args***************************
FinetuningArguments(stage='sft', finetuning_type='lora', num_hidden_layers=32, num_layer_trainable=3, name_module_trainable='mlp', lora_rank=64, lora_alpha=32.0, lora_dropout=0.1, lora_target=['q_proj', 'v_proj'], resume_lora_training=True, ppo_score_norm=False, dpo_beta=0.1)

generating_args***************************
GeneratingArguments(do_sample=True, temperature=0.95, top_p=0.7, top_k=50, num_beams=1, max_length=None, max_new_tokens=512, repetition_penalty=1.0, length_penalty=1.0)