deepspeed llava/train/train_mem.py \

--deepspeed ./scripts/zero2.json \

--model_name_or_path ./checkpoints/$MODEL_VERSION \

--version $PROMPT_VERSION \

--vision_tower openai/clip-vit-large-patch14 \

--pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \

--mm_vision_select_layer -2 \

--mm_use_im_start_end False \

--mm_use_im_patch_token False \

--bf16 True \

--per_device_train_batch_size 16 \

--per_device_eval_batch_size 4 \

--gradient_accumulation_steps 1 \

--evaluation_strategy "no" \

--save_strategy "steps" \

--save_total_limit 1 \

--learning_rate 2e-5 \

--warmup_ratio 0.03 \

--lr_scheduler_type "cosine" \

--logging_steps 1 \

--tf32 True \

--gradient_checkpointing True \

--lazy_preprocess True \

--report_to wandb