rjob submit --delete -e DISTRIBUTED_JOB=true --name="accel-hydra-f5tts-text-token-mask" \
    --gpu=8 --memory=1600000 --cpu=144 \
    --charged-group=speechllm_gpu --namespace=ailab-speechllm \
    --private-machine=group \
    --mount=gpfs://gpfs1/xuxuenan:/mnt/shared-storage-user/xuxuenan \
    --mount=gpfs://gpfs1/brainllm-share:/mnt/shared-storage-user/brainllm-share \
    --image=registry.h.pjlab.org.cn/ailab-puyu-puyu_gpu/lmdeploy:v0.10.1-cu12.8 \
    --host-network=true \
    -- bash -c '\
cd /mnt/shared-storage-user/xuxuenan/workspace/accel_hydra/examples/f5tts
export WANDB_MODE=offline
accelerate_bin=/mnt/shared-storage-user/xuxuenan/miniconda3/envs/uniflow-audio/bin/accelerate
${accelerate_bin} launch \
    --num_processes $((PROC_PER_NODE * NODE_COUNT)) \
    --num_machines ${NODE_COUNT} \
    --machine_rank ${NODE_RANK} \
    --main_process_ip ${MASTER_ADDR} \
    --mixed_precision bf16 \
    train.py \
    model=f5tts_accel_hydra_drop_text_token \
    exp_name=f5tts_base_libritts_text_token_mask
'

    # --charged-group=brainllm_gpu \