#!/bin/bash

vllm \
  serve \
  meta-llama/Llama-2-7b-hf \
  --tensor-parallel-size 1 \
  --dtype auto \
  --gpu-memory-utilization 0.88 \
  --max-model-len 8192 \
  --max-num-seqs 3 \
  --max-num-batched-tokens 8192
