usage: /workspace/rust-lab/pxa4/pxa-v4.0.1/bin/llama-server [options]

general:

  -h,    --help, --usage          print usage and exit
         --version                show version and build info
  -v,    --verbose                print verbose information
         --minilog                print important information
         --verbosity N            set specific verbosity level (default: 0)
         --verbose-prompt         print a verbose prompt before generation (default: false)
  -dr,   --dry-run                skip loading tensors in the files
         --no-display-prompt      don't print prompt at generation (default: false)
  -co,   --color                  colorise output to distinguish prompt and user input from generations (default: false)
  -s,    --seed SEED              RNG seed (default: -1, use random seed for < 0)
  -t,    --threads N              number of threads to use during generation (default: 12)
  -tb,   --threads-batch N        number of threads to use during batch and prompt processing (default: same as --threads)
  -tm,   --threads-mtmd N         number of threads to use during multimodal image processing (default: same as --threads-batch)
  -td,   --threads-draft N        number of threads to use during generation (default: same as --threads)
  -tbd,  --threads-batch-draft N  number of threads to use during batch and prompt processing (default: same as --threads-draft)
  -ps,   --p-split N              speculative decoding split probability (default: 0.1)
  -lcs,  --lookup-cache-static FNAME
                                  path to static lookup cache to use for lookup decoding (not updated by generation)
  -lcd,  --lookup-cache-dynamic FNAME
                                  path to dynamic lookup cache to use for lookup decoding (updated by generation)
  -c,    --ctx-size N             size of the prompt context (default: 0, 0 = loaded from model)
  -cd,   --ctx-size-draft N       size of the prompt context for the draft model (default: 0, 0 = loaded from model)
  --ctx-checkpoints N             max number of context checkpoints to create per slot (default: up to 32, sized to the free RAM at load: 15% of it across all slots; a flag given here always wins)
  --ctx-checkpoints-interval N    minimum number of tokens between each context checkpoint.  (default: 512, <=0 disable)
  --ctx-checkpoints-tolerance N   the number of tokens before the full prompt to create the checkpoint.  (default: 5, <=0 disable)
  --ctx-checkpoints-eviction NAME 
                                  checkpoint eviction strategy: fifo, variance, or auto (auto=variance). Variance preserves coverage with uniform spacing. (default: variance)
  -cram, --cache-ram N            set the maximum cache size in MiB (default: 8192, capped to 15% of the free RAM at load; -1 - no limit, 0 - disable; a value given here always wins)
  -crs,  --cache-ram-similarity N 
                                  max of similarity of prompt tokens to cache tokens that triggers prompt cache (default: 0.50).
  -cram-n-min --cache-ram-n-min N 
                                  minimum number of the cached tokens that triggers prompt cache (default: 0).
  -n,    --predict N              number of tokens to predict (default: -1, -1 = infinity, -2 = until context filled)
  -b,    --batch-size N           logical maximum batch size (default: 2048)
  -ub,   --ubatch-size N          physical maximum batch size (default: 512)
         --keep N                 number of tokens to keep from the initial prompt (default: 0, -1 = all)
         --chunks N               max number of chunks to process (default: -1, -1 = all)
  -no-fa, --no-flash-attn         disable Flash Attention (default: enabled)
  -fa, --flash-attn [auto|on|off|0|1]
                                  set Flash Attention; bare -fa means on (default: on)
  -mla,  --mla-use                enable MLA (default: 3)
  -amb,  --attention-max-batch    max batch size for attention computations (default: 0)
  -no-fmoe, --no-fused-moe        disable fused MoE (default: enabled)
  -ger,  --grouped-expert-routing 
                                  enable grouped expert routing (default: disabled)
  -no-fug, --no-fused-up-gate     disable fused up-gate (default: enabled)
  -no-mmad, --no-fused-mul-multiadd
                                  disable fused mul-multi_add (default: enabled)
  -gr, --graph-reuse              enable graph reuse (default: enabled)
  -no-gr, --no-graph-reuse        disable graph reuse (default: disabled)
  -ser,  --smart-expert-reduction 
                                  experts reduction (default: -1,0)
  -mqkv,  --merge-qkv,            merge Q,K,V (default: 0)
  -muge,  --merge-up-gate-experts,
                                  merge ffn_up/gate_exps (default: 0)
  -khad,  --k-cache-hadamard,     Use Hadamard transform for K-cache (default: 0)
  -vhad,  --v-cache-hadamard,     Use Hadamard transform for V-cache (default: 0)
  -smf16, --split-mode-f16,       Use f16 for data exchange between GPUs (default: 1)
  -smf32, --split-mode-f32,       Use f32 for data exchange between GPUs (default: 0)
  -grt, --graph-reduce-type       Type for data exchange between GPUs (default: f16)
  -gap, --graph-attn-precision    Flash-attn precision under -sm graph (default: f16)
  -smgs, --split-mode-graph-scheduling,
                                  Force Split Mode Graph Scheduling (default: 0)
  -sas,  --scheduler_async,       Async evaluation of compute graphs: 0)
  -vq, --validate-quants          validate quantized data while loading the model (default: 0)
  -p,    --prompt PROMPT          prompt to start generation with
                                  in conversation mode, this will be used as system prompt
                                  (default: '')
  -f,    --file FNAME             a file containing the prompt (default: none)
         --in-file FNAME          an input file (repeat to specify multiple files)
  -bf,   --binary-file FNAME      binary file containing the prompt (default: none)
  -e,    --escape                 process escapes sequences (\n, \r, \t, \', \", \\) (default: true)
         --no-escape              do not process escape sequences
  -ptc,  --print-token-count N    print token count every N tokens (default: -1)
         --prompt-cache FNAME     file to cache prompt state for faster startup (default: none)
         --prompt-cache-all       if specified, saves user input and generations to cache as well
                                  not supported with --interactive or other interactive options
         --prompt-cache-ro        if specified, uses the prompt cache but does not update it
  -r,    --reverse-prompt PROMPT  halt generation at PROMPT, return control in interactive mode
                                  can be specified more than once for multiple prompts
  -sp,   --special                special tokens output enabled (default: false)
  -cnv,  --conversation           run in conversation mode, does not print special tokens and suffix/prefix
                                  if suffix/prefix are not specified, default chat template will be used
                                  (default: false)
  -i,    --interactive            run in interactive mode (default: false)
  -if,   --interactive-first      run in interactive mode and wait for input right away (default: false)
  -mli,  --multiline-input        allows you to write or paste multiple lines without ending each in '\'
         --in-prefix-bos          prefix BOS to user inputs, preceding the `--in-prefix` string
         --in-prefix STRING       string to prefix user inputs with (default: empty)
         --in-suffix STRING       string to suffix after user inputs with (default: empty)
         --no-warmup              skip warming up the model with an empty run
         --spm-infill             use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. (default: disabled)

sampling:

         --samplers SAMPLERS      samplers that will be used for generation in the order, separated by ';'
                                  (default: dry;top_k;tfs_z;typical_p;top_p;min_p;xtc;top_n_sigma;temperature;adaptive_p;)
         --sampling-seq SEQUENCE  simplified sequence for samplers that will be used (default: dkfypmxntws)
         --ignore-eos             ignore end of stream token and continue generating (implies --logit-bias EOS-inf)
         --penalize-nl            penalize newline tokens (default: false)
         --temp N                 temperature (default: 0.8)
         --top-k N                top-k sampling (default: 40, 0 = disabled)
         --top-p N                top-p sampling (default: 0.9, 1.0 = disabled)
         --min-p N                min-p sampling (default: 0.1, 0.0 = disabled)
         --tfs N                  tail free sampling, parameter z (default: 1.0, 1.0 = disabled)
         --typical N              locally typical sampling, parameter p (default: 1.0, 1.0 = disabled)
         --repeat-last-n N        last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size)
         --repeat-penalty N       penalize repeat sequence of tokens (default: 1.0, 1.0 = disabled)
         --presence-penalty N     repeat alpha presence penalty (default: 0.0, 0.0 = disabled)
         --frequency-penalty N    repeat alpha frequency penalty (default: 0.0, 0.0 = disabled)
         --dynatemp-range N       dynamic temperature range (default: 0.0, 0.0 = disabled)
         --dynatemp-exp N         dynamic temperature exponent (default: 1.0)
         --mirostat N             use Mirostat sampling.
                                  Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.
                                  (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)
         --mirostat-lr N          Mirostat learning rate, parameter eta (default: 0.1)
         --mirostat-ent N         Mirostat target entropy, parameter tau (default: 5.0)
         --xtc-probability p      xtc probability (default: 0.0, 0.0 = disabled)
         --xtc-threshold t        xtc threshold (default: 1.0, >0.5 = disabled)
         --top-n-sigma t          top-n-sigma parmeter (default: 0.0, 0.0 = disabled)
         --adaptive-target        adaptive-p sampling: (default: -1.00, <0.0 = disabled)
         --adaptive-decay         adaptive-p sampling: (default: 0.90)
         --adaptive-updt-w-cur    adaptive-p sampling: (default: false)
         --banned-string-file     file path of the list of banned strings on each line
         --banned-n               number of tokens banned in the phrase during rewind. -1 means all tokens: (default: 1)
         --allowlist-unicode-rule 
                                  rule for allowlisting unicode script and/or codepoints. disabled without any rule. format: `LOWER..UPPER,SCRIPT:BIAS`
                                  if unspecified: LOWER = 0, UPPER = -1(=max), SCRIPT="", BIAS = 0. at least one of LOWER, UPPER, or SCRIPT is required
                                  
         --allowlist-pieces       allowlist each token in argument. inherits max BIAS in --allowlist-unicode-rule. overrides --allowlist-unicode-rule
         --allowlist-keyword      keyword to expire earlier allowlist rules if matched during generation. does not affect later rules
         --allowlist-keyword-delay
                                  # tokens to delay matching for the first keyword (default: 296352751302)
         -l TOKEN_ID(+/-)BIAS     modifies the likelihood of token appearing in the completion,
                                  i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',
                                  or `--logit-bias 15043-1` to decrease likelihood of token ' Hello'
         --expiring-logit-bias-file
                                  ported from parent-upstream PR #1731
                                  
         --cfg-negative-prompt PROMPT
                                  negative prompt to use for guidance (default: '')
         --cfg-negative-prompt-file FNAME
                                  negative prompt file to use for guidance
         --cfg-scale N            strength of guidance (default: 1.0, 1.0 = disable)

template:

         --jinja                  set custom jinja chat template (default: template taken from model's metadata)
                                  if suffix/prefix are specified, template will be disabled
                                  only commonly used templates are accepted:
                                  https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
         --parallel-tool-calls    enable parallel tool calls
                                  
         --chat-template JINJA_TEMPLATE
                                  use jinja template for chat (default: disabled)
                                  
         --chat-template-file file_with_JINJA_TEMPLATE
                                  load jinja template for chat from the file
                                  
         --reasoning-format FORMAT
                                  controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:
                                  - none: leaves thoughts unparsed in `message.content`
                                  - deepseek: puts thoughts in `message.reasoning_content` (except in streaming mode, which behaves as `none`)
                                  - deepseek-legacy: keeps `<think>` tags in `message.content` while also populating `message.reasoning_content`
                                  (default: none)
  -rea,  --reasoning              [on|off|auto]Use reasoning/thinking in the chat ('on', 'off', or 'auto', default: 'auto' (detect from template))
         --chat-template-kwargs JSON
                                  sets additional params for the json template parser
         --reasoning-budget N     token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
         --reasoning-tokens FORMAT
                                  exclude reasoning tokens to select the slot more accurately.
                                  none: include all tokens
                                  auto: exclude all tokens between <think> and </think>
                                  Or comma separated start and end tokens such as [THINK],[/THINK]
                                  (default: auto)
         --reasoning-budget-message
                                  message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)
         --skip-chat-parsing      force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)
         --reasoning-budget N     token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)
         --no-prefill-assistant   whether to prefill the assistant's response if the last message is an assistant message (default: prefill enabled)
                                  when this flag is set, if the last message is an assistant message then it will be treated as a full message and not prefilled
                                  
         -ptc, --parallel-tool-calls
                                  enable parallel tool calls
                                  

grammar:

         --grammar GRAMMAR        BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '')
         --grammar-file FNAME     file to read grammar from
  -j,    --json-schema SCHEMA     JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
                                  For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead

embedding:

         --pooling {none,mean,cls,last}
                                  pooling type for embeddings, use model default if unspecified
         --attention {causal,non-causal}
                                  attention type for embeddings, use model default if unspecified

context hacking:

         --rope-scaling {none,linear,yarn}
                                  RoPE frequency scaling method, defaults to linear unless specified by the model
         --rope-scale N           RoPE context scaling factor, expands context by a factor of N
         --rope-freq-base N       RoPE base frequency, used by NTK-aware scaling (default: loaded from model)
         --rope-freq-scale N      RoPE frequency scaling factor, expands context by a factor of 1/N
         --yarn-orig-ctx N        YaRN: original context size of model (default: 0 = model training context size)
         --yarn-ext-factor N      YaRN: extrapolation mix factor (default: -1.0, 0.0 = full interpolation)
         --yarn-attn-factor N     YaRN: scale sqrt(t) or attention magnitude (default: -1.0)
         --yarn-beta-slow N       YaRN: high correction dim or alpha (default: -1.0)
         --yarn-beta-fast N       YaRN: low correction dim or beta (default: -1.0)
  -gan,  --grp-attn-n N           group-attention factor (default: 1)
  -gaw,  --grp-attn-w N           group-attention width (default: 512.0)
  -dkvc, --dump-kv-cache          verbose print of the KV cache
  -nkvo, --no-kv-offload          disable KV offload
  -ctk,  --cache-type-k TYPE      KV cache data type for K (default: f16)
  -ctv,  --cache-type-v TYPE      KV cache data type for V (default: f16)
  -ctk-first, --cache-type-k-first TYPE,N
                                  KV cache data type for the first N layers of K (default: f16,-1)
  -ctv-last,  --cache-type-k-last  TYPE,N
                                  KV cache data type for the last N layers of K  (default: f16,-1)
  -ctv-first, --cache-type-v-first TYPE,N
                                  KV cache data type for the first N layers of V (default: f16,-1)
  -ctk-last,  --cache-type-v-last  TYPE,N
                                  KV cache data type for the last N layers of V  (default: f16,-1)
  -mtprot, --mtp-requantize-output-tensor type
                                  Use output requantized to type for MTP (default: )
  -ctkd, --cache-type-k-draft TYPE
                                  KV cache data type for K for the draft model
  -ctvd, --cache-type-v-draft TYPE
                                  KV cache data type for V for the draft model

perplexity:

         --all-logits             return logits for all tokens in the batch (default: false)
         --hellaswag              compute HellaSwag score over random tasks from datafile supplied with -f
         --hellaswag-tasks N      number of tasks to use when computing the HellaSwag score (default: 400)
         --winogrande             compute Winogrande score over random tasks from datafile supplied with -f
         --winogrande-tasks N     number of tasks to use when computing the Winogrande score (default: 0)
         --multiple-choice        compute multiple choice score over random tasks from datafile supplied with -f
         --multiple-choice-tasks N
                                  number of tasks to use when computing the multiple choice score (default: 0)
         --kl-divergence          computes KL-divergence to logits provided via --kl-divergence-base
         --ppl-stride N           stride for perplexity calculation (default: 0)
         --ppl-output-type {0,1}  output type for perplexity calculation (default: 0)

parallel:

  -dt,   --defrag-thold N         KV cache defragmentation threshold (default: -1.0, < 0 - disabled)
  -mea,  --max-extra-alloc        Max extra VRAM allocation per GPU (default: 256)
  -np,   --parallel N             number of parallel sequences to decode (default: 1)
  -ns,   --sequences N            number of sequences to decode (default: 1)
  -kvu,  --kv-unified             server: let every slot address the full attention-KV ring instead of n_ctx/n_parallel;
                                  requires the shared-ring admission guard (default: disabled)
                                  (env: LLAMA_ARG_KV_UNIFIED / PXA_KV_UNIFIED)
  -no-kvu, --no-kv-unified        server: keep the static n_ctx/n_parallel per-slot split (default)
  -cb,   --cont-batching          enable continuous batching (a.k.a dynamic batching) (default: enabled)
  -nocb, --no-cont-batching       disable continuous batching

multi-modality:

         --mmproj FILE            path to a multimodal projector file for LLaVA. see examples/llava/README.md
         --image FILE             path to an image file. use with multimodal models. Specify multiple times for batching
         --image-min-tokens N     minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
         --image-max-tokens N     maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
         --mtmd-kq-type TYPE      data type for multimodality K*Q (default: f32)
         --no-context-shift       disable context-shift.
  --context-shift (auto|on|off|0|1)
                                  set context-shift (default: on)

backend:

         --rpc SERVERS            comma separated list of RPC servers
  -cuda, --cuda-params            comma separate list of cuda parameters
  -draft, --draft-params          comma separate list of draft model parameters
         --mlock                  force system to keep model in RAM rather than swapping or compressing
         --no-mmap                do not memory-map model (slower load but may reduce pageouts if not using mlock)
         --run-time-repack        repack tensors if interleaved variant is available
         --cpu-moe                keep all MoE weights in CPU memory
         --n-cpu-moe N            keep MoE weights of the first N layers in CPU memory
         --defer-experts          defer expert mmap residency on Linux to reduce model load time
         --prefetch-experts       stream mmap'd MoE expert weights into the page cache on Linux
         --prefetch-experts-threads N
                                  number of expert prefetch workers, tune to drive speed/type (default: auto)
         --fit-margin N           safety margin in MiB when auto-fitting model offloading
  -wgt, --worst-graph-tokens N    number of tokens to use for worst-case graph
         --fit                    automatically determine which tensors to offload to the GPU(s)
         --numa TYPE              attempt optimizations that help on some NUMA systems
                                    - distribute: spread execution evenly over all nodes
                                    - isolate: only spawn threads on CPUs on the node that execution started on
                                    - numactl: use the CPU map provided by numactl
                                  if run without this previously, it is recommended to drop the system page cache before using this
                                  see https://github.com/ggerganov/llama.cpp/issues/1437
  -ngl,  --gpu-layers N           number of layers to store in VRAM
  -ngld, --gpu-layers-draft N     number of layers to store in VRAM for the draft model
  -sm,   --split-mode SPLIT_MODE  how to split the model across multiple GPUs, one of:
                                    - none: use one GPU only
                                    - layer (default): split layers and KV across GPUs
                                    - tensor: PXA tensor split - split every splittable weight and the KV
                                              cache across GPUs. Admitted by a capability check on the file;
                                              if it cannot be had you get one line saying why, not a silent
                                              fall back to layer. Needs >= 2 GPUs and flash attention.
                                    - attn: the attention/head split as it shipped before 'tensor'
                                    - graph: split model tensors and computation graph across GPUs
                                  
  -ts,   --tensor-split SPLIT     fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1
  -dev,   --device dev1,dev2      comma-separated list of devices to use for offloading (none = don't offload)
                                  Example: CUDA0,CUDA1,RPC[192.168.0.1:8080]
                                  
  -devd,   --device-draft dev1,dev2
                                  comma-separated list of devices to use for offloading for the draft model (none = don't offload)
                                  Example: CUDA0,CUDA1,RPC[192.168.0.1:8080]
                                  
  -mg,   --main-gpu i             the GPU to use for the model (with split-mode = none),
                                  or for intermediate results and KV (with split-mode = row) (default: 0)
  --max-gpu i                     max. number of GPUs to use at a time with split mode 'graph', (default: 0)

model:

         --check-tensors          check model tensor data for invalid values (default: false)
         --override-kv KEY=TYPE:VALUE
                                  advanced option to override model metadata by key. may be specified multiple times.
                                  types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false
         --lora FNAME             apply LoRA adapter (can be repeated to use multiple adapters)
         --lora-scaled FNAME S    apply LoRA adapter with user defined scaling S (can be repeated to use multiple adapters)
         --control-vector FNAME   add a control vector
                                  note: this argument can be repeated to add multiple control vectors
         --control-vector-scaled FNAME SCALE
                                  add a control vector with user defined scaling SCALE
                                  note: this argument can be repeated to add multiple scaled control vectors
         --control-vector-layer-range START END
                                  layer range to apply the control vector(s) to, start and end inclusive
  -m,    --model FNAME            model path (default: models/$filename with filename from --hf-file
                                  or --model-url if set, otherwise models/7B/ggml-model-f16.gguf)
  -md,   --model-draft FNAME      draft model for speculative decoding (default: unused)
  -mu,   --model-url MODEL_URL    model download url (default: unused)
  -hfr,  --hf-repo REPO           Hugging Face model repository (default: unused)
  -hff,  --hf-file FILE           Hugging Face model file (default: unused)
  -hft,  --hf-token TOKEN         Hugging Face access token (default: value from HF_TOKEN environment variable)
  --recurrent-ckpt-mode MODE      checkpoint strategy for recurrent/hybrid speculative decoding
                                    auto         auto-select: per-step if CUDA full-GPU, gpu-fallback otherwise (default)
                                    per-step     save SSM state per draft step in VRAM; no re-decode on rejection
                                    gpu-fallback copy state to GPU buffer; re-decode on rejection
                                    cpu          serialise state via llama_state_seq; re-decode on rejection
  --spec-type SPEC[:k=v,...]      canonical speculative stage entry; repeat for a supported two-stage chain.
                                  types: none, draft, mtp, ngram-cache, ngram-simple, ngram-map-k, ngram-map-k4v, ngram-mod, suffix
                                  alias: ngram (also spelled prompt-lookup) = the measured-best n-gram variant with its measured
                                         knobs pre-filled -- the one to reach for on a model WITHOUT an MTP head; keys you name win
                                  canonical keys: n_max,n_min,p_min,ngram_size_n,ngram_size_m,ngram_min_hits,suffix_min_match_len,suffix_max_depth,suffix_corpus
                                  for comma-bearing string values, quote the value inside the stage payload for normal shell use
                                  if argv is passed directly without shell unescaping, the parser also accepts escaped commas as \,
                                  examples: --spec-type ngram
                                            --spec-type mtp:n_max=1,p_min=0.0
                                            --spec-type ngram --spec-type mtp   (cascade: n-gram on a match, MTP otherwise)
                                            --spec-type ngram-mod:n_max=64,n_min=2,ngram_size_n=8 --spec-type mtp:n_max=1,p_min=0.0
                                            --spec-type "suffix:n_max=16,n_min=2,suffix_min_match_len=5,suffix_
  --spec-autotune                 automatically tune speculative params to maximize tokens/sec

retrieval:

         --context-file FNAME     file to load context from (repeat to specify multiple files)
         --chunk-size N           minimum length of embedded text chunks (default: 64)
         --chunk-separator STRING 
                                  separator between chunks (default: '
                                  ')

passkey:

         --junk N                 number of times to repeat the junk text (default: 250)
         --pos N                  position of the passkey in the junk text (default: -1)

imatrix:

  -o,    --output FNAME           output file (default: 'imatrix.dat')
         --output-draft FNAME     paired draft output file (default: derived from --output)
         --output-frequency N     output the imatrix every N iterations (default: 10)
         --save-frequency N       save an imatrix copy every N iterations (default: 0)
         --process-output         collect data for the output tensor (default: false)
         --no-ppl                 do not compute perplexity (default: true)
         --chunk N                start processing the input from chunk N (default: 0)

bench:

  -pps                            is the prompt shared across parallel sequences (default: false)
  -npp n0,n1,...                  number of prompt tokens
  -ntg n0,n1,...                  number of text generation tokens
  -npl n0,n1,...                  number of parallel prompts

embedding:

         --embd-normalize         normalisation for embendings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)
         --embd-output-format     empty = default, "array" = [[],[]...], "json" = openai style, "json+" = same "json" + cosine similarity matrix
         --embd-separator         separator of embendings (default \n) for example "<#sep#>"

server:

         --host HOST              ip address to listen (default: 127.0.0.1)
         --port PORT              port to listen (default: 8080)
         --hot-model 'NAME=PATH [flags]'
                                  PXA hot swap: register one more model (repeatable). All registered models stay
                                  in pinned host RAM, one is on the cards; the request's model field picks it and
                                  a switch re-uploads it in parallel over every card, parking the other model's KV
                                  cache in RAM. Flags after PATH apply to that model only (e.g. -c 8192 -sm layer).
         --sleep-idle-seconds N   with --hot-model: after N seconds with no request in flight, park the model and
                                  free the cards. The next request loads it back. 0 (default) keeps the cards.
         --path PATH              path to serve static files from (default: )
         --embedding(s)           restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
         --webui NAME             controls which webui to server:
                                  - none: disable webui
                                  - auto: default webui 
                                  - llamacpp: llamacpp webui 
                                  (default: auto)
         --ui-mcp-proxy, --webui-mcp-proxy
                                  experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
         --api-key KEY            API key to use for authentication (default: none)
         --api-key-file FNAME     path to file containing API keys (default: none)
         --ssl-key-file FNAME     path to file a PEM-encoded SSL private key
         --ssl-cert-file FNAME    path to file a PEM-encoded SSL certificate
         --timeout N              server read/write timeout in seconds (default: 600)
         --threads-http N         number of threads used to process HTTP requests (default: -1)
         --system-prompt-file FNAME
                                  set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications
         --log-format {text,json} 
                                  log output format: json or text (default: json)
         --metrics                enable prometheus compatible metrics endpoint (default: disabled)
         --no-slots               disables slots monitoring endpoint (default: enabled)
         --slot-save-path PATH    path to save slot kv cache (default: disabled)
         --chat-template JINJA_TEMPLATE
                                  set custom jinja chat template (default: template taken from model's metadata)
                                  only commonly used templates are accepted:
                                  https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
  -sps,  --slot-prompt-similarity SIMILARITY
                                  how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled)
                                  
         --lora-init-without-apply
                                  load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled)

logging:

         --simple-io              use basic IO for better compatibility in subprocesses and limited consoles
  -ld,   --logdir LOGDIR          path under which to save YAML logs (no logging if unset)
         --log-test               Run simple logging test
         --log-disable            Disable trace logs
         --log-enable             Enable trace logs
         --log-file FNAME         Specify a log filename (without extension)
         --log-new                Create a separate new log file on start. Each log file will have unique name: "<name>.<ID>.log"
         --log-append             Don't truncate the old log file.

cvector:

  -o,    --output FNAME           output file (default: 'control_vector.gguf')
         --positive-file FNAME    positive prompts file, one prompt per line (default: 'examples/cvector-generator/positive.txt')
         --negative-file FNAME    negative prompts file, one prompt per line (default: 'examples/cvector-generator/negative.txt')
         --pca-batch N            batch size used for PCA. Larger batch runs faster, but uses more memory (default: 100)
         --pca-iter N             number of iterations used for PCA (default: 1000)
         --method {pca,mean}      dimensionality reduction method to be used (default: pca)

export-lora:

  -m,    --model                  model path from which to load base model (default '')
         --lora FNAME             path to LoRA adapter  (can be repeated to use multiple adapters)
         --lora-scaled FNAME S    path to LoRA adapter with user defined scaling S  (can be repeated to use multiple adapters)
  -t,    --threads N              number of threads to use during computation (default: 12)
  -o,    --output FNAME           output file (default: 'ggml-lora-merged-f16.gguf')

