censored@cfia-CENSORED:~/CENSORED/llama.cpp$ ./build/bin/llama-server --host 0.0.0.0 --port 8080 --model /opt/CENSORED/models/llm/Qwen2.5-7B-Instruct-Q4_K_M.gguf --flash-attn --cache-type-k q8_0 --cache-type-v q8_0 --ctx-size 122880 --threads-http 15 -np 15 --tensor-split 1.0,0.0,0.0 -sm none -ngl 99999 ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no ggml_cuda_init: found 3 CUDA devices: Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes Device 1: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes Device 2: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes build: 4338 (7b1ec53f) with cc (Ubuntu 12.3.0-1ubuntu1~22.04) 12.3.0 for x86_64-linux-gnu system info: n_threads = 16, n_threads_batch = 16, total_threads = 32 system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 520,610,700,750 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | main: HTTP server is listening, hostname: 0.0.0.0, port: 8080, http threads: 15 main: loading model srv load_model: loading model '/opt/CENSORED/models/llm/Qwen2.5-7B-Instruct-Q4_K_M.gguf' llama_load_model_from_file: using device CUDA0 (NVIDIA GeForce RTX 4090) - 23595 MiB free llama_model_loader: loaded meta data with 38 key-value pairs and 339 tensors from /opt/CENSORED/models/llm/Qwen2.5-7B-Instruct-Q4_K_M.gguf (version GGUF V3 (latest)) llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. llama_model_loader: - kv 0: general.architecture str = qwen2 llama_model_loader: - kv 1: general.type str = model llama_model_loader: - kv 2: general.name str = Qwen2.5 7B Instruct llama_model_loader: - kv 3: general.finetune str = Instruct llama_model_loader: - kv 4: general.basename str = Qwen2.5 llama_model_loader: - kv 5: general.size_label str = 7B llama_model_loader: - kv 6: general.license str = apache-2.0 llama_model_loader: - kv 7: general.license.link str = https://huggingface.co/Qwen/Qwen2.5-7... llama_model_loader: - kv 8: general.base_model.count u32 = 1 llama_model_loader: - kv 9: general.base_model.0.name str = Qwen2.5 7B llama_model_loader: - kv 10: general.base_model.0.organization str = Qwen llama_model_loader: - kv 11: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen2.5-7B llama_model_loader: - kv 12: general.tags arr[str,2] = ["chat", "text-generation"] llama_model_loader: - kv 13: general.languages arr[str,1] = ["en"] llama_model_loader: - kv 14: qwen2.block_count u32 = 28 llama_model_loader: - kv 15: qwen2.context_length u32 = 32768 llama_model_loader: - kv 16: qwen2.embedding_length u32 = 3584 llama_model_loader: - kv 17: qwen2.feed_forward_length u32 = 18944 llama_model_loader: - kv 18: qwen2.attention.head_count u32 = 28 llama_model_loader: - kv 19: qwen2.attention.head_count_kv u32 = 4 llama_model_loader: - kv 20: qwen2.rope.freq_base f32 = 1000000.000000 llama_model_loader: - kv 21: qwen2.attention.layer_norm_rms_epsilon f32 = 0.000001 llama_model_loader: - kv 22: general.file_type u32 = 15 llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 llama_model_loader: - kv 24: tokenizer.ggml.pre str = qwen2 llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,152064] = ["!", "\"", "#", "$", "%", "&", "'", ... llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,152064] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... llama_model_loader: - kv 28: tokenizer.ggml.eos_token_id u32 = 151645 llama_model_loader: - kv 29: tokenizer.ggml.padding_token_id u32 = 151643 llama_model_loader: - kv 30: tokenizer.ggml.bos_token_id u32 = 151643 llama_model_loader: - kv 31: tokenizer.ggml.add_bos_token bool = false llama_model_loader: - kv 32: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>... llama_model_loader: - kv 33: general.quantization_version u32 = 2 llama_model_loader: - kv 34: quantize.imatrix.file str = /models_out/Qwen2.5-7B-Instruct-GGUF/... llama_model_loader: - kv 35: quantize.imatrix.dataset str = /training_dir/calibration_datav3.txt llama_model_loader: - kv 36: quantize.imatrix.entries_count i32 = 196 llama_model_loader: - kv 37: quantize.imatrix.chunks_count i32 = 128 llama_model_loader: - type f32: 141 tensors llama_model_loader: - type q4_K: 169 tensors llama_model_loader: - type q6_K: 29 tensors llm_load_vocab: special tokens cache size = 22 llm_load_vocab: token to piece cache size = 0.9310 MB llm_load_print_meta: format = GGUF V3 (latest) llm_load_print_meta: arch = qwen2 llm_load_print_meta: vocab type = BPE llm_load_print_meta: n_vocab = 152064 llm_load_print_meta: n_merges = 151387 llm_load_print_meta: vocab_only = 0 llm_load_print_meta: n_ctx_train = 32768 llm_load_print_meta: n_embd = 3584 llm_load_print_meta: n_layer = 28 llm_load_print_meta: n_head = 28 llm_load_print_meta: n_head_kv = 4 llm_load_print_meta: n_rot = 128 llm_load_print_meta: n_swa = 0 llm_load_print_meta: n_embd_head_k = 128 llm_load_print_meta: n_embd_head_v = 128 llm_load_print_meta: n_gqa = 7 llm_load_print_meta: n_embd_k_gqa = 512 llm_load_print_meta: n_embd_v_gqa = 512 llm_load_print_meta: f_norm_eps = 0.0e+00 llm_load_print_meta: f_norm_rms_eps = 1.0e-06 llm_load_print_meta: f_clamp_kqv = 0.0e+00 llm_load_print_meta: f_max_alibi_bias = 0.0e+00 llm_load_print_meta: f_logit_scale = 0.0e+00 llm_load_print_meta: n_ff = 18944 llm_load_print_meta: n_expert = 0 llm_load_print_meta: n_expert_used = 0 llm_load_print_meta: causal attn = 1 llm_load_print_meta: pooling type = 0 llm_load_print_meta: rope type = 2 llm_load_print_meta: rope scaling = linear llm_load_print_meta: freq_base_train = 1000000.0 llm_load_print_meta: freq_scale_train = 1 llm_load_print_meta: n_ctx_orig_yarn = 32768 llm_load_print_meta: rope_finetuned = unknown llm_load_print_meta: ssm_d_conv = 0 llm_load_print_meta: ssm_d_inner = 0 llm_load_print_meta: ssm_d_state = 0 llm_load_print_meta: ssm_dt_rank = 0 llm_load_print_meta: ssm_dt_b_c_rms = 0 llm_load_print_meta: model type = 7B llm_load_print_meta: model ftype = Q4_K - Medium llm_load_print_meta: model params = 7.62 B llm_load_print_meta: model size = 4.36 GiB (4.91 BPW) llm_load_print_meta: general.name = Qwen2.5 7B Instruct llm_load_print_meta: BOS token = 151643 '<|endoftext|>' llm_load_print_meta: EOS token = 151645 '<|im_end|>' llm_load_print_meta: EOT token = 151645 '<|im_end|>' llm_load_print_meta: PAD token = 151643 '<|endoftext|>' llm_load_print_meta: LF token = 148848 'ÄĬ' llm_load_print_meta: FIM PRE token = 151659 '<|fim_prefix|>' llm_load_print_meta: FIM SUF token = 151661 '<|fim_suffix|>' llm_load_print_meta: FIM MID token = 151660 '<|fim_middle|>' llm_load_print_meta: FIM PAD token = 151662 '<|fim_pad|>' llm_load_print_meta: FIM REP token = 151663 '<|repo_name|>' llm_load_print_meta: FIM SEP token = 151664 '<|file_sep|>' llm_load_print_meta: EOG token = 151643 '<|endoftext|>' llm_load_print_meta: EOG token = 151645 '<|im_end|>' llm_load_print_meta: EOG token = 151662 '<|fim_pad|>' llm_load_print_meta: EOG token = 151663 '<|repo_name|>' llm_load_print_meta: EOG token = 151664 '<|file_sep|>' llm_load_print_meta: max token length = 256 llm_load_tensors: offloading 28 repeating layers to GPU llm_load_tensors: offloading output layer to GPU llm_load_tensors: offloaded 29/29 layers to GPU llm_load_tensors: CUDA0 model buffer size = 4168.09 MiB llm_load_tensors: CPU_Mapped model buffer size = 292.36 MiB ................................................................................. llama_new_context_with_model: n_seq_max = 15 llama_new_context_with_model: n_ctx = 122880 llama_new_context_with_model: n_ctx_per_seq = 8192 llama_new_context_with_model: n_batch = 2048 llama_new_context_with_model: n_ubatch = 512 llama_new_context_with_model: flash_attn = 1 llama_new_context_with_model: freq_base = 1000000.0 llama_new_context_with_model: freq_scale = 1 llama_new_context_with_model: n_ctx_per_seq (8192) < n_ctx_train (32768) -- the full capacity of the model will not be utilized llama_kv_cache_init: CUDA0 KV buffer size = 3570.00 MiB llama_new_context_with_model: KV self size = 3570.00 MiB, K (q8_0): 1785.00 MiB, V (q8_0): 1785.00 MiB llama_new_context_with_model: CUDA_Host output buffer size = 8.70 MiB llama_new_context_with_model: CUDA0 compute buffer size = 388.00 MiB llama_new_context_with_model: CUDA_Host compute buffer size = 247.01 MiB llama_new_context_with_model: graph nodes = 875 llama_new_context_with_model: graph splits = 2 common_init_from_params: setting dry_penalty_last_n to ctx_size = 122880 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) srv init: initializing slots, n_slots = 15 slot init: id 0 | task -1 | new slot n_ctx_slot = 8192 slot init: id 1 | task -1 | new slot n_ctx_slot = 8192 slot init: id 2 | task -1 | new slot n_ctx_slot = 8192 slot init: id 3 | task -1 | new slot n_ctx_slot = 8192 slot init: id 4 | task -1 | new slot n_ctx_slot = 8192 slot init: id 5 | task -1 | new slot n_ctx_slot = 8192 slot init: id 6 | task -1 | new slot n_ctx_slot = 8192 slot init: id 7 | task -1 | new slot n_ctx_slot = 8192 slot init: id 8 | task -1 | new slot n_ctx_slot = 8192 slot init: id 9 | task -1 | new slot n_ctx_slot = 8192 slot init: id 10 | task -1 | new slot n_ctx_slot = 8192 slot init: id 11 | task -1 | new slot n_ctx_slot = 8192 slot init: id 12 | task -1 | new slot n_ctx_slot = 8192 slot init: id 13 | task -1 | new slot n_ctx_slot = 8192 slot init: id 14 | task -1 | new slot n_ctx_slot = 8192 main: model loaded main: chat template, built_in: 1, chat_example: '<|im_start|>system You are a helpful assistant<|im_end|> <|im_start|>user Hello<|im_end|> <|im_start|>assistant Hi there<|im_end|> <|im_start|>user How are you?<|im_end|> <|im_start|>assistant ' main: server is listening on http://0.0.0.0:8080 - starting the main loop srv update_slots: all slots are idle slot launch_slot_: id 0 | task 0 | processing task slot update_slots: id 0 | task 0 | new prompt, n_ctx_slot = 8192, n_keep = 8192, n_prompt_tokens = 2310 slot update_slots: id 0 | task 0 | kv cache rm [0, end) slot update_slots: id 0 | task 0 | prompt processing progress, n_past = 2048, n_tokens = 2048, progress = 0.886580 slot update_slots: id 0 | task 0 | kv cache rm [2048, end) slot update_slots: id 0 | task 0 | prompt processing progress, n_past = 2310, n_tokens = 262, progress = 1.000000 slot update_slots: id 0 | task 0 | prompt done, n_past = 2310, n_tokens = 262 slot release: id 0 | task 0 | stop processing: n_past = 2408, truncated = 0 slot print_timing: id 0 | task 0 | prompt eval time = 257.79 ms / 2310 tokens ( 0.11 ms per token, 8960.68 tokens per second) eval time = 754.05 ms / 99 tokens ( 7.62 ms per token, 131.29 tokens per second) total time = 1011.84 ms / 2409 tokens srv update_slots: all slots are idle slot launch_slot_: id 0 | task 101 | processing task slot update_slots: id 0 | task 101 | new prompt, n_ctx_slot = 8192, n_keep = 8192, n_prompt_tokens = 2310 slot update_slots: id 0 | task 101 | need to evaluate at least 1 token to generate logits, n_past = 2310, n_prompt_tokens = 2310 slot update_slots: id 0 | task 101 | kv cache rm [2309, end) slot update_slots: id 0 | task 101 | prompt processing progress, n_past = 2310, n_tokens = 1, progress = 0.000433 slot update_slots: id 0 | task 101 | prompt done, n_past = 2310, n_tokens = 1 slot release: id 0 | task 101 | stop processing: n_past = 2397, truncated = 0 slot print_timing: id 0 | task 101 | prompt eval time = 8.38 ms / 1 tokens ( 8.38 ms per token, 119.33 tokens per second) eval time = 658.42 ms / 88 tokens ( 7.48 ms per token, 133.65 tokens per second) total time = 666.80 ms / 89 tokens srv update_slots: all slots are idle slot launch_slot_: id 0 | task 190 | processing task slot update_slots: id 0 | task 190 | new prompt, n_ctx_slot = 8192, n_keep = 8192, n_prompt_tokens = 2310 slot update_slots: id 0 | task 190 | need to evaluate at least 1 token to generate logits, n_past = 2310, n_prompt_tokens = 2310 slot update_slots: id 0 | task 190 | kv cache rm [2309, end) slot update_slots: id 0 | task 190 | prompt processing progress, n_past = 2310, n_tokens = 1, progress = 0.000433 slot update_slots: id 0 | task 190 | prompt done, n_past = 2310, n_tokens = 1 slot release: id 0 | task 190 | stop processing: n_past = 2405, truncated = 0 slot print_timing: id 0 | task 190 | prompt eval time = 8.42 ms / 1 tokens ( 8.42 ms per token, 118.79 tokens per second) eval time = 719.02 ms / 96 tokens ( 7.49 ms per token, 133.52 tokens per second) total time = 727.43 ms / 97 tokens srv update_slots: all slots are idle slot launch_slot_: id 0 | task 287 | processing task slot update_slots: id 0 | task 287 | new prompt, n_ctx_slot = 8192, n_keep = 8192, n_prompt_tokens = 2311 slot update_slots: id 0 | task 287 | kv cache rm [100, end) slot update_slots: id 0 | task 287 | prompt processing progress, n_past = 2148, n_tokens = 2048, progress = 0.886196 slot update_slots: id 0 | task 287 | kv cache rm [2148, end) slot update_slots: id 0 | task 287 | prompt processing progress, n_past = 2311, n_tokens = 163, progress = 0.956729 slot update_slots: id 0 | task 287 | prompt done, n_past = 2311, n_tokens = 163 slot release: id 0 | task 287 | stop processing: n_past = 2383, truncated = 0 slot print_timing: id 0 | task 287 | prompt eval time = 215.83 ms / 2211 tokens ( 0.10 ms per token, 10244.03 tokens per second) eval time = 544.57 ms / 73 tokens ( 7.46 ms per token, 134.05 tokens per second) total time = 760.40 ms / 2284 tokens srv update_slots: all slots are idle