mirror of
https://github.com/ikawrakow/ik_llama.cpp.git
synced 2026-06-28 04:30:15 -05:00
489 lines
20 KiB
C++
489 lines
20 KiB
C++
#pragma once
|
|
|
|
#include "llama-impl.h"
|
|
#include "llama-cparams.h"
|
|
#include "llama-sampling.h"
|
|
|
|
#include "llama-spec-features.h"
|
|
|
|
struct llama_model;
|
|
|
|
#include <vector>
|
|
#include <map>
|
|
#include <set>
|
|
#include <memory>
|
|
|
|
struct llama_kv_cell {
|
|
llama_pos pos = -1;
|
|
llama_pos delta = 0;
|
|
int32_t src = 0; // used by recurrent state models to copy states
|
|
|
|
std::set<llama_seq_id> seq_id;
|
|
|
|
bool has_seq_id(const llama_seq_id & id) const {
|
|
return seq_id.find(id) != seq_id.end();
|
|
}
|
|
|
|
bool is_empty() const {
|
|
return seq_id.empty();
|
|
}
|
|
|
|
bool is_same_seq(const llama_kv_cell & other) const {
|
|
return seq_id == other.seq_id;
|
|
}
|
|
};
|
|
|
|
// ring-buffer of cached KV data
|
|
struct llama_kv_cache {
|
|
bool has_shift = false;
|
|
bool do_defrag = false;
|
|
bool do_copy = false;
|
|
bool recurrent = false; // with recurrent state models, a cell can hold the state for more than one past token
|
|
bool hybrid = false;
|
|
bool v_trans = true; // the value tensor is transposed
|
|
|
|
// Note: The value of head isn't only used to optimize searching
|
|
// for a free KV slot. llama_decode_internal also uses it, so it
|
|
// cannot be freely changed after a slot has been allocated.
|
|
uint32_t head = 0;
|
|
uint32_t size = 0;
|
|
uint32_t used = 0; // used cells (i.e. at least one seq_id)
|
|
|
|
// computed before each graph build
|
|
uint32_t n = 0;
|
|
|
|
ggml_type type_k = GGML_TYPE_F16;
|
|
ggml_type type_v = GGML_TYPE_F16;
|
|
|
|
std::vector<llama_kv_cell> cells;
|
|
|
|
std::vector<struct ggml_tensor *> k_l; // per layer
|
|
std::vector<struct ggml_tensor *> v_l;
|
|
std::vector<struct ggml_tensor *> s_l; // per layer recurrent state storage (Qwen3Next)
|
|
|
|
// When true, the delta_net graph builder will enable per-step SSM state saves
|
|
bool save_per_step_ssm = false;
|
|
|
|
std::vector<llama_split_tensor> split_k_l;
|
|
std::vector<llama_split_tensor> split_v_l;
|
|
std::vector<llama_split_tensor> split_s_l;
|
|
|
|
// Per-device replicas of the MLA compressed-latent KV cache (-sm graph for DEEPSEEK2/GLM_DSA/MISTRAL4).
|
|
std::vector<llama_split_tensor> replicated_k_l;
|
|
|
|
std::vector<struct ggml_context *> ctxs;
|
|
std::vector<ggml_backend_buffer_t> bufs;
|
|
|
|
size_t total_size() const {
|
|
size_t size = 0;
|
|
for (ggml_backend_buffer_t buf : bufs) {
|
|
size += ggml_backend_buffer_get_size(buf);
|
|
}
|
|
return size;
|
|
}
|
|
|
|
// GPU-resident checkpoint for recurrent/hybrid speculative decoding
|
|
struct gpu_checkpoint {
|
|
std::vector<llama_kv_cell> cells_snapshot;
|
|
uint32_t head_snapshot = 0;
|
|
uint32_t used_snapshot = 0;
|
|
|
|
std::vector<ggml_tensor *> s_l_shadow;
|
|
|
|
std::vector<std::vector<ggml_tensor *>> split_s_l_shadow;
|
|
|
|
// Per-step SSM state checkpoints for speculative decoding.
|
|
std::vector<std::vector<ggml_tensor *>> per_step_ssm;
|
|
|
|
// Per-step conv feature buffer: stores qkv_mixed features from the
|
|
// verification forward pass so conv state can be reconstructed at any step.
|
|
// One tensor per recurrent layer, each sized [conv_dim * max_tokens].
|
|
//std::vector<std::vector<ggml_tensor *>> per_step_qkv;
|
|
std::vector<std::vector<ggml_tensor *>> per_step_conv;
|
|
|
|
int32_t per_step_n_tokens = 0;
|
|
int32_t per_step_max_allocated = 0;
|
|
int64_t per_step_ssm_state_size = 0;
|
|
int64_t per_step_conv_state_dim = 0;
|
|
int64_t per_step_conv_dim = 0;
|
|
int32_t per_step_d_conv = 0;
|
|
|
|
int selected_spec_mode = -1;
|
|
int fixed_spec_mode = LLAMA_SPEC_CKPT_NONE;
|
|
int32_t fixed_max_tokens = 0;
|
|
|
|
// Serialised sequence state for CPU mode
|
|
std::vector<uint8_t> cpu_state_data;
|
|
|
|
// Separate storage for per-step allocations
|
|
std::vector<struct ggml_context *> per_step_ctxs;
|
|
std::vector<ggml_backend_buffer_t> per_step_bufs;
|
|
|
|
std::vector<struct ggml_context *> shadow_ctxs;
|
|
std::vector<ggml_backend_buffer_t> shadow_bufs;
|
|
|
|
bool allocated = false;
|
|
bool shadow_conv_only = false;
|
|
bool saved = false;
|
|
|
|
~gpu_checkpoint() {
|
|
for (struct ggml_context * ctx : shadow_ctxs) {
|
|
ggml_free(ctx);
|
|
}
|
|
for (ggml_backend_buffer_t buf : shadow_bufs) {
|
|
ggml_backend_buffer_free(buf);
|
|
}
|
|
for (struct ggml_context * ctx : per_step_ctxs) {
|
|
ggml_free(ctx);
|
|
}
|
|
for (ggml_backend_buffer_t buf : per_step_bufs) {
|
|
ggml_backend_buffer_free(buf);
|
|
}
|
|
}
|
|
};
|
|
|
|
gpu_checkpoint ckpt;
|
|
|
|
bool checkpoint_alloc_shadows(bool conv_only_shadow = false);
|
|
bool checkpoint_supported() const;
|
|
bool checkpoint_save(ggml_backend_sched_t sched);
|
|
bool checkpoint_restore(ggml_backend_sched_t sched);
|
|
void checkpoint_delete();
|
|
|
|
// Per-step checkpoint: allocate, restore step k's full state (SSM + conv) to cache
|
|
bool per_step_alloc(const llama_model & model, int max_tokens);
|
|
bool per_step_restore(const llama_model & model, ggml_backend_sched_t sched, int step);
|
|
|
|
~llama_kv_cache() {
|
|
for (struct ggml_context * ctx : ctxs) {
|
|
ggml_free(ctx);
|
|
}
|
|
for (ggml_backend_buffer_t buf : bufs) {
|
|
ggml_backend_buffer_free(buf);
|
|
}
|
|
}
|
|
};
|
|
|
|
struct llama_control_vector {
|
|
std::vector<struct ggml_tensor *> tensors; // per layer
|
|
std::vector<struct ggml_context *> ctxs;
|
|
std::vector<ggml_backend_buffer_t> bufs;
|
|
|
|
int32_t layer_start = -1;
|
|
int32_t layer_end = -1;
|
|
|
|
struct ggml_tensor * tensor_for(int il) const {
|
|
if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) {
|
|
return nullptr;
|
|
}
|
|
return tensors[il];
|
|
}
|
|
|
|
struct ggml_tensor * apply_to(struct ggml_context * ctx, struct ggml_tensor * cur, int il) const {
|
|
ggml_tensor * layer_dir = tensor_for(il);
|
|
if (layer_dir != nullptr) {
|
|
cur = ggml_add(ctx, cur, layer_dir);
|
|
}
|
|
return cur;
|
|
}
|
|
|
|
~llama_control_vector() {
|
|
for (struct ggml_context * ctx : ctxs) {
|
|
ggml_free(ctx);
|
|
}
|
|
for (ggml_backend_buffer_t buf : bufs) {
|
|
ggml_backend_buffer_free(buf);
|
|
}
|
|
}
|
|
};
|
|
|
|
struct llama_context {
|
|
|
|
llama_context(const llama_model & model);
|
|
|
|
~llama_context();
|
|
|
|
const struct llama_model & model;
|
|
|
|
struct llama_cparams cparams;
|
|
struct llama_sampling sampling;
|
|
struct llama_kv_cache kv_self;
|
|
struct llama_context * mtp_target_ctx = nullptr;
|
|
struct llama_control_vector cvec;
|
|
|
|
std::vector<float> scale_data;
|
|
|
|
std::unordered_map<struct llama_lora_adapter *, float> lora_adapters;
|
|
|
|
std::vector<ggml_backend_t> backends;
|
|
#ifdef GGML_USE_METAL
|
|
ggml_backend_t backend_metal = nullptr;
|
|
#endif
|
|
#ifdef GGML_USE_BLAS
|
|
ggml_backend_t backend_blas = nullptr;
|
|
#endif
|
|
ggml_backend_t backend_cpu = nullptr;
|
|
|
|
bool has_evaluated_once = false;
|
|
|
|
int64_t t_start_us;
|
|
int64_t t_load_us;
|
|
int64_t t_p_eval_us = 0;
|
|
int64_t t_eval_us = 0;
|
|
|
|
int64_t t_compute_start_us = 0;
|
|
int64_t n_queued_tokens = 0;
|
|
|
|
int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1)
|
|
int32_t n_eval = 0; // number of eval calls
|
|
|
|
// host buffer for the model output (logits and embeddings)
|
|
ggml_backend_buffer_t buf_output = nullptr;
|
|
|
|
// decode output (2-dimensional array: [n_outputs][n_vocab])
|
|
size_t logits_size = 0; // capacity (of floats) for logits
|
|
float * logits = nullptr;
|
|
|
|
std::vector<int32_t> output_ids; // map batch token positions to ids of the logits and embd buffers
|
|
size_t output_size = 0; // capacity (of tokens positions) for the output buffers
|
|
int32_t n_outputs = 0; // number of actually-used outputs in the current ubatch or last logical batch
|
|
int32_t n_outputs_embd = 0; // number of embedding rows produced for the current logical batch
|
|
|
|
bool logits_all = false;
|
|
|
|
// embeddings output (2-dimensional array: [n_outputs][n_embd])
|
|
// populated only when pooling_type == LLAMA_POOLING_TYPE_NONE
|
|
size_t embd_size = 0; // capacity (of floats) for embeddings
|
|
float * embd = nullptr;
|
|
|
|
// sequence embeddings output (map of [n_embd] vectors)
|
|
// populated only when pooling_type != LLAMA_POOLING_TYPE_NONE
|
|
std::map<llama_seq_id, std::vector<float>> embd_seq;
|
|
|
|
// whether we are computing encoder output or decoder output
|
|
bool is_encoding = false;
|
|
|
|
// output of the encoder part of the encoder-decoder models
|
|
std::vector<float> embd_enc;
|
|
std::vector<std::set<llama_seq_id>> seq_ids_enc;
|
|
|
|
// memory buffers used to evaluate the model
|
|
std::vector<uint8_t> buf_compute_meta;
|
|
ggml_backend_sched_t sched = nullptr;
|
|
|
|
ggml_abort_callback abort_callback = nullptr;
|
|
void * abort_callback_data = nullptr;
|
|
|
|
const float * draft_input_hidden_state = nullptr;
|
|
size_t draft_input_hidden_state_n_floats = 0;
|
|
std::vector<float> draft_input_hidden_state_owned;
|
|
|
|
struct dflash_runtime {
|
|
struct target_window_state {
|
|
const float * features = nullptr;
|
|
size_t features_n_floats = 0;
|
|
int32_t features_n_rows = 0;
|
|
const float * append_features = nullptr;
|
|
size_t append_features_n_floats = 0;
|
|
int32_t append_features_n_rows = 0;
|
|
const llama_pos * positions = nullptr;
|
|
size_t positions_n = 0;
|
|
uint64_t version = 0;
|
|
int32_t keep_rows = 0;
|
|
int32_t append_rows = 0;
|
|
bool replace = false;
|
|
std::vector<float> features_owned;
|
|
std::vector<float> append_features_owned;
|
|
std::vector<llama_pos> positions_owned;
|
|
std::vector<float> features_padded;
|
|
std::vector<llama_pos> pos_ctx_data;
|
|
std::vector<float> kq_mask_data;
|
|
std::vector<float> kq_mask_swa_data;
|
|
};
|
|
|
|
struct kv_runtime_state {
|
|
std::vector<struct ggml_tensor *> k_ctx_cache;
|
|
std::vector<struct ggml_tensor *> v_ctx_cache;
|
|
std::vector<struct ggml_tensor *> k_ctx_workspace;
|
|
std::vector<struct ggml_tensor *> v_ctx_workspace;
|
|
struct ggml_context * cache_ctx = nullptr;
|
|
std::vector<ggml_backend_buffer_t> cache_bufs;
|
|
int32_t cache_write_pos = 0;
|
|
int32_t cache_n_filled = 0;
|
|
int32_t cache_update_rows = 0;
|
|
int32_t cache_reserved_rows = 0;
|
|
int32_t cache_view_write_pos = 0;
|
|
int32_t cache_view_n_filled = 0;
|
|
uint64_t cache_applied_window_version = 0;
|
|
bool cache_valid = false;
|
|
bool cache_view_valid = false;
|
|
int32_t workspace_write_pos = 0;
|
|
int32_t workspace_n_filled = 0;
|
|
int32_t workspace_reserved_rows = 0;
|
|
int32_t workspace_token_capacity = 0;
|
|
int32_t workspace_n_kv_total = 0;
|
|
uint64_t workspace_applied_window_version = 0;
|
|
bool workspace_valid = false;
|
|
bool workspace_sync_pending = false;
|
|
std::vector<uint8_t> cache_compute_meta;
|
|
std::vector<uint8_t> workspace_compute_meta;
|
|
ggml_backend_sched_t cache_sched = nullptr;
|
|
ggml_backend_sched_t workspace_sched = nullptr;
|
|
ggml_cgraph * cache_graph = nullptr;
|
|
ggml_cgraph * workspace_graph = nullptr;
|
|
int32_t cache_graph_rows = 0;
|
|
int32_t cache_graph_write_pos = 0;
|
|
int32_t workspace_graph_rows = 0;
|
|
int32_t workspace_graph_write_pos = 0;
|
|
struct ggml_tensor * cache_input_target_features = nullptr;
|
|
struct ggml_tensor * cache_input_pos_ctx = nullptr;
|
|
struct ggml_tensor * kq_mask_tensor = nullptr;
|
|
struct ggml_tensor * kq_mask_swa_tensor = nullptr;
|
|
};
|
|
|
|
struct capture_state {
|
|
std::vector<int32_t> layer_ids;
|
|
std::vector<std::vector<float>> layer_rows;
|
|
int32_t row_count = 0;
|
|
int32_t row_width = 0;
|
|
uint64_t capture_batch_id = 0;
|
|
std::vector<uint64_t> layer_seen_batch_id;
|
|
ggml_backend_sched_eval_callback prev_cb_eval = nullptr;
|
|
void * prev_cb_eval_user_data = nullptr;
|
|
};
|
|
|
|
struct input_state {
|
|
struct ggml_tensor * target_features = nullptr; // F32 [n_target_features, cross_ctx]
|
|
struct ggml_tensor * pos_ctx = nullptr; // I32 [cross_ctx]
|
|
struct ggml_tensor * kq_mask = nullptr; // F32 [cross_ctx + n_batch, GGML_PAD(n_batch)]
|
|
struct ggml_tensor * kq_mask_swa = nullptr; // F32 [cross_ctx + n_batch, GGML_PAD(n_batch)]
|
|
};
|
|
|
|
target_window_state target;
|
|
kv_runtime_state kv;
|
|
std::unique_ptr<capture_state> capture;
|
|
std::vector<float> feature_view_buffer;
|
|
input_state inputs;
|
|
int32_t visible_cross_ctx = 0;
|
|
llama_dflash_profile_stats profile;
|
|
};
|
|
dflash_runtime dflash;
|
|
using dflash_capture_state = dflash_runtime::capture_state;
|
|
|
|
const float * & dflash_target_features = dflash.target.features;
|
|
size_t & dflash_target_features_n_floats = dflash.target.features_n_floats;
|
|
int32_t & dflash_target_features_n_rows = dflash.target.features_n_rows;
|
|
const float * & dflash_target_append_features = dflash.target.append_features;
|
|
size_t & dflash_target_append_features_n_floats = dflash.target.append_features_n_floats;
|
|
int32_t & dflash_target_append_features_n_rows = dflash.target.append_features_n_rows;
|
|
const llama_pos * & dflash_target_positions = dflash.target.positions;
|
|
size_t & dflash_target_positions_n = dflash.target.positions_n;
|
|
uint64_t & dflash_target_window_version = dflash.target.version;
|
|
int32_t & dflash_target_window_keep_rows = dflash.target.keep_rows;
|
|
int32_t & dflash_target_window_append_rows = dflash.target.append_rows;
|
|
bool & dflash_target_window_replace = dflash.target.replace;
|
|
std::vector<float> & dflash_target_features_owned = dflash.target.features_owned;
|
|
std::vector<float> & dflash_target_append_features_owned = dflash.target.append_features_owned;
|
|
std::vector<llama_pos> & dflash_target_positions_owned = dflash.target.positions_owned;
|
|
std::vector<float> & dflash_target_features_padded = dflash.target.features_padded;
|
|
std::vector<float> & dflash_feature_view_buffer = dflash.feature_view_buffer;
|
|
std::vector<llama_pos> & dflash_pos_ctx_data = dflash.target.pos_ctx_data;
|
|
std::vector<float> & dflash_kq_mask_data = dflash.target.kq_mask_data;
|
|
std::vector<float> & dflash_kq_mask_swa_data = dflash.target.kq_mask_swa_data;
|
|
int32_t & dflash_visible_cross_ctx = dflash.visible_cross_ctx;
|
|
std::vector<struct ggml_tensor *> & dflash_k_ctx_cache = dflash.kv.k_ctx_cache;
|
|
std::vector<struct ggml_tensor *> & dflash_v_ctx_cache = dflash.kv.v_ctx_cache;
|
|
|
|
// Argmax token IDs from the DFlash draft graph, computed via GPU argmax.
|
|
// Populated in llama_decode_internal after graph compute.
|
|
std::vector<llama_token> dflash_draft_tokens;
|
|
struct ggml_tensor * dflash_draft_tokens_tensor = nullptr;
|
|
|
|
std::vector<struct ggml_tensor *> & dflash_k_ctx_workspace = dflash.kv.k_ctx_workspace;
|
|
std::vector<struct ggml_tensor *> & dflash_v_ctx_workspace = dflash.kv.v_ctx_workspace;
|
|
struct ggml_context * & dflash_cache_ctx = dflash.kv.cache_ctx;
|
|
std::vector<ggml_backend_buffer_t> & dflash_cache_bufs = dflash.kv.cache_bufs;
|
|
int32_t & dflash_kv_cache_write_pos = dflash.kv.cache_write_pos;
|
|
int32_t & dflash_kv_cache_n_filled = dflash.kv.cache_n_filled;
|
|
int32_t & dflash_kv_cache_update_rows = dflash.kv.cache_update_rows;
|
|
int32_t & dflash_kv_cache_reserved_rows = dflash.kv.cache_reserved_rows;
|
|
int32_t & dflash_kv_cache_view_write_pos = dflash.kv.cache_view_write_pos;
|
|
int32_t & dflash_kv_cache_view_n_filled = dflash.kv.cache_view_n_filled;
|
|
uint64_t & dflash_kv_cache_applied_window_version = dflash.kv.cache_applied_window_version;
|
|
bool & dflash_kv_cache_valid = dflash.kv.cache_valid;
|
|
bool & dflash_kv_cache_view_valid = dflash.kv.cache_view_valid;
|
|
int32_t & dflash_kv_workspace_write_pos = dflash.kv.workspace_write_pos;
|
|
int32_t & dflash_kv_workspace_n_filled = dflash.kv.workspace_n_filled;
|
|
int32_t & dflash_kv_workspace_reserved_rows = dflash.kv.workspace_reserved_rows;
|
|
int32_t & dflash_kv_workspace_token_capacity = dflash.kv.workspace_token_capacity;
|
|
int32_t & dflash_kv_workspace_n_kv_total = dflash.kv.workspace_n_kv_total;
|
|
uint64_t & dflash_kv_workspace_applied_window_version = dflash.kv.workspace_applied_window_version;
|
|
bool & dflash_kv_workspace_valid = dflash.kv.workspace_valid;
|
|
bool & dflash_kv_workspace_sync_pending = dflash.kv.workspace_sync_pending;
|
|
std::vector<uint8_t> & dflash_buf_compute_meta = dflash.kv.cache_compute_meta;
|
|
std::vector<uint8_t> & dflash_workspace_buf_compute_meta = dflash.kv.workspace_compute_meta;
|
|
ggml_backend_sched_t & dflash_sched = dflash.kv.cache_sched;
|
|
ggml_backend_sched_t & dflash_workspace_sched = dflash.kv.workspace_sched;
|
|
ggml_cgraph * & dflash_kv_graph = dflash.kv.cache_graph;
|
|
ggml_cgraph * & dflash_kv_workspace_graph = dflash.kv.workspace_graph;
|
|
int32_t & dflash_kv_graph_rows = dflash.kv.cache_graph_rows;
|
|
int32_t & dflash_kv_graph_write_pos = dflash.kv.cache_graph_write_pos;
|
|
int32_t & dflash_kv_workspace_graph_rows = dflash.kv.workspace_graph_rows;
|
|
int32_t & dflash_kv_workspace_graph_write_pos = dflash.kv.workspace_graph_write_pos;
|
|
struct ggml_tensor * & dflash_kv_input_target_features = dflash.kv.cache_input_target_features;
|
|
struct ggml_tensor * & dflash_kv_input_pos_ctx = dflash.kv.cache_input_pos_ctx;
|
|
struct ggml_tensor * & dflash_kq_mask_tensor = dflash.kv.kq_mask_tensor;
|
|
struct ggml_tensor * & dflash_kq_mask_swa_tensor = dflash.kv.kq_mask_swa_tensor;
|
|
std::unique_ptr<dflash_capture_state> & dflash_capture = dflash.capture;
|
|
llama_dflash_profile_stats & dflash_profile = dflash.profile;
|
|
struct ggml_tensor * & inp_dflash_target_features = dflash.inputs.target_features;
|
|
struct ggml_tensor * & inp_dflash_pos_ctx = dflash.inputs.pos_ctx;
|
|
struct ggml_tensor * & inp_dflash_kq_mask = dflash.inputs.kq_mask;
|
|
struct ggml_tensor * & inp_dflash_kq_mask_swa = dflash.inputs.kq_mask_swa;
|
|
|
|
// input tensors
|
|
struct ggml_tensor * inp_tokens; // I32 [n_batch]
|
|
struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch]
|
|
struct ggml_tensor * inp_pos; // I32 [n_batch]
|
|
struct ggml_tensor * inp_out_ids; // I32 [n_outputs]
|
|
struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch]
|
|
struct ggml_tensor * inp_KQ_mask_swa; // F32 [kv_size, n_batch]
|
|
struct ggml_tensor * inp_K_shift; // I32 [kv_size]
|
|
struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch]
|
|
struct ggml_tensor * inp_cls; // I32 [n_batch]
|
|
struct ggml_tensor * inp_s_copy; // I32 [kv_size]
|
|
struct ggml_tensor * inp_s_mask; // F32 [1, n_kv]
|
|
struct ggml_tensor * inp_s_seq; // I32 [n_kv, n_batch]
|
|
struct ggml_tensor * inp_s_seq_qnext; // I32 [1, n_batch]
|
|
struct ggml_tensor * inp_pos_bucket; // I32 [n_batch|n_kv, n_batch]
|
|
struct ggml_tensor * inp_embd_enc; // F32 [n_embd, n_outputs_enc]
|
|
struct ggml_tensor * inp_KQ_mask_cross; // F32 [n_outputs_enc, n_batch]
|
|
struct ggml_tensor * inp_scale = nullptr; // F32 [n_tokens]
|
|
struct ggml_tensor * inp_mtp_states = nullptr;
|
|
|
|
ggml_backend_t ggml_backend_by_name(const char * name);
|
|
|
|
struct Prev;
|
|
std::unique_ptr<Prev> prev;
|
|
std::unique_ptr<Prev> prev_mtp;
|
|
|
|
void reset_scheduler();
|
|
bool can_reuse_graph(const llama_batch & u_batch);
|
|
|
|
struct CacheCopy {
|
|
ggml_tensor * cpy = nullptr;
|
|
size_t step = 0;
|
|
};
|
|
std::vector<CacheCopy> cache_copies;
|
|
|
|
bool update_cache_copies();
|
|
|
|
bool ensure_dflash_kv_cache_tensors(int32_t cross_ctx);
|
|
void free_dflash_kv_cache_tensors();
|
|
|
|
bool prepare_mtp_graph_inputs(
|
|
struct llama_context & lctx);
|
|
void set_mtp_op_type(llama_mtp_op_type value);
|
|
|
|
};
|