using System; using System.Collections.Generic; using System.Runtime.InteropServices; using System.Text; namespace LLama.Native { using llama_token = Int32; internal unsafe partial class NativeApi { private const string libraryName = "llama"; [DllImport(libraryName)] public static extern LLamaContextParams llama_context_default_params(); [DllImport(libraryName)] public static extern bool llama_mmap_supported(); [DllImport(libraryName)] public static extern bool llama_mlock_supported(); /// /// Various functions for loading a ggml llama model. /// Allocate (almost) all memory needed for the model. /// Return NULL on failure /// /// /// /// [DllImport(libraryName)] public static extern IntPtr llama_init_from_file(string path_model, LLamaContextParams params_); /// /// Frees all allocated memory /// /// [DllImport(libraryName)] public static extern void llama_free(IntPtr ctx); /// /// Returns 0 on success /// /// /// /// /// how many threads to use. If <=0, will use std::thread::hardware_concurrency(), else the number given /// not great API - very likely to change /// Returns 0 on success [DllImport(libraryName)] public static extern int llama_model_quantize(string fname_inp, string fname_out, LLamaFtype ftype, int nthread); /// /// Apply a LoRA adapter to a loaded model /// path_base_model is the path to a higher quality model to use as a base for /// the layers modified by the adapter. Can be NULL to use the current loaded model. /// The model needs to be reloaded before applying a new adapter, otherwise the adapter /// will be applied on top of the previous one /// /// /// /// /// /// Returns 0 on success [DllImport(libraryName)] public static extern int llama_apply_lora_from_file(SafeLLamaContextHandle ctx, string path_lora, string path_base_model, int n_threads); /// /// Returns the number of tokens in the KV cache /// /// /// [DllImport(libraryName)] public static extern int llama_get_kv_cache_token_count(SafeLLamaContextHandle ctx); /// /// Sets the current rng seed. /// /// /// [DllImport(libraryName)] public static extern void llama_set_rng_seed(SafeLLamaContextHandle ctx, int seed); /// /// Returns the maximum size in bytes of the state (rng, logits, embedding /// and kv_cache) - will often be smaller after compacting tokens /// /// /// [DllImport(libraryName)] public static extern ulong llama_get_state_size(SafeLLamaContextHandle ctx); /// /// Copies the state to the specified destination address. /// Destination needs to have allocated enough memory. /// Returns the number of bytes copied /// /// /// /// [DllImport(libraryName)] public static extern ulong llama_copy_state_data(SafeLLamaContextHandle ctx, byte[] dest); /// /// Set the state reading from the specified address /// Returns the number of bytes read /// /// /// /// [DllImport(libraryName)] public static extern ulong llama_set_state_data(SafeLLamaContextHandle ctx, byte[] src); /// /// Load session file /// /// /// /// /// /// /// [DllImport(libraryName)] public static extern bool llama_load_session_file(SafeLLamaContextHandle ctx, string path_session, llama_token[] tokens_out, ulong n_token_capacity, ulong* n_token_count_out); /// /// Save session file /// /// /// /// /// /// [DllImport(libraryName)] public static extern bool llama_save_session_file(SafeLLamaContextHandle ctx, string path_session, llama_token[] tokens, ulong n_token_count); /// /// Run the llama inference to obtain the logits and probabilities for the next token. /// tokens + n_tokens is the provided batch of new tokens to process /// n_past is the number of tokens to use from previous eval calls /// /// /// /// /// /// /// Returns 0 on success [DllImport(libraryName)] public static extern int llama_eval(SafeLLamaContextHandle ctx, llama_token[] tokens, int n_tokens, int n_past, int n_threads); /// /// Convert the provided text into tokens. /// The tokens pointer must be large enough to hold the resulting tokens. /// Returns the number of tokens on success, no more than n_max_tokens /// Returns a negative number on failure - the number of tokens that would have been returned /// /// /// /// /// /// /// [DllImport(libraryName)] public static extern int llama_tokenize(SafeLLamaContextHandle ctx, string text, llama_token[] tokens, int n_max_tokens, bool add_bos); [DllImport(libraryName)] public static extern int llama_n_vocab(SafeLLamaContextHandle ctx); [DllImport(libraryName)] public static extern int llama_n_ctx(SafeLLamaContextHandle ctx); [DllImport(libraryName)] public static extern int llama_n_embd(SafeLLamaContextHandle ctx); /// /// Token logits obtained from the last call to llama_eval() /// The logits for the last token are stored in the last row /// Can be mutated in order to change the probabilities of the next token /// Rows: n_tokens /// Cols: n_vocab /// /// /// [DllImport(libraryName)] public static extern float* llama_get_logits(SafeLLamaContextHandle ctx); /// /// Get the embeddings for the input /// shape: [n_embd] (1-dimensional) /// /// /// [DllImport(libraryName)] public static extern float* llama_get_embeddings(SafeLLamaContextHandle ctx); /// /// Token Id -> String. Uses the vocabulary in the provided context /// /// /// /// Pointer to a string. [DllImport(libraryName)] public static extern IntPtr llama_token_to_str(SafeLLamaContextHandle ctx, llama_token token); [DllImport(libraryName)] public static extern llama_token llama_token_bos(); [DllImport(libraryName)] public static extern llama_token llama_token_eos(); [DllImport(libraryName)] public static extern llama_token llama_token_nl(); [DllImport(libraryName)] public static extern void llama_print_timings(SafeLLamaContextHandle ctx); [DllImport(libraryName)] public static extern void llama_reset_timings(SafeLLamaContextHandle ctx); /// /// Print system information /// /// [DllImport(libraryName)] public static extern IntPtr llama_print_system_info(); } }