using LLama.Abstractions; using System; using System.Collections.Generic; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; namespace LLama.Common { /// /// The parameters for initializing a LLama model. /// public record ModelParams : ILLamaParams { /// /// Model context size (n_ctx) /// public uint ContextSize { get; set; } = 512; /// /// the GPU that is used for scratch and small tensors /// public int MainGpu { get; set; } = 0; /// /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// public int GpuLayerCount { get; set; } = 20; /// /// Seed for the random number generator (seed) /// public uint Seed { get; set; } = 1686349486; /// /// Use f16 instead of f32 for memory kv (memory_f16) /// public bool UseFp16Memory { get; set; } = true; /// /// Use mmap for faster loads (use_mmap) /// public bool UseMemorymap { get; set; } = true; /// /// Use mlock to keep model in memory (use_mlock) /// public bool UseMemoryLock { get; set; } = false; /// /// Compute perplexity over the prompt (perplexity) /// public bool Perplexity { get; set; } = false; /// /// Model path (model) /// public string ModelPath { get; set; } /// /// List of LoRAs to apply /// public AdapterCollection LoraAdapters { get; set; } = new(); /// /// base model path for the lora adapter (lora_base) /// public string LoraBase { get; set; } = string.Empty; /// /// Number of threads (-1 = autodetect) (n_threads) /// public int Threads { get; set; } = Math.Max(Environment.ProcessorCount / 2, 1); /// /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) /// public uint BatchSize { get; set; } = 512; /// /// Whether to use embedding mode. (embedding) Note that if this is set to true, /// The LLamaModel won't produce text response anymore. /// public bool EmbeddingMode { get; set; } = false; /// /// how split tensors should be distributed across GPUs /// public float[]? TensorSplits { get; set; } /// /// RoPE base frequency /// public float RopeFrequencyBase { get; set; } = 10000.0f; /// /// RoPE frequency scaling factor /// public float RopeFrequencyScale { get; set; } = 1.0f; /// /// Use experimental mul_mat_q kernels /// public bool MulMatQ { get; set; } /// /// Load vocab only (no weights) /// public bool VocabOnly { get; set; } /// /// The encoding to use to convert text for the model /// [JsonConverter(typeof(EncodingConverter))] public Encoding Encoding { get; set; } = Encoding.UTF8; /// /// /// /// The model path. [JsonConstructor] public ModelParams(string modelPath) { ModelPath = modelPath; } private ModelParams() { // This constructor (default parameterless constructor) is used by Newtonsoft to deserialize! ModelPath = ""; } /// /// /// /// The model path. /// Model context size (n_ctx) /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// Seed for the random number generator (seed) /// Whether to use f16 instead of f32 for memory kv (memory_f16) /// Whether to use mmap for faster loads (use_mmap) /// Whether to use mlock to keep model in memory (use_mlock) /// Thether to compute perplexity over the prompt (perplexity) /// Lora adapter path (lora_adapter) /// Base model path for the lora adapter (lora_base) /// Number of threads (-1 = autodetect) (n_threads) /// Batch size for prompt processing (must be >=32 to use BLAS) (n_batch) /// Whether to use embedding mode. (embedding) Note that if this is set to true, The LLamaModel won't produce text response anymore. /// RoPE base frequency. /// RoPE frequency scaling factor /// Use experimental mul_mat_q kernels /// The encoding to use to convert text for the model [Obsolete("Use object initializer to set all optional parameters")] public ModelParams(string modelPath, uint contextSize = 512, int gpuLayerCount = 20, uint seed = 1337, bool useFp16Memory = true, bool useMemorymap = true, bool useMemoryLock = false, bool perplexity = false, string loraAdapter = "", string loraBase = "", int threads = -1, uint batchSize = 512, bool embeddingMode = false, float ropeFrequencyBase = 10000.0f, float ropeFrequencyScale = 1f, bool mulMatQ = false, string encoding = "UTF-8") { ContextSize = contextSize; GpuLayerCount = gpuLayerCount; Seed = seed; UseFp16Memory = useFp16Memory; UseMemorymap = useMemorymap; UseMemoryLock = useMemoryLock; Perplexity = perplexity; ModelPath = modelPath; LoraBase = loraBase; Threads = threads == -1 ? Math.Max(Environment.ProcessorCount / 2, 1) : threads; BatchSize = batchSize; EmbeddingMode = embeddingMode; RopeFrequencyBase = ropeFrequencyBase; RopeFrequencyScale = ropeFrequencyScale; MulMatQ = mulMatQ; Encoding = Encoding.GetEncoding(encoding); LoraAdapters.Add(new LoraAdapter(loraAdapter, 1)); } } internal class EncodingConverter : JsonConverter { public override Encoding? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { var name = reader.GetString(); if (name == null) return null; return Encoding.GetEncoding(name); } public override void Write(Utf8JsonWriter writer, Encoding value, JsonSerializerOptions options) { writer.WriteStringValue(value.WebName); } } }