using LLama.Abstractions; using System; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using LLama.Native; namespace LLama.Common { /// /// The parameters for initializing a LLama model. /// public record ModelParams : ILLamaParams { /// /// Model context size (n_ctx) /// public uint ContextSize { get; set; } = 512; /// /// the GPU that is used for scratch and small tensors /// public int MainGpu { get; set; } = 0; /// /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// public int GpuLayerCount { get; set; } = 20; /// /// Seed for the random number generator (seed) /// public uint Seed { get; set; } = 0xFFFFFFFF; /// /// Use f16 instead of f32 for memory kv (memory_f16) /// public bool UseFp16Memory { get; set; } = true; /// /// Use mmap for faster loads (use_mmap) /// public bool UseMemorymap { get; set; } = true; /// /// Use mlock to keep model in memory (use_mlock) /// public bool UseMemoryLock { get; set; } /// /// Compute perplexity over the prompt (perplexity) /// public bool Perplexity { get; set; } /// /// Model path (model) /// public string ModelPath { get; set; } /// /// List of LoRAs to apply /// public AdapterCollection LoraAdapters { get; set; } = new(); /// /// base model path for the lora adapter (lora_base) /// public string LoraBase { get; set; } = string.Empty; /// /// Number of threads (null = autodetect) (n_threads) /// public uint? Threads { get; set; } /// /// Number of threads to use for batch processing (null = autodetect) (n_threads) /// public uint? BatchThreads { get; set; } /// /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) /// public uint BatchSize { get; set; } = 512; /// /// Whether to use embedding mode. (embedding) Note that if this is set to true, /// The LLamaModel won't produce text response anymore. /// public bool EmbeddingMode { get; set; } /// /// how split tensors should be distributed across GPUs. /// /// "[ 3, 2 ]" will assign 60% of the data to GPU 0 and 40% to GPU 1. [JsonConverter(typeof(TensorSplitsCollectionConverter))] public TensorSplitsCollection TensorSplits { get; set; } = new(); /// /// RoPE base frequency /// public float? RopeFrequencyBase { get; set; } /// /// RoPE frequency scaling factor /// public float? RopeFrequencyScale { get; set; } /// public float? YarnExtrapolationFactor { get; set; } /// public float? YarnAttentionFactor { get; set; } /// public float? YarnBetaFast { get; set; } /// public float? YarnBetaSlow { get; set; } /// public uint? YarnOriginalContext { get; set; } /// public RopeScalingType? YarnScalingType { get; set; } /// /// Use experimental mul_mat_q kernels /// public bool MulMatQ { get; set; } /// /// Load vocab only (no weights) /// public bool VocabOnly { get; set; } /// /// The encoding to use to convert text for the model /// [JsonConverter(typeof(EncodingConverter))] public Encoding Encoding { get; set; } = Encoding.UTF8; /// /// /// /// The model path. [JsonConstructor] public ModelParams(string modelPath) { ModelPath = modelPath; } private ModelParams() { // This constructor (default parameterless constructor) is used by Newtonsoft to deserialize! ModelPath = ""; } /// /// /// /// The model path. /// Model context size (n_ctx) /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// Seed for the random number generator (seed) /// Whether to use f16 instead of f32 for memory kv (memory_f16) /// Whether to use mmap for faster loads (use_mmap) /// Whether to use mlock to keep model in memory (use_mlock) /// Thether to compute perplexity over the prompt (perplexity) /// Lora adapter path (lora_adapter) /// Base model path for the lora adapter (lora_base) /// Number of threads (-1 = autodetect) (n_threads) /// Batch size for prompt processing (must be >=32 to use BLAS) (n_batch) /// Whether to use embedding mode. (embedding) Note that if this is set to true, The LLamaModel won't produce text response anymore. /// RoPE base frequency. /// RoPE frequency scaling factor /// Use experimental mul_mat_q kernels /// The encoding to use to convert text for the model [Obsolete("Use object initializer to set all optional parameters")] public ModelParams(string modelPath, uint contextSize = 512, int gpuLayerCount = 20, uint seed = 1337, bool useFp16Memory = true, bool useMemorymap = true, bool useMemoryLock = false, bool perplexity = false, string loraAdapter = "", string loraBase = "", int threads = -1, uint batchSize = 512, bool embeddingMode = false, float? ropeFrequencyBase = null, float? ropeFrequencyScale = null, bool mulMatQ = false, string encoding = "UTF-8") { ContextSize = contextSize; GpuLayerCount = gpuLayerCount; Seed = seed; UseFp16Memory = useFp16Memory; UseMemorymap = useMemorymap; UseMemoryLock = useMemoryLock; Perplexity = perplexity; ModelPath = modelPath; LoraBase = loraBase; Threads = threads < 1 ? null : (uint)threads; BatchSize = batchSize; EmbeddingMode = embeddingMode; RopeFrequencyBase = ropeFrequencyBase; RopeFrequencyScale = ropeFrequencyScale; MulMatQ = mulMatQ; Encoding = Encoding.GetEncoding(encoding); LoraAdapters.Add(new LoraAdapter(loraAdapter, 1)); } } internal class EncodingConverter : JsonConverter { public override Encoding? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { var name = reader.GetString(); if (name == null) return null; return Encoding.GetEncoding(name); } public override void Write(Utf8JsonWriter writer, Encoding value, JsonSerializerOptions options) { writer.WriteStringValue(value.WebName); } } internal class TensorSplitsCollectionConverter : JsonConverter { public override TensorSplitsCollection? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { var arr = JsonSerializer.Deserialize(ref reader, options) ?? Array.Empty(); return new TensorSplitsCollection(arr); } public override void Write(Utf8JsonWriter writer, TensorSplitsCollection value, JsonSerializerOptions options) { JsonSerializer.Serialize(writer, value.Splits, options); } } }