using LLama.Abstractions;
using System;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using LLama.Native;
namespace LLama.Common
{
///
/// The parameters for initializing a LLama model.
///
public record ModelParams
: ILLamaParams
{
///
public uint? ContextSize { get; set; }
///
public int MainGpu { get; set; } = 0;
///
public int GpuLayerCount { get; set; } = 20;
///
public uint Seed { get; set; } = 0xFFFFFFFF;
///
public bool UseFp16Memory { get; set; } = true;
///
public bool UseMemorymap { get; set; } = true;
///
public bool UseMemoryLock { get; set; }
///
public bool Perplexity { get; set; }
///
public string ModelPath { get; set; }
///
public AdapterCollection LoraAdapters { get; set; } = new();
///
public string LoraBase { get; set; } = string.Empty;
///
public uint? Threads { get; set; }
///
public uint? BatchThreads { get; set; }
///
public uint BatchSize { get; set; } = 512;
///
public bool EmbeddingMode { get; set; }
///
[JsonConverter(typeof(TensorSplitsCollectionConverter))]
public TensorSplitsCollection TensorSplits { get; set; } = new();
///
public float? RopeFrequencyBase { get; set; }
///
public float? RopeFrequencyScale { get; set; }
///
public float? YarnExtrapolationFactor { get; set; }
///
public float? YarnAttentionFactor { get; set; }
///
public float? YarnBetaFast { get; set; }
///
public float? YarnBetaSlow { get; set; }
///
public uint? YarnOriginalContext { get; set; }
///
public RopeScalingType? YarnScalingType { get; set; }
///
public bool MulMatQ { get; set; }
///
public bool VocabOnly { get; set; }
///
[JsonConverter(typeof(EncodingConverter))]
public Encoding Encoding { get; set; } = Encoding.UTF8;
///
///
///
/// The model path.
[JsonConstructor]
public ModelParams(string modelPath)
{
ModelPath = modelPath;
}
private ModelParams()
{
// This constructor (default parameterless constructor) is used by Newtonsoft to deserialize!
ModelPath = "";
}
///
///
///
/// The model path.
/// Model context size (n_ctx)
/// Number of layers to run in VRAM / GPU memory (n_gpu_layers)
/// Seed for the random number generator (seed)
/// Whether to use f16 instead of f32 for memory kv (memory_f16)
/// Whether to use mmap for faster loads (use_mmap)
/// Whether to use mlock to keep model in memory (use_mlock)
/// Thether to compute perplexity over the prompt (perplexity)
/// Lora adapter path (lora_adapter)
/// Base model path for the lora adapter (lora_base)
/// Number of threads (-1 = autodetect) (n_threads)
/// Batch size for prompt processing (must be >=32 to use BLAS) (n_batch)
/// Whether to use embedding mode. (embedding) Note that if this is set to true, The LLamaModel won't produce text response anymore.
/// RoPE base frequency.
/// RoPE frequency scaling factor
/// Use experimental mul_mat_q kernels
/// The encoding to use to convert text for the model
[Obsolete("Use object initializer to set all optional parameters")]
public ModelParams(string modelPath, uint contextSize = 512, int gpuLayerCount = 20,
uint seed = 1337, bool useFp16Memory = true,
bool useMemorymap = true, bool useMemoryLock = false, bool perplexity = false,
string loraAdapter = "", string loraBase = "", int threads = -1, uint batchSize = 512,
bool embeddingMode = false,
float? ropeFrequencyBase = null, float? ropeFrequencyScale = null, bool mulMatQ = false,
string encoding = "UTF-8")
{
ContextSize = contextSize;
GpuLayerCount = gpuLayerCount;
Seed = seed;
UseFp16Memory = useFp16Memory;
UseMemorymap = useMemorymap;
UseMemoryLock = useMemoryLock;
Perplexity = perplexity;
ModelPath = modelPath;
LoraBase = loraBase;
Threads = threads < 1 ? null : (uint)threads;
BatchSize = batchSize;
EmbeddingMode = embeddingMode;
RopeFrequencyBase = ropeFrequencyBase;
RopeFrequencyScale = ropeFrequencyScale;
MulMatQ = mulMatQ;
Encoding = Encoding.GetEncoding(encoding);
LoraAdapters.Add(new LoraAdapter(loraAdapter, 1));
}
}
internal class EncodingConverter
: JsonConverter
{
public override Encoding? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var name = reader.GetString();
if (name == null)
return null;
return Encoding.GetEncoding(name);
}
public override void Write(Utf8JsonWriter writer, Encoding value, JsonSerializerOptions options)
{
writer.WriteStringValue(value.WebName);
}
}
internal class TensorSplitsCollectionConverter
: JsonConverter
{
public override TensorSplitsCollection? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var arr = JsonSerializer.Deserialize(ref reader, options) ?? Array.Empty();
return new TensorSplitsCollection(arr);
}
public override void Write(Utf8JsonWriter writer, TensorSplitsCollection value, JsonSerializerOptions options)
{
JsonSerializer.Serialize(writer, value.Splits, options);
}
}
}