using System;
using System.Collections.Generic;
using System.Linq;
namespace LLama.Native
{
#if NET6_0_OR_GREATER
///
/// Allows configuration of the native llama.cpp libraries to load and use.
/// All configuration must be done before using **any** other LLamaSharp methods!
///
public sealed class NativeLibraryConfig
{
///
/// Get the config instance
///
public static NativeLibraryConfig Instance { get; } = new();
///
/// Check if the native library has already been loaded. Configuration cannot be modified if this is true.
///
public static bool LibraryHasLoaded { get; internal set; } = false;
private string? _libraryPath;
private string? _libraryPathLLava;
private bool _useCuda = true;
private AvxLevel _avxLevel;
private bool _allowFallback = true;
private bool _skipCheck = false;
private bool _logging = false;
private LLamaLogLevel _logLevel = LLamaLogLevel.Info;
///
/// search directory -> priority level, 0 is the lowest.
///
private readonly List _searchDirectories = new List();
private static void ThrowIfLoaded()
{
if (LibraryHasLoaded)
throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!");
}
#region configurators
///
/// Load a specified native library as backend for LLamaSharp.
/// When this method is called, all the other configurations will be ignored.
///
/// The full path to the llama library to load.
/// The full path to the llava library to load.
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig WithLibrary(string? llamaPath, string? llavaPath)
{
ThrowIfLoaded();
_libraryPath = llamaPath;
_libraryPathLLava = llavaPath;
return this;
}
///
/// Configure whether to use cuda backend if possible.
///
///
///
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig WithCuda(bool enable = true)
{
ThrowIfLoaded();
_useCuda = enable;
return this;
}
///
/// Configure the prefferred avx support level of the backend.
///
///
///
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig WithAvx(AvxLevel level)
{
ThrowIfLoaded();
_avxLevel = level;
return this;
}
///
/// Configure whether to allow fallback when there's no match for preferred settings.
///
///
///
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig WithAutoFallback(bool enable = true)
{
ThrowIfLoaded();
_allowFallback = enable;
return this;
}
///
/// Whether to skip the check when you don't allow fallback. This option
/// may be useful under some complex conditions. For example, you're sure
/// you have your cublas configured but LLamaSharp take it as invalid by mistake.
///
///
///
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig SkipCheck(bool enable = true)
{
ThrowIfLoaded();
_skipCheck = enable;
return this;
}
///
/// Whether to output the logs to console when loading the native library with your configuration.
///
///
///
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig WithLogs(bool enable)
{
ThrowIfLoaded();
_logging = enable;
return this;
}
///
/// Enable console logging with the specified log logLevel.
///
///
///
/// Thrown if `LibraryHasLoaded` is true.
public NativeLibraryConfig WithLogs(LLamaLogLevel logLevel = LLamaLogLevel.Info)
{
ThrowIfLoaded();
_logging = true;
_logLevel = logLevel;
return this;
}
///
/// Add self-defined search directories. Note that the file stucture of the added
/// directories must be the same as the default directory. Besides, the directory
/// won't be used recursively.
///
///
///
public NativeLibraryConfig WithSearchDirectories(IEnumerable directories)
{
ThrowIfLoaded();
_searchDirectories.AddRange(directories);
return this;
}
///
/// Add self-defined search directories. Note that the file stucture of the added
/// directories must be the same as the default directory. Besides, the directory
/// won't be used recursively.
///
///
///
public NativeLibraryConfig WithSearchDirectory(string directory)
{
ThrowIfLoaded();
_searchDirectories.Add(directory);
return this;
}
#endregion
internal static Description CheckAndGatherDescription(LibraryName library)
{
if (Instance._allowFallback && Instance._skipCheck)
throw new ArgumentException("Cannot skip the check when fallback is allowed.");
var path = library switch
{
LibraryName.Llama => Instance._libraryPath,
LibraryName.LlavaShared => Instance._libraryPathLLava,
_ => throw new ArgumentException($"Unknown library name '{library}'", nameof(library)),
};
return new Description(
path,
library,
Instance._useCuda,
Instance._avxLevel,
Instance._allowFallback,
Instance._skipCheck,
Instance._logging,
Instance._logLevel,
Instance._searchDirectories.Concat(new[] { "./" }).ToArray()
);
}
internal static string AvxLevelToString(AvxLevel level)
{
return level switch
{
AvxLevel.None => string.Empty,
AvxLevel.Avx => "avx",
AvxLevel.Avx2 => "avx2",
AvxLevel.Avx512 => "avx512",
_ => throw new ArgumentException($"Unknown AvxLevel '{level}'")
};
}
///
/// Private constructor prevents new instances of this class being created
///
private NativeLibraryConfig()
{
// Automatically detect the highest supported AVX level
if (System.Runtime.Intrinsics.X86.Avx.IsSupported)
_avxLevel = AvxLevel.Avx;
if (System.Runtime.Intrinsics.X86.Avx2.IsSupported)
_avxLevel = AvxLevel.Avx2;
if (CheckAVX512())
_avxLevel = AvxLevel.Avx512;
}
private static bool CheckAVX512()
{
if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported)
return false;
// ReSharper disable UnusedVariable (ebx is used when < NET8)
var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0);
// ReSharper restore UnusedVariable
var vnni = (ecx & 0b_1000_0000_0000) != 0;
#if NET8_0_OR_GREATER
var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported;
var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported;
var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported;
#else
var f = (ebx & (1 << 16)) != 0;
var bw = (ebx & (1 << 30)) != 0;
var vbmi = (ecx & 0b_0000_0000_0010) != 0;
#endif
return vnni && vbmi && bw && f;
}
///
/// Avx support configuration
///
public enum AvxLevel
{
///
/// No AVX
///
None,
///
/// Advanced Vector Extensions (supported by most processors after 2011)
///
Avx,
///
/// AVX2 (supported by most processors after 2013)
///
Avx2,
///
/// AVX512 (supported by some processors after 2016, not widely supported)
///
Avx512,
}
internal record Description(string? Path, LibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging, LLamaLogLevel LogLevel, string[] SearchDirectories)
{
public override string ToString()
{
string avxLevelString = AvxLevel switch
{
AvxLevel.None => "NoAVX",
AvxLevel.Avx => "AVX",
AvxLevel.Avx2 => "AVX2",
AvxLevel.Avx512 => "AVX512",
_ => "Unknown"
};
string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }";
return $"NativeLibraryConfig Description:\n" +
$"- LibraryName: {Library}\n" +
$"- Path: '{Path}'\n" +
$"- PreferCuda: {UseCuda}\n" +
$"- PreferredAvxLevel: {avxLevelString}\n" +
$"- AllowFallback: {AllowFallback}\n" +
$"- SkipCheck: {SkipCheck}\n" +
$"- Logging: {Logging}\n" +
$"- LogLevel: {LogLevel}\n" +
$"- SearchDirectories and Priorities: {searchDirectoriesString}";
}
}
}
#endif
internal enum LibraryName
{
Llama,
LlavaShared
}
internal static class LibraryNameExtensions
{
public static string GetLibraryName(this LibraryName name)
{
switch (name)
{
case LibraryName.Llama:
return NativeApi.libraryName;
case LibraryName.LlavaShared:
return NativeApi.llavaLibraryName;
default:
throw new ArgumentOutOfRangeException(nameof(name), name, null);
}
}
}
}