From 52be1eec506af2b24274b29eb95a1d0494c14ddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 06:49:37 +0000 Subject: [PATCH 01/11] build(deps): bump Microsoft.VisualStudio.Validation Bumps [Microsoft.VisualStudio.Validation](https://github.com/microsoft/vs-validation) from 17.6.11 to 17.8.8. - [Release notes](https://github.com/microsoft/vs-validation/releases) - [Commits](https://github.com/microsoft/vs-validation/commits) --- updated-dependencies: - dependency-name: Microsoft.VisualStudio.Validation dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- LLama.WebAPI/LLama.WebAPI.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama.WebAPI/LLama.WebAPI.csproj b/LLama.WebAPI/LLama.WebAPI.csproj index bdb2ad8a..a8c1179c 100644 --- a/LLama.WebAPI/LLama.WebAPI.csproj +++ b/LLama.WebAPI/LLama.WebAPI.csproj @@ -7,7 +7,7 @@ - + From 217c67b7578bf9a80fe5c879b93b0af25419e5e4 Mon Sep 17 00:00:00 2001 From: Rinne Date: Sat, 25 Nov 2023 01:44:29 +0800 Subject: [PATCH 02/11] fix: chinese encoding error. --- LLama.Examples/Assets/chat-with-bob.txt | 16 +++-- .../Assets/chat-with-kunkun-chinese.txt | 8 +++ LLama.Examples/Examples/ChatChineseGB2312.cs | 69 +++++++++++++++++++ LLama.Examples/Examples/Runner.cs | 1 + LLama.Examples/LLama.Examples.csproj | 5 +- LLama/Native/NativeApi.cs | 14 ++++ 6 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 LLama.Examples/Assets/chat-with-kunkun-chinese.txt create mode 100644 LLama.Examples/Examples/ChatChineseGB2312.cs diff --git a/LLama.Examples/Assets/chat-with-bob.txt b/LLama.Examples/Assets/chat-with-bob.txt index ad494d83..20f07542 100644 --- a/LLama.Examples/Assets/chat-with-bob.txt +++ b/LLama.Examples/Assets/chat-with-bob.txt @@ -1,7 +1,9 @@ -Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. - -User: Hello, Bob. -Bob: Hello. How may I help you today? -User: Please tell me the largest city in Europe. -Bob: Sure. The largest city in Europe is Moscow, the capital of Russia. -User: \ No newline at end of file +[s]##Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. +## +[u]##Hello, Bob. +## +[a]##Hello. How may I help you today? +## +[u]##Please tell me the largest city in Europe. +## +[a]## \ No newline at end of file diff --git a/LLama.Examples/Assets/chat-with-kunkun-chinese.txt b/LLama.Examples/Assets/chat-with-kunkun-chinese.txt new file mode 100644 index 00000000..295e24d5 --- /dev/null +++ b/LLama.Examples/Assets/chat-with-kunkun-chinese.txt @@ -0,0 +1,8 @@ +Ö¸ÁÏÂÃæÊÇÒ»¶ÎÄãºÍÓû§µÄ¶Ô»°£¬Äã½ÐÀ¤À¤£¬ÊÇÒ»¸öÔÚ¸÷·½Ã涼ӵÓзḻ¾­ÑéµÄÖúÀí£¬Äã·Ç³£ÀÖÓڻشðÓû§µÄÎÊÌâºÍ°ïÖúÓû§¡£ + +ÓÑô£ºÄãºÃ£¬À¤À¤¡£ +À¤À¤£ºÄãºÃ£¬ÓÐʲôÎÒÄÜ°ïÖúÄãµÄÂ𣿠+ÓÑô£ºÖйúµÄÊ׶¼ÊÇÄÄ×ù³ÇÊУ¿ +À¤À¤£ºÖйúµÄÊ׶¼ÊDZ±¾©ÊС£ +ÓÑô£ºÌØÀÊÆÕÊÇË­£¿ +À¤À¤£º \ No newline at end of file diff --git a/LLama.Examples/Examples/ChatChineseGB2312.cs b/LLama.Examples/Examples/ChatChineseGB2312.cs new file mode 100644 index 00000000..ff27b962 --- /dev/null +++ b/LLama.Examples/Examples/ChatChineseGB2312.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using LLama.Common; + +namespace LLama.Examples.Examples +{ + public class ChatChineseGB2312 + { + private static string ConvertFromEncodingToAnother(string input, Encoding original, Encoding target) + { + byte[] bytes = original.GetBytes(input); + var convertedBytes = Encoding.Convert(original, target, bytes); + return target.GetString(convertedBytes); + } + + public static async Task Run() + { + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); // Register gb2312 encoding + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/chat-with-kunkun-chinese.txt", encoding: Encoding.GetEncoding("gb2312")).Trim(); + prompt = ConvertFromEncodingToAnother(prompt, Encoding.GetEncoding("gb2312"), Encoding.UTF8); + + var parameters = new ModelParams(modelPath) + { + ContextSize = 1024, + Seed = 1337, + GpuLayerCount = 20, + Encoding = Encoding.UTF8 + }; + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters); + var executor = new InteractiveExecutor(context); + + var session = new ChatSession(executor).WithHistoryTransform(new LLamaTransforms.DefaultHistoryTransform("用户")); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("This example shows how to use Chinese with gb2312 encoding, which is common in windows. It's recommended" + + " to use https://huggingface.co/hfl/chinese-alpaca-2-7b-gguf/blob/main/ggml-model-q5_0.gguf, which has been verified by LLamaSharp developers."); + Console.ForegroundColor = ConsoleColor.White; + + // show the prompt + Console.Write(prompt); + while (true) + { + await foreach (var text in session.ChatAsync(prompt, new InferenceParams() + { + Temperature = 0.3f, + TopK = 5, + TopP = 0.85f, + AntiPrompts = new List { "用户：" }, + MaxTokens = 2048, + RepeatPenalty = 1.05f + })) + { + //Console.Write(text); + Console.Write(ConvertFromEncodingToAnother(text, Encoding.UTF8, Encoding.GetEncoding("gb2312"))); + } + + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } + } +} diff --git a/LLama.Examples/Examples/Runner.cs b/LLama.Examples/Examples/Runner.cs index f2f1351f..aadc1990 100644 --- a/LLama.Examples/Examples/Runner.cs +++ b/LLama.Examples/Examples/Runner.cs @@ -23,6 +23,7 @@ public class Runner { "Coding Assistant.", CodingAssistant.Run }, { "Batch Decoding.", BatchedDecoding.Run }, { "SK Kernel Memory.", KernelMemory.Run }, + { "Chinese gb2312 chat", ChatChineseGB2312.Run }, { "Exit", async () => Environment.Exit(0) } }; diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index 42170e5c..3cd4ba1e 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -1,4 +1,4 @@ - + Exe @@ -67,6 +67,9 @@ PreserveNewest + + PreserveNewest + diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 074a8e9f..9ef2d1aa 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -9,6 +9,17 @@ namespace LLama.Native { using llama_token = Int32; + public enum LLamaTokenType + { + LLAMA_TOKEN_TYPE_UNDEFINED = 0, + LLAMA_TOKEN_TYPE_NORMAL = 1, + LLAMA_TOKEN_TYPE_UNKNOWN = 2, + LLAMA_TOKEN_TYPE_CONTROL = 3, + LLAMA_TOKEN_TYPE_USER_DEFINED = 4, + LLAMA_TOKEN_TYPE_UNUSED = 5, + LLAMA_TOKEN_TYPE_BYTE = 6, + } + /// /// Callback from llama.cpp with log messages /// @@ -243,6 +254,9 @@ namespace LLama.Native } } + [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern LLamaTokenType llama_token_get_type(SafeLlamaModelHandle model, llama_token token); + /// /// Get the size of the context window for the model for this context /// From 605d1ddb3fdd1b31482c700cc11e220dbec3ccf9 Mon Sep 17 00:00:00 2001 From: Rinne Date: Sat, 25 Nov 2023 02:06:25 +0800 Subject: [PATCH 03/11] revert a change. --- LLama.Examples/Assets/chat-with-bob.txt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/LLama.Examples/Assets/chat-with-bob.txt b/LLama.Examples/Assets/chat-with-bob.txt index 20f07542..ad494d83 100644 --- a/LLama.Examples/Assets/chat-with-bob.txt +++ b/LLama.Examples/Assets/chat-with-bob.txt @@ -1,9 +1,7 @@ -[s]##Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. -## -[u]##Hello, Bob. -## -[a]##Hello. How may I help you today? -## -[u]##Please tell me the largest city in Europe. -## -[a]## \ No newline at end of file +Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision. + +User: Hello, Bob. +Bob: Hello. How may I help you today? +User: Please tell me the largest city in Europe. +Bob: Sure. The largest city in Europe is Moscow, the capital of Russia. +User: \ No newline at end of file From 6d86219d710c19b09a1a122b59c9feb7b96a6884 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 06:17:47 +0000 Subject: [PATCH 04/11] build(deps): bump xunit.runner.visualstudio from 2.5.3 to 2.5.4 Bumps [xunit.runner.visualstudio](https://github.com/xunit/visualstudio.xunit) from 2.5.3 to 2.5.4. - [Release notes](https://github.com/xunit/visualstudio.xunit/releases) - [Commits](https://github.com/xunit/visualstudio.xunit/compare/2.5.3...2.5.4) --- updated-dependencies: - dependency-name: xunit.runner.visualstudio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- LLama.Unittest/LLama.Unittest.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama.Unittest/LLama.Unittest.csproj b/LLama.Unittest/LLama.Unittest.csproj index 148f7660..fbaee5ed 100644 --- a/LLama.Unittest/LLama.Unittest.csproj +++ b/LLama.Unittest/LLama.Unittest.csproj @@ -16,7 +16,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all From 36adc4af32161dca3f0c1a4a997748d3b9c7e03e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 06:17:53 +0000 Subject: [PATCH 05/11] build(deps): bump Spectre.Console from 0.47.0 to 0.48.0 Bumps [Spectre.Console](https://github.com/spectreconsole/spectre.console) from 0.47.0 to 0.48.0. - [Release notes](https://github.com/spectreconsole/spectre.console/releases) - [Commits](https://github.com/spectreconsole/spectre.console/compare/0.47.0...0.48.0) --- updated-dependencies: - dependency-name: Spectre.Console dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- LLama.Examples/LLama.Examples.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index 87b57dcd..d158f05f 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -1,4 +1,4 @@ - + Exe @@ -31,7 +31,7 @@ - + From adb3dcecc620d326a600b1fedd8b148ad3156a98 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Mon, 27 Nov 2023 23:05:28 +0000 Subject: [PATCH 06/11] Update README.md Minor spelling fixes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3d2643e..c96cd5a9 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **The C#/.NET binding of [llama.cpp](https://github.com/ggerganov/llama.cpp). It provides higher-level APIs to inference the LLaMA Models and deploy it on local device with C#/.NET. It works on -both Windows, Linux and MAC without requirment for compiling llama.cpp yourself. Even without GPU or not enought GPU memory, you can still apply LLaMA models well with this repo. 🤗** +both Windows, Linux and MAC without requirement for compiling llama.cpp yourself. Even without a GPU or not enough GPU memory, you can still apply LLaMA models well with this repo. 🤗** **Furthermore, it provides integrations with other projects such as [semantic-kernel](https://github.com/microsoft/semantic-kernel), [kernel-memory](https://github.com/microsoft/kernel-memory) and [BotSharp](https://github.com/SciSharp/BotSharp) to provide higher-level applications.** From b05c3154f49225900f3fc4405fff78c3c500ce03 Mon Sep 17 00:00:00 2001 From: Rinne Date: Tue, 28 Nov 2023 20:58:32 +0800 Subject: [PATCH 07/11] feat: allow customized search path for native library loading. --- LLama/ChatSession.cs | 24 +++++--- LLama/Native/NativeApi.Load.cs | 20 ++++--- LLama/Native/NativeLibraryConfig.cs | 88 ++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 17 deletions(-) diff --git a/LLama/ChatSession.cs b/LLama/ChatSession.cs index 7ee99590..5c535a6b 100644 --- a/LLama/ChatSession.cs +++ b/LLama/ChatSession.cs @@ -152,14 +152,24 @@ namespace LLama foreach (var inputTransform in InputTransformPipeline) prompt = inputTransform.Transform(prompt); - History.Messages.Add(new ChatHistory.Message(AuthorRole.User, prompt)); - - if (_executor is InteractiveExecutor executor) + // TODO: need to be refactored. + if (_executor is InteractiveExecutor executor && ((InteractiveExecutorState)executor.GetStateData()).IsPromptRun) { - InteractiveExecutorState state = (InteractiveExecutorState)executor.GetStateData(); - prompt = state.IsPromptRun - ? HistoryTransform.HistoryToText(History) - : prompt; + History.Messages.Add(new ChatHistory.Message(AuthorRole.System, prompt)); + var converted_prompt = HistoryTransform.HistoryToText(History); + // Avoid missing anti-prompt. + if (!prompt.EndsWith("\n") && !prompt.EndsWith("\r\n")) + { + prompt = converted_prompt.Trim(); + } + else + { + prompt = converted_prompt; + } + } + else + { + History.Messages.Add(new ChatHistory.Message(AuthorRole.User, prompt)); } StringBuilder sb = new(); diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 148f1735..36ae55bf 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using System.Text.Json; @@ -258,6 +259,7 @@ namespace LLama.Native enableLogging = configuration.Logging; // We move the flag to avoid loading library when the variable is called else where. NativeLibraryConfig.LibraryHasLoaded = true; + Log(configuration.ToString(), LogLevel.Information); if (!string.IsNullOrEmpty(configuration.Path)) { @@ -273,6 +275,7 @@ namespace LLama.Native var libraryTryLoadOrder = GetLibraryTryOrder(configuration); + string[] preferredPaths = configuration.SearchDirectories.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToArray(); string[] possiblePathPrefix = new string[] { System.AppDomain.CurrentDomain.BaseDirectory, Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "" @@ -280,19 +283,22 @@ namespace LLama.Native var tryFindPath = (string filename) => { - int i = 0; - while (!File.Exists(filename)) + foreach(var path in preferredPaths) { - if (i < possiblePathPrefix.Length) + if (File.Exists(Path.Combine(path, filename))) { - filename = Path.Combine(possiblePathPrefix[i], filename); - i++; + return Path.Combine(path, filename); } - else + } + + foreach(var path in possiblePathPrefix) + { + if (File.Exists(Path.Combine(path, filename))) { - break; + return Path.Combine(path, filename); } } + return filename; }; diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs index 76f89357..fcef9c34 100644 --- a/LLama/Native/NativeLibraryConfig.cs +++ b/LLama/Native/NativeLibraryConfig.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; namespace LLama.Native { @@ -27,6 +29,13 @@ namespace LLama.Native private bool _allowFallback = true; private bool _skipCheck = false; private bool _logging = false; + /// + /// search directory -> priority level, 0 is the lowest. + /// + private Dictionary _searchDirectories = new Dictionary() + { + { "./", 0 } + }; private static void ThrowIfLoaded() { @@ -120,13 +129,62 @@ namespace LLama.Native return this; } + /// + /// Add self-defined search directories. Note that the file stucture of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// The directories and corresponding priorities, in which 0 is the lowest. The default path has priority 0. + /// + public NativeLibraryConfig WithSearchDirectories(IDictionary directoriesAndPriorities) + { + ThrowIfLoaded(); + + foreach(var (directory, priority) in directoriesAndPriorities) + { + if(priority < 0) + { + throw new ArgumentException("Priority must be a positive number."); + } + _searchDirectories[directory] = priority; + } + return this; + } + + /// + /// Add self-defined search directories. Note that the file stucture of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// The priority of your added search path. 0 is the lowest. The default path has priority 0. + /// + public NativeLibraryConfig WithSearchDirectory(string directory, int priority) + { + ThrowIfLoaded(); + + if (priority < 0) + { + throw new ArgumentException("Priority must be a positive number."); + } + _searchDirectories[directory] = priority; + return this; + } + internal static Description CheckAndGatherDescription() { if (Instance._allowFallback && Instance._skipCheck) { throw new ArgumentException("Cannot skip the check when fallback is allowed."); } - return new Description(Instance._libraryPath, Instance._useCuda, Instance._avxLevel, Instance._allowFallback, Instance._skipCheck, Instance._logging); + return new Description( + Instance._libraryPath, + Instance._useCuda, + Instance._avxLevel, + Instance._allowFallback, + Instance._skipCheck, + Instance._logging, + Instance._searchDirectories); } internal static string AvxLevelToString(AvxLevel level) @@ -183,7 +241,31 @@ namespace LLama.Native Avx512, } - internal record Description(string Path, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging); + internal record Description(string Path, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging, Dictionary SearchDirectories) + { + public override string ToString() + { + string avxLevelString = AvxLevel switch + { + AvxLevel.None => "NoAVX", + AvxLevel.Avx => "AVX", + AvxLevel.Avx2 => "AVX2", + AvxLevel.Avx512 => "AVX512", + _ => "Unknown" + }; + + string searchDirectoriesString = string.Join(", ", SearchDirectories.Select(kv => $"[{kv.Key}: {kv.Value}]")); + + return $"NativeLibraryConfig Description:\n" + + $"- Path: {Path}\n" + + $"- PreferCuda: {UseCuda}\n" + + $"- PreferredAvxLevel: {avxLevelString}\n" + + $"- AllowFallback: {AllowFallback}\n" + + $"- SkipCheck: {SkipCheck}\n" + + $"- Logging: {Logging}\n" + + $"- SearchDirectories and Priorities: {searchDirectoriesString}"; + } + } } #endif - } +} From ffc347a3f302f0d0817bb56acd80a03d532f60ae Mon Sep 17 00:00:00 2001 From: Rinne Date: Wed, 29 Nov 2023 00:16:00 +0800 Subject: [PATCH 08/11] resolve comments. --- LLama/Native/NativeApi.Load.cs | 2 +- LLama/Native/NativeLibraryConfig.cs | 33 ++++++++--------------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 36ae55bf..d8a88725 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -275,7 +275,7 @@ namespace LLama.Native var libraryTryLoadOrder = GetLibraryTryOrder(configuration); - string[] preferredPaths = configuration.SearchDirectories.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToArray(); + string[] preferredPaths = configuration.SearchDirectories; string[] possiblePathPrefix = new string[] { System.AppDomain.CurrentDomain.BaseDirectory, Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "" diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs index fcef9c34..e5135970 100644 --- a/LLama/Native/NativeLibraryConfig.cs +++ b/LLama/Native/NativeLibraryConfig.cs @@ -32,10 +32,7 @@ namespace LLama.Native /// /// search directory -> priority level, 0 is the lowest. /// - private Dictionary _searchDirectories = new Dictionary() - { - { "./", 0 } - }; + private List _searchDirectories = new List(); private static void ThrowIfLoaded() { @@ -134,20 +131,13 @@ namespace LLama.Native /// directories must be the same as the default directory. Besides, the directory /// won't be used recursively. /// - /// The directories and corresponding priorities, in which 0 is the lowest. The default path has priority 0. + /// /// - public NativeLibraryConfig WithSearchDirectories(IDictionary directoriesAndPriorities) + public NativeLibraryConfig WithSearchDirectories(IEnumerable directories) { ThrowIfLoaded(); - foreach(var (directory, priority) in directoriesAndPriorities) - { - if(priority < 0) - { - throw new ArgumentException("Priority must be a positive number."); - } - _searchDirectories[directory] = priority; - } + _searchDirectories.AddRange(directories); return this; } @@ -157,17 +147,12 @@ namespace LLama.Native /// won't be used recursively. /// /// - /// The priority of your added search path. 0 is the lowest. The default path has priority 0. /// - public NativeLibraryConfig WithSearchDirectory(string directory, int priority) + public NativeLibraryConfig WithSearchDirectory(string directory) { ThrowIfLoaded(); - if (priority < 0) - { - throw new ArgumentException("Priority must be a positive number."); - } - _searchDirectories[directory] = priority; + _searchDirectories.Add(directory); return this; } @@ -184,7 +169,7 @@ namespace LLama.Native Instance._allowFallback, Instance._skipCheck, Instance._logging, - Instance._searchDirectories); + Instance._searchDirectories.Concat(new string[] { "./" }).ToArray()); } internal static string AvxLevelToString(AvxLevel level) @@ -241,7 +226,7 @@ namespace LLama.Native Avx512, } - internal record Description(string Path, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging, Dictionary SearchDirectories) + internal record Description(string Path, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging, string[] SearchDirectories) { public override string ToString() { @@ -254,7 +239,7 @@ namespace LLama.Native _ => "Unknown" }; - string searchDirectoriesString = string.Join(", ", SearchDirectories.Select(kv => $"[{kv.Key}: {kv.Value}]")); + string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }"; return $"NativeLibraryConfig Description:\n" + $"- Path: {Path}\n" + From 35624d688cf37848063f80b1171062fd057e8fc5 Mon Sep 17 00:00:00 2001 From: Rinne Date: Wed, 29 Nov 2023 01:06:55 +0800 Subject: [PATCH 09/11] ci: fix error in auto-release. --- .github/prepare_release.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/prepare_release.sh b/.github/prepare_release.sh index bfc42ed7..a4c96bbf 100755 --- a/.github/prepare_release.sh +++ b/.github/prepare_release.sh @@ -23,11 +23,7 @@ fi mkdir ./temp; mkdir ./temp/runtimes; # For sure it could be done better but cp -R did not work on osx -mkdir ./temp/runtimes/osx-arm64 -mkdir ./temp/runtimes/osx-x64 cp ./LLama/runtimes/*.* ./temp/runtimes/; -cp ./LLama/runtimes/osx-arm64/*.* ./temp/runtimes/osx-arm64/; -cp ./LLama/runtimes/osx-x64/*.* ./temp/runtimes/osx-x64; cp ./LLama/runtimes/build/*.* ./temp/; # get the current version From c067570803cc0ab65983cd6cc138ea999e353801 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Tue, 28 Nov 2023 18:13:46 +0000 Subject: [PATCH 10/11] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c96cd5a9..7218230d 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ The llama.cpp commit id will help if you want to compile a DLL yourself. | v0.5.1 | [Llama2 7b GGUF](https://huggingface.co/TheBloke/llama-2-7B-Guanaco-QLoRA-GGUF)| 6b73ef1 | | v0.6.0 | | [cb33f43](https://github.com/ggerganov/llama.cpp/commit/cb33f43a2a9f5a5a5f8d290dd97c625d9ba97a2f) | | v0.7.0, v0.8.0 | [Thespis-13B](https://huggingface.co/TheBloke/Thespis-13B-v0.5-GGUF/tree/main?not-for-all-audiences=true), [LLaMA2-7B](https://huggingface.co/TheBloke/llama-2-7B-Guanaco-QLoRA-GGUF) | [207b519](https://github.com/ggerganov/llama.cpp/commit/207b51900e15cc7f89763a3bb1c565fe11cbb45d) | +| v0.8.1 | | [e937066](https://github.com/ggerganov/llama.cpp/commit/e937066420b79a757bf80e9836eb12b88420a218) | ## License From 93eb885253a5500d310be8fc928abc129400f401 Mon Sep 17 00:00:00 2001 From: mark van tilburg Date: Fri, 1 Dec 2023 08:54:49 +0100 Subject: [PATCH 11/11] Update README.md Update example to the latest version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7218230d..af517804 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Console.Write(prompt); // run the inference in a loop to chat with LLM while (prompt != "stop") { - foreach (var text in session.Chat(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) + await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) { Console.Write(text); }