From 629430a087be3c69df2e98d2865272882edcd0fd Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Thu, 16 Nov 2023 14:09:14 -0600 Subject: [PATCH 01/11] Correctly format followup messages in turn-based (chat) inference --- LLama/ChatSession.cs | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/LLama/ChatSession.cs b/LLama/ChatSession.cs index 7ee99590..748d2ef3 100644 --- a/LLama/ChatSession.cs +++ b/LLama/ChatSession.cs @@ -159,15 +159,15 @@ namespace LLama InteractiveExecutorState state = (InteractiveExecutorState)executor.GetStateData(); prompt = state.IsPromptRun ? HistoryTransform.HistoryToText(History) - : prompt; + : HistoryTransform.HistoryToText(HistoryTransform.TextToHistory(AuthorRole.User, prompt)); } StringBuilder sb = new(); - await foreach (var result in ChatAsyncInternal(prompt, inferenceParams, cancellationToken)) + await foreach (var textToken in ChatAsyncInternal(prompt, inferenceParams, cancellationToken)) { - yield return result; - sb.Append(result); + yield return textToken; + sb.Append(textToken); } string assistantMessage = sb.ToString(); @@ -180,7 +180,7 @@ namespace LLama { foreach (var stopToken in inferenceParams.AntiPrompts) { - assistantMessage = assistantMessage.Replace(stopToken, ""); + assistantMessage = assistantMessage.Replace(stopToken, "").Trim(); } } @@ -209,27 +209,37 @@ namespace LLama { InteractiveExecutorState state = (InteractiveExecutorState)executor.GetStateData(); - prompt = state.IsPromptRun - ? HistoryTransform.HistoryToText(History) - : history.Messages.Last().Content; + if (state.IsPromptRun) + { + prompt = HistoryTransform.HistoryToText(History); + } + else + { + ChatHistory.Message lastMessage = history.Messages.Last(); + prompt = HistoryTransform.HistoryToText(HistoryTransform.TextToHistory(lastMessage.AuthorRole, lastMessage.Content)); + } } else { - prompt = history.Messages.Last().Content; + ChatHistory.Message lastMessage = history.Messages.Last(); + prompt = HistoryTransform.HistoryToText(HistoryTransform.TextToHistory(lastMessage.AuthorRole, lastMessage.Content)); } - await foreach (var result in ChatAsyncInternal(prompt, inferenceParams, cancellationToken)) + await foreach (var textToken in ChatAsyncInternal(prompt, inferenceParams, cancellationToken)) { - yield return result; + yield return textToken; } } private async IAsyncEnumerable ChatAsyncInternal(string prompt, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + Console.ForegroundColor = ConsoleColor.Gray; + Console.WriteLine(prompt); + var results = _executor.InferAsync(prompt, inferenceParams, cancellationToken); - await foreach (var item in OutputTransform.TransformAsync(results).WithCancellation(cancellationToken)) + await foreach (var textToken in OutputTransform.TransformAsync(results).WithCancellation(cancellationToken)) { - yield return item; + yield return textToken; } } } From 75932afc62993f2ef4132ec36975175749ba5ebd Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Thu, 16 Nov 2023 14:25:41 -0600 Subject: [PATCH 02/11] Remove debug output --- LLama/ChatSession.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/LLama/ChatSession.cs b/LLama/ChatSession.cs index 748d2ef3..d1504a08 100644 --- a/LLama/ChatSession.cs +++ b/LLama/ChatSession.cs @@ -233,9 +233,6 @@ namespace LLama private async IAsyncEnumerable ChatAsyncInternal(string prompt, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - Console.ForegroundColor = ConsoleColor.Gray; - Console.WriteLine(prompt); - var results = _executor.InferAsync(prompt, inferenceParams, cancellationToken); await foreach (var textToken in OutputTransform.TransformAsync(results).WithCancellation(cancellationToken)) { From f1eac82ecc4f49403ec06026aba3df7efdc8cb36 Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Sat, 25 Nov 2023 09:23:37 -0600 Subject: [PATCH 03/11] Update target frameworks with .NET 8 --- LLama.Examples/LLama.Examples.csproj | 2 +- LLama.KernelMemory/LLamaSharp.KernelMemory.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index b7369172..5053c038 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net6.0;net7.0;net8.0 enable enable AnyCPU;x64 diff --git a/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj b/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj index 78d4712b..3867b7e1 100644 --- a/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj +++ b/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj @@ -1,7 +1,7 @@ - net6.0;net7.0 + net6.0;net7.0;net8.0 enable enable From cb480f04afdba27d0712b18c336ac80ad1a698e9 Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Sat, 25 Nov 2023 09:24:03 -0600 Subject: [PATCH 04/11] Prevent compilation errors due to duplicated assembly info --- LLama/LLamaSharp.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 0e029c2d..5e7de5f4 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -28,6 +28,7 @@ AnyCPU;x64;Arm64 LLamaSharp Debug;Release;GPU + false From 67e6d633fd3564e0016b08fbdf8ed63bc038c36a Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Sat, 25 Nov 2023 09:28:39 -0600 Subject: [PATCH 05/11] Rebuild ChatSession class - Saves with serialized ChatHistory of session - Only allows use of ChatHistory.Message (instead of raw text) for easy post-processing with IHistoryTransform implementation - Provides History Management methods - Allows user to regenerate last assistant message --- LLama/ChatSession.cs | 645 +++++++++++++++++++++++++----------- LLama/Common/ChatHistory.cs | 40 ++- 2 files changed, 483 insertions(+), 202 deletions(-) diff --git a/LLama/ChatSession.cs b/LLama/ChatSession.cs index d1504a08..2985bd5f 100644 --- a/LLama/ChatSession.cs +++ b/LLama/ChatSession.cs @@ -1,243 +1,496 @@ -using LLama.Abstractions; -using LLama.Common; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.CompilerServices; -using System.Text; using System.Threading; using System.Threading.Tasks; +using LLama.Abstractions; +using LLama.Common; using static LLama.InteractiveExecutor; -namespace LLama +namespace LLama; + +/// +/// The main chat session class. +/// +public class ChatSession { + private const string _modelStateFilename = "ModelState.st"; + private const string _executorStateFilename = "ExecutorState.json"; + private const string _hsitoryFilename = "ChatHistory.json"; + /// - /// The main chat session class. - /// - public class ChatSession - { - private readonly ILLamaExecutor _executor; - private readonly ChatHistory _history; - - private const string _executorStateFilename = "ExecutorState.json"; - private const string _modelStateFilename = "ModelState.st"; - - /// - /// The executor for this session. - /// - public ILLamaExecutor Executor => _executor; - /// - /// The chat history for this session. - /// - public ChatHistory History => _history; - /// - /// The history transform used in this session. - /// - public IHistoryTransform HistoryTransform { get; set; } = new LLamaTransforms.DefaultHistoryTransform(); - /// - /// The input transform pipeline used in this session. - /// - public List InputTransformPipeline { get; set; } = new(); - /// - /// The output transform used in this session. - /// - public ITextStreamTransform OutputTransform = new LLamaTransforms.EmptyTextOutputStreamTransform(); - - /// - /// - /// - /// The executor for this session - public ChatSession(ILLamaExecutor executor) - { - _executor = executor; - _history = new ChatHistory(); - } - - /// - /// Use a custom history transform. - /// - /// - /// - public ChatSession WithHistoryTransform(IHistoryTransform transform) - { - HistoryTransform = transform; - return this; - } - - /// - /// Add a text transform to the input transform pipeline. - /// - /// - /// - public ChatSession AddInputTransform(ITextTransform transform) - { - InputTransformPipeline.Add(transform); - return this; - } - - /// - /// Use a custom output transform. - /// - /// - /// - public ChatSession WithOutputTransform(ITextStreamTransform transform) - { - OutputTransform = transform; - return this; - } - - /// - /// - /// - /// The directory name to save the session. If the directory does not exist, a new directory will be created. - public virtual void SaveSession(string path) - { - if (!Directory.Exists(path)) - { - Directory.CreateDirectory(path); - } - _executor.Context.SaveState(Path.Combine(path, _modelStateFilename)); - if (Executor is StatelessExecutor) - { + /// The executor for this session. + /// + public ILLamaExecutor Executor { get; private set; } - } - else if (Executor is StatefulExecutorBase statefulExecutor) - { - statefulExecutor.SaveState(Path.Combine(path, _executorStateFilename)); - } - else - { - throw new System.NotImplementedException("You're using a customized executor. Please inherit ChatSession and rewrite the method."); - } + /// + /// The chat history for this session. + /// + public ChatHistory History { get; private set; } = new(); + + /// + /// The history transform used in this session. + /// + public IHistoryTransform HistoryTransform { get; set; } = new LLamaTransforms.DefaultHistoryTransform(); + + /// + /// The input transform pipeline used in this session. + /// + public List InputTransformPipeline { get; set; } = new(); + + /// + /// The output transform used in this session. + /// + public ITextStreamTransform OutputTransform = new LLamaTransforms.EmptyTextOutputStreamTransform(); + + /// + /// Create a new chat session. + /// + /// The executor for this session + public ChatSession(ILLamaExecutor executor) + { + // Check if executor has StatefulExecutorBase as base class + if (executor is not StatefulExecutorBase) + { + throw new ArgumentException("Executor must have a StatefulExecutorBase", nameof(executor)); } - /// - /// - /// - /// The directory name to load the session. - public virtual void LoadSession(string path) + Executor = executor; + } + + /// + /// Create a new chat session with a custom history. + /// + /// + /// + public ChatSession(ILLamaExecutor executor, ChatHistory history) + : this(executor) + { + History = history; + } + + /// + /// Use a custom history transform. + /// + /// + /// + public ChatSession WithHistoryTransform(IHistoryTransform transform) + { + HistoryTransform = transform; + return this; + } + + /// + /// Add a text transform to the input transform pipeline. + /// + /// + /// + public ChatSession AddInputTransform(ITextTransform transform) + { + InputTransformPipeline.Add(transform); + return this; + } + + /// + /// Use a custom output transform. + /// + /// + /// + public ChatSession WithOutputTransform(ITextStreamTransform transform) + { + OutputTransform = transform; + return this; + } + + /// + /// Save a session from a directory. + /// + /// + /// + /// + public void SaveSession(string path) + { + if (string.IsNullOrWhiteSpace(path)) { - if (!Directory.Exists(path)) - { - throw new FileNotFoundException($"Directory {path} does not exist."); - } - _executor.Context.LoadState(Path.Combine(path, _modelStateFilename)); - if (Executor is StatelessExecutor) - { + throw new ArgumentException("Path cannot be null or whitespace", nameof(path)); + } - } - else if (Executor is StatefulExecutorBase statefulExecutor) + if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: true); + } + + Directory.CreateDirectory(path); + + string modelStateFilePath = Path.Combine(path, _modelStateFilename); + Executor.Context.SaveState(modelStateFilePath); + + string executorStateFilepath = Path.Combine(path, _executorStateFilename); + ((StatefulExecutorBase)Executor).SaveState(executorStateFilepath); + + string historyFilepath = Path.Combine(path, _hsitoryFilename); + File.WriteAllText(historyFilepath, History.ToJson()); + } + + /// + /// Load a session from a directory. + /// + /// + /// + /// + public void LoadSession(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("Path cannot be null or whitespace", nameof(path)); + } + + if (!Directory.Exists(path)) + { + throw new ArgumentException("Directory does not exist", nameof(path)); + } + + string modelStateFilePath = Path.Combine(path, _modelStateFilename); + Executor.Context.LoadState(modelStateFilePath); + + string executorStateFilepath = Path.Combine(path, _executorStateFilename); + ((StatefulExecutorBase)Executor).LoadState(executorStateFilepath); + + string historyFilepath = Path.Combine(path, _hsitoryFilename); + string historyJson = File.ReadAllText(historyFilepath); + History = ChatHistory.FromJson(historyJson) + ?? throw new ArgumentException("History file is invalid", nameof(path)); + } + + /// + /// Add a message to the chat history. + /// + /// + /// + public ChatSession AddMessage(ChatHistory.Message message) + { + // If current message is a system message, only allow the history to be empty + if (message.AuthorRole == AuthorRole.System && History.Messages.Count > 0) + { + throw new ArgumentException("Cannot add a system message after another message", nameof(message)); + } + + // If current message is a user message, only allow the history to be empty, + // or the previous message to be a system message or assistant message. + if (message.AuthorRole == AuthorRole.User) + { + ChatHistory.Message? lastMessage = History.Messages.LastOrDefault(); + if (lastMessage is not null && lastMessage.AuthorRole == AuthorRole.User) { - statefulExecutor.LoadState(Path.Combine(path, _executorStateFilename)); + throw new ArgumentException("Cannot add a user message after another user message", nameof(message)); } - else + } + + // If the current message is an assistant message, + // the previous message must be a user message. + if (message.AuthorRole == AuthorRole.Assistant) + { + ChatHistory.Message? lastMessage = History.Messages.LastOrDefault(); + if (lastMessage is null + || lastMessage.AuthorRole != AuthorRole.User) { - throw new System.NotImplementedException("You're using a customized executor. Please inherit ChatSession and rewrite the method."); + throw new ArgumentException("Assistant message must be preceeded with a user message", nameof(message)); } } - /// - /// Generates a response for a given user prompt and manages history state for the user. - /// This will always pass the whole history to the model. Don't pass a whole history - /// to this method as the user prompt will be appended to the history of the current session. - /// If more control is needed, use the other overload of this method that accepts a ChatHistory object. - /// - /// - /// - /// - /// Returns generated text of the assistant message. - public async IAsyncEnumerable ChatAsync(string prompt, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + History.AddMessage(message.AuthorRole, message.Content); + return this; + } + + /// + /// Add a system message to the chat history. + /// + /// + /// + public ChatSession AddSystemMessage(string content) + => AddMessage(new ChatHistory.Message(AuthorRole.System, content)); + + /// + /// Add an assistant message to the chat history. + /// + /// + /// + public ChatSession AddAssistantMessage(string content) + => AddMessage(new ChatHistory.Message(AuthorRole.Assistant, content)); + + /// + /// Add a user message to the chat history. + /// + /// + /// + public ChatSession AddUserMessage(string content) + => AddMessage(new ChatHistory.Message(AuthorRole.User, content)); + + /// + /// Remove the last message from the chat history. + /// + /// + public ChatSession RemoveLastMessage() + { + History.Messages.RemoveAt(History.Messages.Count - 1); + return this; + } + + /// + /// Replace a user message with a new message and remove all messages after the new message. + /// This is useful when the user wants to edit a message. And regenerate the response. + /// + /// + /// + /// + public ChatSession ReplaceUserMessage( + ChatHistory.Message oldMessage, + ChatHistory.Message newMessage) + { + if (oldMessage.AuthorRole != AuthorRole.User) + { + throw new ArgumentException("Old message must be a user message", nameof(oldMessage)); + } + + if (newMessage.AuthorRole != AuthorRole.User) { - foreach (var inputTransform in InputTransformPipeline) - prompt = inputTransform.Transform(prompt); + throw new ArgumentException("New message must be a user message", nameof(newMessage)); + } - History.Messages.Add(new ChatHistory.Message(AuthorRole.User, prompt)); + int index = History.Messages.IndexOf(oldMessage); + if (index == -1) + { + throw new ArgumentException("Old message does not exist in history", nameof(oldMessage)); + } - if (_executor is InteractiveExecutor executor) - { - InteractiveExecutorState state = (InteractiveExecutorState)executor.GetStateData(); - prompt = state.IsPromptRun - ? HistoryTransform.HistoryToText(History) - : HistoryTransform.HistoryToText(HistoryTransform.TextToHistory(AuthorRole.User, prompt)); - } + History.Messages[index] = newMessage; + + // Remove all message after the new message + History.Messages.RemoveRange(index + 1, History.Messages.Count - index - 1); + + return this; + } - StringBuilder sb = new(); + /// + /// Chat with the model. + /// + /// + /// + /// + /// + /// + /// + public async IAsyncEnumerable ChatAsync( + ChatHistory.Message message, + bool applyInputTransformPipeline, + IInferenceParams? inferenceParams = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // The message must be a user message + if (message.AuthorRole != AuthorRole.User) + { + throw new ArgumentException("Message must be a user message", nameof(message)); + } - await foreach (var textToken in ChatAsyncInternal(prompt, inferenceParams, cancellationToken)) + // Apply input transform pipeline + if (applyInputTransformPipeline) + { + foreach (var inputTransform in InputTransformPipeline) { - yield return textToken; - sb.Append(textToken); + message.Content = inputTransform.Transform(message.Content); } + } - string assistantMessage = sb.ToString(); + // Add the user's message to the history + AddUserMessage(message.Content); + + // Prepare prompt variable + string prompt; + + // Check if the session history was restored from a previous session + // or added as part of new chat session history. + InteractiveExecutorState state = (InteractiveExecutorState)((StatefulExecutorBase)Executor).GetStateData(); + + // If "IsPromptRun" is true, the session was newly started. + if (state.IsPromptRun) + { + // If the session history was added as part of new chat session history, + // convert the complete history includsing system message and manually added history + // to a prompt that adhere to the prompt template specified in the HistoryTransform class implementation. + prompt = HistoryTransform.HistoryToText(History); + } + else + { + // If the session was restored from a previous session, + // convert only the current message to the prompt with the prompt template + // specified in the HistoryTransform class implementation that is provided. + ChatHistory singleMessageHistory = HistoryTransform.TextToHistory(message.AuthorRole, message.Content); + prompt = HistoryTransform.HistoryToText(singleMessageHistory); + } + + string assistantMessage = string.Empty; + + await foreach ( + string textToken + in ChatAsyncInternal( + prompt, + inferenceParams, + cancellationToken)) + { + assistantMessage += textToken; + yield return textToken; + } - // Remove end tokens from the assistant message - // if defined in inferenceParams.AntiPrompts. - // We only want the response that was generated and not tokens - // that are delimiting the beginning or end of the response. - if (inferenceParams?.AntiPrompts != null) + // Add the assistant message to the history + AddAssistantMessage(assistantMessage); + } + + /// + /// Chat with the model. + /// + /// + /// + /// + /// + public IAsyncEnumerable ChatAsync( + ChatHistory.Message message, + IInferenceParams? inferenceParams = null, + CancellationToken cancellationToken = default) + { + return ChatAsync( + message, + applyInputTransformPipeline: true, + inferenceParams, + cancellationToken); + } + + /// + /// Chat with the model. + /// + /// + /// + /// + /// + /// + /// + public IAsyncEnumerable ChatAsync( + ChatHistory history, + bool applyInputTransformPipeline, + IInferenceParams? inferenceParams = null, + CancellationToken cancellationToken = default) + { + ChatHistory.Message lastMessage = history.Messages.LastOrDefault() + ?? throw new ArgumentException("History must contain at least one message", nameof(history)); + + foreach ( + ChatHistory.Message message + in history.Messages.Take(history.Messages.Count - 1)) + { + // Apply input transform pipeline + if (applyInputTransformPipeline + && message.AuthorRole == AuthorRole.User) { - foreach (var stopToken in inferenceParams.AntiPrompts) + foreach ( + var inputTransform + in InputTransformPipeline) { - assistantMessage = assistantMessage.Replace(stopToken, "").Trim(); + message.Content = inputTransform.Transform(message.Content); } } - History.Messages.Add(new ChatHistory.Message(AuthorRole.Assistant, assistantMessage)); + AddMessage(message); } - /// - /// Generates a response for a given chat history. This method does not manage history state for the user. - /// If you want to e.g. truncate the history of a session to fit into the model's context window, - /// use this method and pass the truncated history to it. If you don't need this control, use the other - /// overload of this method that accepts a user prompt instead. - /// - /// - /// - /// - /// Returns generated text of the assistant message. - public async IAsyncEnumerable ChatAsync(ChatHistory history, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + return ChatAsync( + lastMessage, + applyInputTransformPipeline, + inferenceParams, + cancellationToken); + } + + /// + /// Chat with the model. + /// + /// + /// + /// + /// + public IAsyncEnumerable ChatAsync( + ChatHistory history, + IInferenceParams? inferenceParams = null, + CancellationToken cancellationToken = default) + { + return ChatAsync( + history, + applyInputTransformPipeline: true, + inferenceParams, + cancellationToken); + } + + /// + /// Regenerate the last assistant message. + /// + /// + /// + /// + /// + public async IAsyncEnumerable RegenerateAssistantMessageAsync( + InferenceParams? inferenceParams = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // Make sure the last message is an assistant message (reponse from the LLM). + ChatHistory.Message? lastAssistantMessage = History.Messages.LastOrDefault(); + + if (lastAssistantMessage is null + || lastAssistantMessage.AuthorRole != AuthorRole.Assistant) { - if (history.Messages.Count == 0) - { - throw new ArgumentException("History must contain at least one message."); - } + throw new InvalidOperationException("Last message must be an assistant message"); + } - string prompt; - if (_executor is InteractiveExecutor executor) - { - InteractiveExecutorState state = (InteractiveExecutorState)executor.GetStateData(); + // Remove the last assistant message from the history. + RemoveLastMessage(); - if (state.IsPromptRun) - { - prompt = HistoryTransform.HistoryToText(History); - } - else - { - ChatHistory.Message lastMessage = history.Messages.Last(); - prompt = HistoryTransform.HistoryToText(HistoryTransform.TextToHistory(lastMessage.AuthorRole, lastMessage.Content)); - } - } - else - { - ChatHistory.Message lastMessage = history.Messages.Last(); - prompt = HistoryTransform.HistoryToText(HistoryTransform.TextToHistory(lastMessage.AuthorRole, lastMessage.Content)); - } + // Get the last user message. + ChatHistory.Message? lastUserMessage = History.Messages.LastOrDefault(); - await foreach (var textToken in ChatAsyncInternal(prompt, inferenceParams, cancellationToken)) - { - yield return textToken; - } + if (lastUserMessage is null + || lastUserMessage.AuthorRole != AuthorRole.User) + { + throw new InvalidOperationException("Last message must be a user message"); } - private async IAsyncEnumerable ChatAsyncInternal(string prompt, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + // Remove the last user message from the history. + RemoveLastMessage(); + + // Regenerate the assistant message. + await foreach ( + string textToken + in ChatAsync( + lastUserMessage, + applyInputTransformPipeline: false, + inferenceParams, + cancellationToken)) { - var results = _executor.InferAsync(prompt, inferenceParams, cancellationToken); - await foreach (var textToken in OutputTransform.TransformAsync(results).WithCancellation(cancellationToken)) - { - yield return textToken; - } + yield return textToken; + } + } + + private async IAsyncEnumerable ChatAsyncInternal( + string prompt, + IInferenceParams? inferenceParams = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var results = Executor.InferAsync(prompt, inferenceParams, cancellationToken); + + await foreach ( + string textToken + in OutputTransform + .TransformAsync(results) + .WithCancellation(cancellationToken)) + { + yield return textToken; } } -} \ No newline at end of file +} diff --git a/LLama/Common/ChatHistory.cs b/LLama/Common/ChatHistory.cs index 7224b314..3f038874 100644 --- a/LLama/Common/ChatHistory.cs +++ b/LLama/Common/ChatHistory.cs @@ -1,4 +1,7 @@ using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using System.Text.Json.Serialization; namespace LLama.Common { @@ -43,11 +46,14 @@ namespace LLama.Common /// /// Role of the message author, e.g. user/assistant/system /// + [JsonConverter(typeof(JsonStringEnumConverter))] + [JsonPropertyName("author_role")] public AuthorRole AuthorRole { get; set; } /// /// Message content /// + [JsonPropertyName("content")] public string Content { get; set; } /// @@ -65,15 +71,14 @@ namespace LLama.Common /// /// List of messages in the chat /// - public List Messages { get; } + [JsonPropertyName("messages")] + public List Messages { get; set; } = new(); /// /// Create a new instance of the chat content class /// - public ChatHistory() - { - this.Messages = new List(); - } + [JsonConstructor] + public ChatHistory() { } /// /// Add a message to the chat history @@ -84,6 +89,29 @@ namespace LLama.Common { this.Messages.Add(new Message(authorRole, content)); } - } + /// + /// Serialize the chat history to JSON + /// + /// + public string ToJson() + { + return JsonSerializer.Serialize( + this, + new JsonSerializerOptions() + { + WriteIndented = true + }); + } + + /// + /// Deserialize a chat history from JSON + /// + /// + /// + public static ChatHistory? FromJson(string json) + { + return JsonSerializer.Deserialize(json); + } + } } From 73d17259542b082ce6c49f6adb3a1d892d8449ad Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Sat, 25 Nov 2023 09:29:00 -0600 Subject: [PATCH 06/11] Modified / updated ChatSession examples --- LLama.Examples/Assets/chat-with-bob.json | 24 +++++ .../Examples/ChatSessionStripRoleName.cs | 76 ++++++------- .../Examples/ChatSessionWithHistory.cs | 100 ++++++++++++++++++ .../Examples/ChatSessionWithRoleName.cs | 76 ++++++------- LLama.Examples/Examples/LoadAndSaveSession.cs | 13 ++- LLama.Examples/Examples/Runner.cs | 5 +- LLama.Examples/LLama.Examples.csproj | 3 + 7 files changed, 217 insertions(+), 80 deletions(-) create mode 100644 LLama.Examples/Assets/chat-with-bob.json create mode 100644 LLama.Examples/Examples/ChatSessionWithHistory.cs diff --git a/LLama.Examples/Assets/chat-with-bob.json b/LLama.Examples/Assets/chat-with-bob.json new file mode 100644 index 00000000..52dc3910 --- /dev/null +++ b/LLama.Examples/Assets/chat-with-bob.json @@ -0,0 +1,24 @@ +{ + "messages": [ + { + "author_role": "System", + "content": "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision." + }, + { + "author_role": "User", + "content": "Hello, Bob." + }, + { + "author_role": "Assistant", + "content": "Hello. How may I help you today?" + }, + { + "author_role": "User", + "content": "Please tell me the largest city in Europe." + }, + { + "author_role": "Assistant", + "content": "Sure. The largest city in Europe is Istanbul, Turkey." + } + ] +} diff --git a/LLama.Examples/Examples/ChatSessionStripRoleName.cs b/LLama.Examples/Examples/ChatSessionStripRoleName.cs index 41362c4a..b39ac3ef 100644 --- a/LLama.Examples/Examples/ChatSessionStripRoleName.cs +++ b/LLama.Examples/Examples/ChatSessionStripRoleName.cs @@ -1,44 +1,44 @@ -using LLama.Common; +// using LLama.Common; -namespace LLama.Examples.Examples -{ - public class ChatSessionStripRoleName - { - public static async Task Run() - { - Console.Write("Please input your model path: "); - var modelPath = Console.ReadLine(); - var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); +// namespace LLama.Examples.Examples +// { +// public class ChatSessionStripRoleName +// { +// public static async Task Run() +// { +// Console.Write("Please input your model path: "); +// var modelPath = Console.ReadLine(); +// var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - var parameters = new ModelParams(modelPath) - { - ContextSize = 1024, - Seed = 1337, - GpuLayerCount = 5 - }; - using var model = LLamaWeights.LoadFromFile(parameters); - using var context = model.CreateContext(parameters); - var executor = new InteractiveExecutor(context); +// var parameters = new ModelParams(modelPath) +// { +// ContextSize = 1024, +// Seed = 1337, +// GpuLayerCount = 5 +// }; +// using var model = LLamaWeights.LoadFromFile(parameters); +// using var context = model.CreateContext(parameters); +// var executor = new InteractiveExecutor(context); - var session = new ChatSession(executor).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); +// var session = new ChatSession(executor).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine("The chat session has started. The role names won't be printed."); - Console.ForegroundColor = ConsoleColor.White; +// Console.ForegroundColor = ConsoleColor.Yellow; +// Console.WriteLine("The chat session has started. The role names won't be printed."); +// Console.ForegroundColor = ConsoleColor.White; - // show the prompt - Console.Write(prompt); - while (true) - { - await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) - { - Console.Write(text); - } +// // show the prompt +// Console.Write(prompt); +// while (true) +// { +// await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) +// { +// Console.Write(text); +// } - Console.ForegroundColor = ConsoleColor.Green; - prompt = Console.ReadLine(); - Console.ForegroundColor = ConsoleColor.White; - } - } - } -} +// Console.ForegroundColor = ConsoleColor.Green; +// prompt = Console.ReadLine(); +// Console.ForegroundColor = ConsoleColor.White; +// } +// } +// } +// } diff --git a/LLama.Examples/Examples/ChatSessionWithHistory.cs b/LLama.Examples/Examples/ChatSessionWithHistory.cs new file mode 100644 index 00000000..27f4912b --- /dev/null +++ b/LLama.Examples/Examples/ChatSessionWithHistory.cs @@ -0,0 +1,100 @@ +using DocumentFormat.OpenXml.Bibliography; +using LLama.Common; + +namespace LLama.Examples.Examples; + +public class ChatSessionWithHistory +{ + public static async Task Run() + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + + var parameters = new ModelParams(modelPath) + { + ContextSize = 1024, + Seed = 1337, + GpuLayerCount = 5 + }; + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters); + var executor = new InteractiveExecutor(context); + + ChatSession session; + if (Directory.Exists("Assets/chat-with-bob")) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Loading session from disk."); + Console.ForegroundColor = ConsoleColor.White; + + session = new ChatSession(executor); + session.LoadSession("Assets/chat-with-bob"); + } + else + { + var chatHistoryJson = File.ReadAllText("Assets/chat-with-bob.json"); + ChatHistory chatHistory = ChatHistory.FromJson(chatHistoryJson) ?? new ChatHistory(); + + session = new ChatSession(executor, chatHistory); + } + + session.WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform( + new string[] { "User:", "Assistant:" }, + redundancyLength: 8)); + + InferenceParams inferenceParams = new InferenceParams() + { + Temperature = 0.9f, + AntiPrompts = new List { "User:" } + }; + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started."); + + // show the prompt + Console.ForegroundColor = ConsoleColor.Green; + string userInput = Console.ReadLine() ?? ""; + + while (userInput != "exit") + { + if (userInput == "save") + { + session.SaveSession("Assets/chat-with-bob"); + // await session.LoadSessionAsync("Assets/chat-with-bob"); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Session saved."); + } + else if (userInput == "regenerate") + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Regenerating last response ..."); + + await foreach ( + var text + in session.RegenerateAssistantMessageAsync( + inferenceParams)) + { + Console.ForegroundColor = ConsoleColor.White; + Console.Write(text); + } + } + else + { + await foreach ( + var text + in session.ChatAsync( + new ChatHistory.Message(AuthorRole.User, userInput), + inferenceParams)) + { + Console.ForegroundColor = ConsoleColor.White; + Console.Write(text); + } + } + + Console.ForegroundColor = ConsoleColor.Green; + userInput = Console.ReadLine() ?? ""; + + Console.ForegroundColor = ConsoleColor.White; + } + } +} diff --git a/LLama.Examples/Examples/ChatSessionWithRoleName.cs b/LLama.Examples/Examples/ChatSessionWithRoleName.cs index c9ea9023..e5c180b7 100644 --- a/LLama.Examples/Examples/ChatSessionWithRoleName.cs +++ b/LLama.Examples/Examples/ChatSessionWithRoleName.cs @@ -1,44 +1,44 @@ -using LLama.Common; +// using LLama.Common; -namespace LLama.Examples.Examples -{ - public class ChatSessionWithRoleName - { - public static async Task Run() - { - Console.Write("Please input your model path: "); - var modelPath = Console.ReadLine(); - var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); +// namespace LLama.Examples.Examples +// { +// public class ChatSessionWithRoleName +// { +// public static async Task Run() +// { +// Console.Write("Please input your model path: "); +// var modelPath = Console.ReadLine(); +// var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - var parameters = new ModelParams(modelPath) - { - ContextSize = 1024, - Seed = 1337, - GpuLayerCount = 5 - }; - using var model = LLamaWeights.LoadFromFile(parameters); - using var context = model.CreateContext(parameters); - var executor = new InteractiveExecutor(context); +// var parameters = new ModelParams(modelPath) +// { +// ContextSize = 1024, +// Seed = 1337, +// GpuLayerCount = 5 +// }; +// using var model = LLamaWeights.LoadFromFile(parameters); +// using var context = model.CreateContext(parameters); +// var executor = new InteractiveExecutor(context); - var session = new ChatSession(executor); +// var session = new ChatSession(executor); - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result."); - Console.ForegroundColor = ConsoleColor.White; +// Console.ForegroundColor = ConsoleColor.Yellow; +// Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result."); +// Console.ForegroundColor = ConsoleColor.White; - // show the prompt - Console.Write(prompt); - while (true) - { - await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) - { - Console.Write(text); - } +// // show the prompt +// Console.Write(prompt); +// while (true) +// { +// await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) +// { +// Console.Write(text); +// } - Console.ForegroundColor = ConsoleColor.Green; - prompt = Console.ReadLine(); - Console.ForegroundColor = ConsoleColor.White; - } - } - } -} +// Console.ForegroundColor = ConsoleColor.Green; +// prompt = Console.ReadLine(); +// Console.ForegroundColor = ConsoleColor.White; +// } +// } +// } +// } diff --git a/LLama.Examples/Examples/LoadAndSaveSession.cs b/LLama.Examples/Examples/LoadAndSaveSession.cs index 91068091..678d3eb9 100644 --- a/LLama.Examples/Examples/LoadAndSaveSession.cs +++ b/LLama.Examples/Examples/LoadAndSaveSession.cs @@ -1,4 +1,5 @@ -using LLama.Common; +using DocumentFormat.OpenXml.Bibliography; +using LLama.Common; namespace LLama.Examples.Examples { @@ -30,7 +31,15 @@ namespace LLama.Examples.Examples Console.Write(prompt); while (true) { - await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) + await foreach ( + var text + in session.ChatAsync( + new ChatHistory.Message(AuthorRole.User, prompt), + new InferenceParams() + { + Temperature = 0.6f, + AntiPrompts = new List { "User:" } + })) { Console.Write(text); } diff --git a/LLama.Examples/Examples/Runner.cs b/LLama.Examples/Examples/Runner.cs index aca0a7da..43c12f87 100644 --- a/LLama.Examples/Examples/Runner.cs +++ b/LLama.Examples/Examples/Runner.cs @@ -6,8 +6,9 @@ public class Runner { private static readonly Dictionary> Examples = new() { - { "Run a chat session without stripping the role names.", ChatSessionWithRoleName.Run }, - { "Run a chat session with the role names stripped.", ChatSessionStripRoleName.Run }, + { "Run a chat session with history.", ChatSessionWithHistory.Run }, + // { "Run a chat session without stripping the role names.", ChatSessionWithRoleName.Run }, + // { "Run a chat session with the role names stripped.", ChatSessionStripRoleName.Run }, { "Interactive mode chat by using executor.", InteractiveModeExecute.Run }, { "Instruct mode chat by using executor.", InstructModeExecute.Run }, { "Stateless mode chat by using executor.", StatelessModeExecute.Run }, diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index 5053c038..c2491218 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -41,6 +41,9 @@ + + PreserveNewest + PreserveNewest From 422605d98063ef5da2cef8f33819b2d35c5c43df Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Mon, 27 Nov 2023 08:26:52 -0600 Subject: [PATCH 07/11] Re-add ChatSession examples --- .../Examples/ChatSessionStripRoleName.cs | 105 ++++++++++-------- .../Examples/ChatSessionWithHistory.cs | 2 - .../Examples/ChatSessionWithRoleName.cs | 102 +++++++++-------- LLama.Examples/Examples/Runner.cs | 4 +- 4 files changed, 121 insertions(+), 92 deletions(-) diff --git a/LLama.Examples/Examples/ChatSessionStripRoleName.cs b/LLama.Examples/Examples/ChatSessionStripRoleName.cs index b39ac3ef..1246db59 100644 --- a/LLama.Examples/Examples/ChatSessionStripRoleName.cs +++ b/LLama.Examples/Examples/ChatSessionStripRoleName.cs @@ -1,44 +1,61 @@ -// using LLama.Common; - -// namespace LLama.Examples.Examples -// { -// public class ChatSessionStripRoleName -// { -// public static async Task Run() -// { -// Console.Write("Please input your model path: "); -// var modelPath = Console.ReadLine(); -// var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - -// var parameters = new ModelParams(modelPath) -// { -// ContextSize = 1024, -// Seed = 1337, -// GpuLayerCount = 5 -// }; -// using var model = LLamaWeights.LoadFromFile(parameters); -// using var context = model.CreateContext(parameters); -// var executor = new InteractiveExecutor(context); - -// var session = new ChatSession(executor).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); - -// Console.ForegroundColor = ConsoleColor.Yellow; -// Console.WriteLine("The chat session has started. The role names won't be printed."); -// Console.ForegroundColor = ConsoleColor.White; - -// // show the prompt -// Console.Write(prompt); -// while (true) -// { -// await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) -// { -// Console.Write(text); -// } - -// Console.ForegroundColor = ConsoleColor.Green; -// prompt = Console.ReadLine(); -// Console.ForegroundColor = ConsoleColor.White; -// } -// } -// } -// } +using LLama.Common; + +namespace LLama.Examples.Examples; + +public class ChatSessionStripRoleName +{ + public static async Task Run() + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + + var parameters = new ModelParams(modelPath) + { + ContextSize = 1024, + Seed = 1337, + GpuLayerCount = 5 + }; + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters); + var executor = new InteractiveExecutor(context); + + var chatHistoryJson = File.ReadAllText("Assets/chat-with-bob.json"); + ChatHistory chatHistory = ChatHistory.FromJson(chatHistoryJson) ?? new ChatHistory(); + + ChatSession session = new(executor, chatHistory); + session.WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform( + new string[] { "User:", "Assistant:" }, + redundancyLength: 8)); + + InferenceParams inferenceParams = new InferenceParams() + { + Temperature = 0.9f, + AntiPrompts = new List { "User:" } + }; + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started."); + + // show the prompt + Console.ForegroundColor = ConsoleColor.Green; + string userInput = Console.ReadLine() ?? ""; + + while (userInput != "exit") + { + await foreach ( + var text + in session.ChatAsync( + new ChatHistory.Message(AuthorRole.User, userInput), + inferenceParams)) + { + Console.ForegroundColor = ConsoleColor.White; + Console.Write(text); + } + + Console.ForegroundColor = ConsoleColor.Green; + userInput = Console.ReadLine() ?? ""; + + Console.ForegroundColor = ConsoleColor.White; + } + } +} diff --git a/LLama.Examples/Examples/ChatSessionWithHistory.cs b/LLama.Examples/Examples/ChatSessionWithHistory.cs index 27f4912b..98ba7d75 100644 --- a/LLama.Examples/Examples/ChatSessionWithHistory.cs +++ b/LLama.Examples/Examples/ChatSessionWithHistory.cs @@ -1,4 +1,3 @@ -using DocumentFormat.OpenXml.Bibliography; using LLama.Common; namespace LLama.Examples.Examples; @@ -60,7 +59,6 @@ public class ChatSessionWithHistory if (userInput == "save") { session.SaveSession("Assets/chat-with-bob"); - // await session.LoadSessionAsync("Assets/chat-with-bob"); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Session saved."); } diff --git a/LLama.Examples/Examples/ChatSessionWithRoleName.cs b/LLama.Examples/Examples/ChatSessionWithRoleName.cs index e5c180b7..d6b0d98e 100644 --- a/LLama.Examples/Examples/ChatSessionWithRoleName.cs +++ b/LLama.Examples/Examples/ChatSessionWithRoleName.cs @@ -1,44 +1,58 @@ -// using LLama.Common; - -// namespace LLama.Examples.Examples -// { -// public class ChatSessionWithRoleName -// { -// public static async Task Run() -// { -// Console.Write("Please input your model path: "); -// var modelPath = Console.ReadLine(); -// var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - -// var parameters = new ModelParams(modelPath) -// { -// ContextSize = 1024, -// Seed = 1337, -// GpuLayerCount = 5 -// }; -// using var model = LLamaWeights.LoadFromFile(parameters); -// using var context = model.CreateContext(parameters); -// var executor = new InteractiveExecutor(context); - -// var session = new ChatSession(executor); - -// Console.ForegroundColor = ConsoleColor.Yellow; -// Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result."); -// Console.ForegroundColor = ConsoleColor.White; - -// // show the prompt -// Console.Write(prompt); -// while (true) -// { -// await foreach (var text in session.ChatAsync(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) -// { -// Console.Write(text); -// } - -// Console.ForegroundColor = ConsoleColor.Green; -// prompt = Console.ReadLine(); -// Console.ForegroundColor = ConsoleColor.White; -// } -// } -// } -// } +using LLama.Common; + +namespace LLama.Examples.Examples; + +public class ChatSessionWithRoleName +{ + public static async Task Run() + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + + var parameters = new ModelParams(modelPath) + { + ContextSize = 1024, + Seed = 1337, + GpuLayerCount = 5 + }; + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters); + var executor = new InteractiveExecutor(context); + + var chatHistoryJson = File.ReadAllText("Assets/chat-with-bob.json"); + ChatHistory chatHistory = ChatHistory.FromJson(chatHistoryJson) ?? new ChatHistory(); + + ChatSession session = new(executor, chatHistory); + + InferenceParams inferenceParams = new InferenceParams() + { + Temperature = 0.9f, + AntiPrompts = new List { "User:" } + }; + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started."); + + // show the prompt + Console.ForegroundColor = ConsoleColor.Green; + string userInput = Console.ReadLine() ?? ""; + + while (userInput != "exit") + { + await foreach ( + var text + in session.ChatAsync( + new ChatHistory.Message(AuthorRole.User, userInput), + inferenceParams)) + { + Console.ForegroundColor = ConsoleColor.White; + Console.Write(text); + } + + Console.ForegroundColor = ConsoleColor.Green; + userInput = Console.ReadLine() ?? ""; + + Console.ForegroundColor = ConsoleColor.White; + } + } +} diff --git a/LLama.Examples/Examples/Runner.cs b/LLama.Examples/Examples/Runner.cs index 43c12f87..d7653657 100644 --- a/LLama.Examples/Examples/Runner.cs +++ b/LLama.Examples/Examples/Runner.cs @@ -7,8 +7,8 @@ public class Runner private static readonly Dictionary> Examples = new() { { "Run a chat session with history.", ChatSessionWithHistory.Run }, - // { "Run a chat session without stripping the role names.", ChatSessionWithRoleName.Run }, - // { "Run a chat session with the role names stripped.", ChatSessionStripRoleName.Run }, + { "Run a chat session without stripping the role names.", ChatSessionWithRoleName.Run }, + { "Run a chat session with the role names stripped.", ChatSessionStripRoleName.Run }, { "Interactive mode chat by using executor.", InteractiveModeExecute.Run }, { "Instruct mode chat by using executor.", InstructModeExecute.Run }, { "Stateless mode chat by using executor.", StatelessModeExecute.Run }, From f669a4f5a70bbad5c5341199168244f01ee5cdeb Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Sun, 10 Dec 2023 09:34:11 -0600 Subject: [PATCH 08/11] Update the Chinese chat sample to use new ChatSession integration --- .../Assets/chat-with-kunkun-chinese.json | 24 +++ LLama.Examples/Examples/ChatChineseGB2312.cs | 153 ++++++++++++------ LLama.Examples/Examples/Runner.cs | 2 +- 3 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 LLama.Examples/Assets/chat-with-kunkun-chinese.json diff --git a/LLama.Examples/Assets/chat-with-kunkun-chinese.json b/LLama.Examples/Assets/chat-with-kunkun-chinese.json new file mode 100644 index 00000000..30112327 --- /dev/null +++ b/LLama.Examples/Assets/chat-with-kunkun-chinese.json @@ -0,0 +1,24 @@ +{ + "messages": [ + { + "author_role": "System", + "content": "������һ������û��ĶԻ��������������һ���ڸ����涼ӵ�зḻ�������������dz����ڻش��û�������Ͱ����û���?" + }, + { + "author_role": "User", + "content": "��ã�������?" + }, + { + "author_role": "Assistant", + "content": "��ã���ʲô���ܰ��������" + }, + { + "author_role": "User", + "content": "�й����׶����������У�" + }, + { + "author_role": "Assistant", + "content": "��������˭��" + } + ] +} diff --git a/LLama.Examples/Examples/ChatChineseGB2312.cs b/LLama.Examples/Examples/ChatChineseGB2312.cs index ff27b962..d250a454 100644 --- a/LLama.Examples/Examples/ChatChineseGB2312.cs +++ b/LLama.Examples/Examples/ChatChineseGB2312.cs @@ -1,69 +1,124 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Text; using LLama.Common; -namespace LLama.Examples.Examples +namespace LLama.Examples.Examples; + +public class ChatChineseGB2312 { - public class ChatChineseGB2312 + private static string ConvertEncoding(string input, Encoding original, Encoding target) + { + byte[] bytes = original.GetBytes(input); + var convertedBytes = Encoding.Convert(original, target, bytes); + return target.GetString(convertedBytes); + } + + public static async Task Run() { - private static string ConvertFromEncodingToAnother(string input, Encoding original, Encoding target) + // Register provider for GB2312 encoding + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("This example shows how to use Chinese with gb2312 encoding, which is common in windows. It's recommended" + + " to use https://huggingface.co/hfl/chinese-alpaca-2-7b-gguf/blob/main/ggml-model-q5_0.gguf, which has been verified by LLamaSharp developers."); + Console.ForegroundColor = ConsoleColor.White; + + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + + var parameters = new ModelParams(modelPath) + { + ContextSize = 1024, + Seed = 1337, + GpuLayerCount = 5, + Encoding = Encoding.UTF8 + }; + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters); + var executor = new InteractiveExecutor(context); + + ChatSession session; + if (Directory.Exists("Assets/chat-with-kunkun-chinese")) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Loading session from disk."); + Console.ForegroundColor = ConsoleColor.White; + + session = new ChatSession(executor); + session.LoadSession("Assets/chat-with-kunkun-chinese"); + } + else { - byte[] bytes = original.GetBytes(input); - var convertedBytes = Encoding.Convert(original, target, bytes); - return target.GetString(convertedBytes); + var chatHistoryJson = File.ReadAllText("Assets/chat-with-kunkun-chinese.json"); + ChatHistory chatHistory = ChatHistory.FromJson(chatHistoryJson) ?? new ChatHistory(); + + session = new ChatSession(executor, chatHistory); } - public static async Task Run() + session + .WithHistoryTransform(new LLamaTransforms.DefaultHistoryTransform("用户")) + .WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform( + // User and Assistant in Chinese (User is: 用户, Assistant is: 坤坤) + new string[] { "用户：", "坤坤：" }, + redundancyLength: 8)); + + InferenceParams inferenceParams = new InferenceParams() { - Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); // Register gb2312 encoding - Console.Write("Please input your model path: "); - var modelPath = Console.ReadLine(); - var prompt = File.ReadAllText("Assets/chat-with-kunkun-chinese.txt", encoding: Encoding.GetEncoding("gb2312")).Trim(); - prompt = ConvertFromEncodingToAnother(prompt, Encoding.GetEncoding("gb2312"), Encoding.UTF8); + Temperature = 0.9f, + AntiPrompts = new List { "用户：" } + }; - var parameters = new ModelParams(modelPath) - { - ContextSize = 1024, - Seed = 1337, - GpuLayerCount = 20, - Encoding = Encoding.UTF8 - }; - using var model = LLamaWeights.LoadFromFile(parameters); - using var context = model.CreateContext(parameters); - var executor = new InteractiveExecutor(context); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started."); - var session = new ChatSession(executor).WithHistoryTransform(new LLamaTransforms.DefaultHistoryTransform("用户")); + // show the prompt + Console.ForegroundColor = ConsoleColor.Green; + string userInput = Console.ReadLine() ?? ""; - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine("This example shows how to use Chinese with gb2312 encoding, which is common in windows. It's recommended" + - " to use https://huggingface.co/hfl/chinese-alpaca-2-7b-gguf/blob/main/ggml-model-q5_0.gguf, which has been verified by LLamaSharp developers."); - Console.ForegroundColor = ConsoleColor.White; + while (userInput != "exit") + { + // Convert the encoding from gb2312 to utf8 for the language model + // and later saving to the history json file. + userInput = ConvertEncoding(userInput, Encoding.GetEncoding("gb2312"), Encoding.UTF8); - // show the prompt - Console.Write(prompt); - while (true) + if (userInput == "save") { - await foreach (var text in session.ChatAsync(prompt, new InferenceParams() + session.SaveSession("Assets/chat-with-kunkun-chinese"); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Session saved."); + } + else if (userInput == "regenerate") + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Regenerating last response ..."); + + await foreach ( + var text + in session.RegenerateAssistantMessageAsync( + inferenceParams)) { - Temperature = 0.3f, - TopK = 5, - TopP = 0.85f, - AntiPrompts = new List { "用户：" }, - MaxTokens = 2048, - RepeatPenalty = 1.05f - })) + Console.ForegroundColor = ConsoleColor.White; + + // Convert the encoding from utf8 to gb2312 for the console output. + Console.Write(ConvertEncoding(text, Encoding.UTF8, Encoding.GetEncoding("gb2312"))); + } + } + else + { + await foreach ( + var text + in session.ChatAsync( + new ChatHistory.Message(AuthorRole.User, userInput), + inferenceParams)) { - //Console.Write(text); - Console.Write(ConvertFromEncodingToAnother(text, Encoding.UTF8, Encoding.GetEncoding("gb2312"))); + Console.ForegroundColor = ConsoleColor.White; + Console.Write(text); } - - Console.ForegroundColor = ConsoleColor.Green; - prompt = Console.ReadLine(); - Console.ForegroundColor = ConsoleColor.White; } + + Console.ForegroundColor = ConsoleColor.Green; + userInput = Console.ReadLine() ?? ""; + + Console.ForegroundColor = ConsoleColor.White; } } } diff --git a/LLama.Examples/Examples/Runner.cs b/LLama.Examples/Examples/Runner.cs index 0a37dcba..3d9858e1 100644 --- a/LLama.Examples/Examples/Runner.cs +++ b/LLama.Examples/Examples/Runner.cs @@ -9,6 +9,7 @@ public class Runner { "Run a chat session with history.", ChatSessionWithHistory.Run }, { "Run a chat session without stripping the role names.", ChatSessionWithRoleName.Run }, { "Run a chat session with the role names stripped.", ChatSessionStripRoleName.Run }, + { "Run a chat session in Chinese GB2312 encoding", ChatChineseGB2312.Run }, { "Interactive mode chat by using executor.", InteractiveModeExecute.Run }, { "Instruct mode chat by using executor.", InstructModeExecute.Run }, { "Stateless mode chat by using executor.", StatelessModeExecute.Run }, @@ -24,7 +25,6 @@ public class Runner { "Coding Assistant.", CodingAssistant.Run }, { "Batch Decoding.", BatchedDecoding.Run }, { "SK Kernel Memory.", KernelMemory.Run }, - { "Chinese gb2312 chat", ChatChineseGB2312.Run }, { "Exit", async () => Environment.Exit(0) } }; From 29c5c6e93c806aa5f719c25c9c1987690419999f Mon Sep 17 00:00:00 2001 From: Philipp Bauer Date: Sun, 10 Dec 2023 09:34:32 -0600 Subject: [PATCH 09/11] Update the StatefulChatService to use new ChatSession integration --- LLama.WebAPI/Services/StatefulChatService.cs | 35 +++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/LLama.WebAPI/Services/StatefulChatService.cs b/LLama.WebAPI/Services/StatefulChatService.cs index f1eb3538..f45c98ee 100644 --- a/LLama.WebAPI/Services/StatefulChatService.cs +++ b/LLama.WebAPI/Services/StatefulChatService.cs @@ -11,8 +11,7 @@ public class StatefulChatService : IDisposable private readonly LLamaContext _context; private bool _continue = false; - private const string SystemPrompt = "Transcript of a dialog, where the User interacts with an Assistant. Assistant is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.\n\n" - + "User: "; + private const string SystemPrompt = "Transcript of a dialog, where the User interacts with an Assistant. Assistant is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision."; public StatefulChatService(IConfiguration configuration) { @@ -25,7 +24,9 @@ public class StatefulChatService : IDisposable using var weights = LLamaWeights.LoadFromFile(@params); _context = new LLamaContext(weights, @params); + _session = new ChatSession(new InteractiveExecutor(_context)); + _session.History.AddMessage(Common.AuthorRole.System, SystemPrompt); } public void Dispose() @@ -35,10 +36,8 @@ public class StatefulChatService : IDisposable public async Task Send(SendMessageInput input) { - var userInput = input.Text; if (!_continue) { - userInput = SystemPrompt + userInput; Console.Write(SystemPrompt); _continue = true; } @@ -47,11 +46,14 @@ public class StatefulChatService : IDisposable Console.Write(input.Text); Console.ForegroundColor = ConsoleColor.White; - var outputs = _session.ChatAsync(userInput, new Common.InferenceParams() - { - RepeatPenalty = 1.0f, - AntiPrompts = new string[] { "User:" }, - }); + var outputs = _session.ChatAsync( + new Common.ChatHistory.Message(Common.AuthorRole.User, input.Text), + new Common.InferenceParams() + { + RepeatPenalty = 1.0f, + AntiPrompts = new string[] { "User:" }, + }); + var result = ""; await foreach (var output in outputs) { @@ -64,10 +66,8 @@ public class StatefulChatService : IDisposable public async IAsyncEnumerable SendStream(SendMessageInput input) { - var userInput = input.Text; if (!_continue) { - userInput = SystemPrompt + userInput; Console.Write(SystemPrompt); _continue = true; } @@ -76,11 +76,14 @@ public class StatefulChatService : IDisposable Console.Write(input.Text); Console.ForegroundColor = ConsoleColor.White; - var outputs = _session.ChatAsync(userInput, new Common.InferenceParams() - { - RepeatPenalty = 1.0f, - AntiPrompts = new string[] { "User:" }, - }); + var outputs = _session.ChatAsync( + new Common.ChatHistory.Message(Common.AuthorRole.User, input.Text) + , new Common.InferenceParams() + { + RepeatPenalty = 1.0f, + AntiPrompts = new string[] { "User:" }, + }); + await foreach (var output in outputs) { Console.Write(output); From 836f071cd0aebf1d19cd1eb620e633f8f7e66567 Mon Sep 17 00:00:00 2001 From: Rinne Date: Mon, 11 Dec 2023 22:21:54 +0800 Subject: [PATCH 10/11] fix: Chinese example. --- LLama.Examples/Assets/chat-with-kunkun-chinese.json | 10 +++++----- LLama.Examples/Examples/ChatChineseGB2312.cs | 6 +----- LLama.Examples/LLama.Examples.csproj | 3 +++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/LLama.Examples/Assets/chat-with-kunkun-chinese.json b/LLama.Examples/Assets/chat-with-kunkun-chinese.json index 30112327..cae03029 100644 --- a/LLama.Examples/Assets/chat-with-kunkun-chinese.json +++ b/LLama.Examples/Assets/chat-with-kunkun-chinese.json @@ -2,23 +2,23 @@ "messages": [ { "author_role": "System", - "content": "������һ������û��ĶԻ��������������һ���ڸ����涼ӵ�зḻ�������������dz����ڻش��û�������Ͱ����û���?" + "content": "下面是一段你和用户的对话，你叫坤坤，是一个在各方面都拥有丰富经验的助理，你非常乐于回答用户的问题和帮助用户。" }, { "author_role": "User", - "content": "��ã�������?" + "content": "你好，坤坤。" }, { "author_role": "Assistant", - "content": "��ã���ʲô���ܰ��������" + "content": "你好，有什么我能帮助你的吗？" }, { "author_role": "User", - "content": "�й����׶����������У�" + "content": "中国的首都是哪座城市？" }, { "author_role": "Assistant", - "content": "��������˭��" + "content": "中国的首都是北京市。" } ] } diff --git a/LLama.Examples/Examples/ChatChineseGB2312.cs b/LLama.Examples/Examples/ChatChineseGB2312.cs index d250a454..bb3d3f80 100644 --- a/LLama.Examples/Examples/ChatChineseGB2312.cs +++ b/LLama.Examples/Examples/ChatChineseGB2312.cs @@ -55,11 +55,7 @@ public class ChatChineseGB2312 } session - .WithHistoryTransform(new LLamaTransforms.DefaultHistoryTransform("用户")) - .WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform( - // User and Assistant in Chinese (User is: 用户, Assistant is: 坤坤) - new string[] { "用户：", "坤坤：" }, - redundancyLength: 8)); + .WithHistoryTransform(new LLamaTransforms.DefaultHistoryTransform("用户", "坤坤")); InferenceParams inferenceParams = new InferenceParams() { diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index 958bb6c4..d3acbb82 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -44,6 +44,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest From fb75e06293d904e13a930759dc9b1661fe61a15e Mon Sep 17 00:00:00 2001 From: Rinne Date: Mon, 11 Dec 2023 22:27:47 +0800 Subject: [PATCH 11/11] fix: output prefix of Chinese example. --- LLama.Examples/Examples/ChatChineseGB2312.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LLama.Examples/Examples/ChatChineseGB2312.cs b/LLama.Examples/Examples/ChatChineseGB2312.cs index bb3d3f80..3a9fe6c7 100644 --- a/LLama.Examples/Examples/ChatChineseGB2312.cs +++ b/LLama.Examples/Examples/ChatChineseGB2312.cs @@ -67,6 +67,8 @@ public class ChatChineseGB2312 Console.WriteLine("The chat session has started."); // show the prompt + Console.ForegroundColor = ConsoleColor.White; + Console.Write("用户："); Console.ForegroundColor = ConsoleColor.Green; string userInput = Console.ReadLine() ?? "";