From 2fe4ccfb58fdd0529233693a93c60c90237f99fc Mon Sep 17 00:00:00 2001 From: xbotter Date: Sat, 24 Jun 2023 19:05:02 +0800 Subject: [PATCH] update webapi example --- LLama.WebAPI/Controllers/ChatController.cs | 4 +- LLama.WebAPI/LLama.WebAPI.csproj | 1 + LLama.WebAPI/Program.cs | 3 +- LLama.WebAPI/Services/ChatService.cs | 34 ----------- LLama.WebAPI/Services/StatefulChatService.cs | 56 +++++++++++++++++++ LLama.WebAPI/Services/StatelessChatService.cs | 6 ++ LLama.WebAPI/appsettings.json | 3 +- 7 files changed, 69 insertions(+), 38 deletions(-) delete mode 100644 LLama.WebAPI/Services/ChatService.cs create mode 100644 LLama.WebAPI/Services/StatefulChatService.cs create mode 100644 LLama.WebAPI/Services/StatelessChatService.cs diff --git a/LLama.WebAPI/Controllers/ChatController.cs b/LLama.WebAPI/Controllers/ChatController.cs index cf64a9e5..1953d2af 100644 --- a/LLama.WebAPI/Controllers/ChatController.cs +++ b/LLama.WebAPI/Controllers/ChatController.cs @@ -9,11 +9,11 @@ namespace LLama.WebAPI.Controllers [Route("[controller]")] public class ChatController : ControllerBase { - private readonly ChatService _service; + private readonly StatefulChatService _service; private readonly ILogger _logger; public ChatController(ILogger logger, - ChatService service) + StatefulChatService service) { _logger = logger; _service = service; diff --git a/LLama.WebAPI/LLama.WebAPI.csproj b/LLama.WebAPI/LLama.WebAPI.csproj index 9227d57f..2b14d8a4 100644 --- a/LLama.WebAPI/LLama.WebAPI.csproj +++ b/LLama.WebAPI/LLama.WebAPI.csproj @@ -7,6 +7,7 @@ + diff --git a/LLama.WebAPI/Program.cs b/LLama.WebAPI/Program.cs index 33e8bf81..3f2de200 100644 --- a/LLama.WebAPI/Program.cs +++ b/LLama.WebAPI/Program.cs @@ -9,7 +9,8 @@ builder.Services.AddControllers(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddScoped(); var app = builder.Build(); diff --git a/LLama.WebAPI/Services/ChatService.cs b/LLama.WebAPI/Services/ChatService.cs deleted file mode 100644 index e457e3c2..00000000 --- a/LLama.WebAPI/Services/ChatService.cs +++ /dev/null @@ -1,34 +0,0 @@ -using LLama.OldVersion; -using LLama.WebAPI.Models; - -namespace LLama.WebAPI.Services; - -public class ChatService -{ - private readonly ChatSession _session; - - public ChatService() - { - LLamaModel model = new(new LLamaParams(model: @"ggml-model-q4_0.bin", n_ctx: 512, interactive: true, repeat_penalty: 1.0f, verbose_prompt: false)); - _session = new ChatSession(model) - .WithPromptFile(@"Assets\chat-with-bob.txt") - .WithAntiprompt(new string[] { "User:" }); - } - - public string Send(SendMessageInput input) - { - Console.ForegroundColor = ConsoleColor.Green; - Console.WriteLine(input.Text); - - Console.ForegroundColor = ConsoleColor.White; - var outputs = _session.Chat(input.Text); - var result = ""; - foreach (var output in outputs) - { - Console.Write(output); - result += output; - } - - return result; - } -} diff --git a/LLama.WebAPI/Services/StatefulChatService.cs b/LLama.WebAPI/Services/StatefulChatService.cs new file mode 100644 index 00000000..9c06e8b5 --- /dev/null +++ b/LLama.WebAPI/Services/StatefulChatService.cs @@ -0,0 +1,56 @@ + +using LLama.WebAPI.Models; +using Microsoft; + +namespace LLama.WebAPI.Services; + +public class StatefulChatService : IDisposable +{ + private readonly ChatSession _session; + private readonly LLamaModel _model; + private bool _continue = false; + + private const string SystemPrompt = "Transcript of a dialog, where the User interacts with an Assistant. Assistant is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.\n\n" + + "User: "; + + public StatefulChatService(IConfiguration configuration) + { + _model = new LLamaModel(new Common.ModelParams(configuration["ModelPath"], contextSize: 512)); + _session = new ChatSession(new InteractiveExecutor(_model)) + ;//.WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Assistant:" }, redundancyLength: 8)); + } + + public void Dispose() + { + _model?.Dispose(); + } + + public string Send(SendMessageInput input) + { + var userInput = input.Text; + if (!_continue) + { + userInput = SystemPrompt + userInput; + Console.Write(SystemPrompt); + _continue = true; + } + + Console.ForegroundColor = ConsoleColor.Green; + Console.Write(input.Text); + + Console.ForegroundColor = ConsoleColor.White; + var outputs = _session.Chat(userInput, new Common.InferenceParams() + { + RepeatPenalty = 1.0f, + AntiPrompts = new string[] { "User:" }, + }); + var result = ""; + foreach (var output in outputs) + { + Console.Write(output); + result += output; + } + + return result; + } +} diff --git a/LLama.WebAPI/Services/StatelessChatService.cs b/LLama.WebAPI/Services/StatelessChatService.cs new file mode 100644 index 00000000..c87ae61f --- /dev/null +++ b/LLama.WebAPI/Services/StatelessChatService.cs @@ -0,0 +1,6 @@ +namespace LLama.WebAPI.Services +{ + public class StatelessChatService + { + } +} diff --git a/LLama.WebAPI/appsettings.json b/LLama.WebAPI/appsettings.json index 10f68b8c..09cfdb13 100644 --- a/LLama.WebAPI/appsettings.json +++ b/LLama.WebAPI/appsettings.json @@ -5,5 +5,6 @@ "Microsoft.AspNetCore": "Warning" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "ModelPath": "..\\..\\LLamaModel\\ggml-model-f32-q4_0.bin" }