From 286904920b499b50ab343f85990c509656ef7ef6 Mon Sep 17 00:00:00 2001 From: xbotter Date: Sat, 18 Nov 2023 21:40:04 +0800 Subject: [PATCH] update DefaultInferenceParams in WithLLamaSharpDefaults --- LLama.Examples/Examples/KernelMemory.cs | 8 +++++++- LLama.Examples/Examples/Runner.cs | 3 ++- LLama.KernelMemory/BuilderExtensions.cs | 2 +- LLama.KernelMemory/LlamaSharpTextGeneration.cs | 18 +++++++++--------- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/LLama.Examples/Examples/KernelMemory.cs b/LLama.Examples/Examples/KernelMemory.cs index 0f63447f..0c2b908f 100644 --- a/LLama.Examples/Examples/KernelMemory.cs +++ b/LLama.Examples/Examples/KernelMemory.cs @@ -17,7 +17,13 @@ namespace LLama.Examples.Examples Console.Write("Please input your model path: "); var modelPath = Console.ReadLine(); var memory = new KernelMemoryBuilder() - .WithLLamaSharpDefaults(new LLamaSharpConfig(modelPath)) + .WithLLamaSharpDefaults(new LLamaSharpConfig(modelPath) + { + DefaultInferenceParams = new Common.InferenceParams + { + AntiPrompts = new List { "\n\n" } + } + }) .With(new TextPartitioningOptions { MaxTokensPerParagraph = 300, diff --git a/LLama.Examples/Examples/Runner.cs b/LLama.Examples/Examples/Runner.cs index f2f1351f..aca0a7da 100644 --- a/LLama.Examples/Examples/Runner.cs +++ b/LLama.Examples/Examples/Runner.cs @@ -42,7 +42,8 @@ public class Runner AnsiConsole.Write(new Rule(choice)); await example(); } - + Console.WriteLine("Press any key to continue..."); + Console.ReadKey(); AnsiConsole.Clear(); } } diff --git a/LLama.KernelMemory/BuilderExtensions.cs b/LLama.KernelMemory/BuilderExtensions.cs index 0b92ca6e..dc746dc6 100644 --- a/LLama.KernelMemory/BuilderExtensions.cs +++ b/LLama.KernelMemory/BuilderExtensions.cs @@ -82,7 +82,7 @@ namespace LLamaSharp.KernelMemory var executor = new StatelessExecutor(weights, parameters); var embedder = new LLamaEmbedder(weights, parameters); builder.WithLLamaSharpTextEmbeddingGeneration(new LLamaSharpTextEmbeddingGeneration(embedder)); - builder.WithLLamaSharpTextGeneration(new LlamaSharpTextGeneration(weights, context, executor)); + builder.WithLLamaSharpTextGeneration(new LlamaSharpTextGeneration(weights, context, executor, config?.DefaultInferenceParams)); return builder; } } diff --git a/LLama.KernelMemory/LlamaSharpTextGeneration.cs b/LLama.KernelMemory/LlamaSharpTextGeneration.cs index dab918f6..663a77cf 100644 --- a/LLama.KernelMemory/LlamaSharpTextGeneration.cs +++ b/LLama.KernelMemory/LlamaSharpTextGeneration.cs @@ -15,10 +15,10 @@ namespace LLamaSharp.KernelMemory /// public class LlamaSharpTextGeneration : ITextGeneration, IDisposable { - private readonly LLamaSharpConfig? _config; private readonly LLamaWeights _weights; private readonly StatelessExecutor _executor; private readonly LLamaContext _context; + private readonly InferenceParams? _defaultInferenceParams; private bool _ownsContext = false; private bool _ownsWeights = false; @@ -28,7 +28,6 @@ namespace LLamaSharp.KernelMemory /// The configuration for LLamaSharp. public LlamaSharpTextGeneration(LLamaSharpConfig config) { - this._config = config; var parameters = new ModelParams(config.ModelPath) { ContextSize = config?.ContextSize ?? 2048, @@ -38,6 +37,7 @@ namespace LLamaSharp.KernelMemory _weights = LLamaWeights.LoadFromFile(parameters); _context = _weights.CreateContext(parameters); _executor = new StatelessExecutor(_weights, parameters); + _defaultInferenceParams = config?.DefaultInferenceParams; _ownsWeights = _ownsContext = true; } @@ -48,12 +48,12 @@ namespace LLamaSharp.KernelMemory /// A LLamaWeights object. /// A LLamaContext object. /// An executor. Currently only StatelessExecutor is expected. - public LlamaSharpTextGeneration(LLamaWeights weights, LLamaContext context, StatelessExecutor? executor = null) + public LlamaSharpTextGeneration(LLamaWeights weights, LLamaContext context, StatelessExecutor? executor = null, InferenceParams? inferenceParams = null) { - _config = null; _weights = weights; _context = context; _executor = executor ?? new StatelessExecutor(_weights, _context.Params); + _defaultInferenceParams = inferenceParams; } /// @@ -72,7 +72,7 @@ namespace LLamaSharp.KernelMemory /// public IAsyncEnumerable GenerateTextAsync(string prompt, TextGenerationOptions options, CancellationToken cancellationToken = default) { - return _executor.InferAsync(prompt, OptionsToParams(options, this._config?.DefaultInferenceParams), cancellationToken: cancellationToken); + return _executor.InferAsync(prompt, OptionsToParams(options, this._defaultInferenceParams), cancellationToken: cancellationToken); } private static InferenceParams OptionsToParams(TextGenerationOptions options, InferenceParams? defaultParams) @@ -82,11 +82,11 @@ namespace LLamaSharp.KernelMemory return defaultParams with { AntiPrompts = defaultParams.AntiPrompts.Concat(options.StopSequences).ToList().AsReadOnly(), - Temperature = options.Temperature == default ? defaultParams.Temperature : default, + Temperature = options.Temperature == defaultParams.Temperature ? defaultParams.Temperature : (float)options.Temperature, MaxTokens = options.MaxTokens ?? defaultParams.MaxTokens, - FrequencyPenalty = options.FrequencyPenalty == default ? defaultParams.FrequencyPenalty : default, - PresencePenalty = options.PresencePenalty == default ? defaultParams.PresencePenalty : default, - TopP = options.TopP == default ? defaultParams.TopP : default + FrequencyPenalty = options.FrequencyPenalty == defaultParams.FrequencyPenalty ? defaultParams.FrequencyPenalty : (float)options.FrequencyPenalty, + PresencePenalty = options.PresencePenalty == defaultParams.PresencePenalty ? defaultParams.PresencePenalty : (float)options.PresencePenalty, + TopP = options.TopP == defaultParams.TopP ? defaultParams.TopP : (float)options.TopP }; } else