diff --git a/LLama.Examples/Examples/BatchedExecutorFork.cs b/LLama.Examples/Examples/BatchedExecutorFork.cs index 3366951a..2909719f 100644 --- a/LLama.Examples/Examples/BatchedExecutorFork.cs +++ b/LLama.Examples/Examples/BatchedExecutorFork.cs @@ -31,7 +31,8 @@ public class BatchedExecutorFork Console.WriteLine($"Created executor with model: {name}"); // Evaluate the initial prompt to create one conversation - using var start = executor.Prompt(prompt); + using var start = executor.Create(); + start.Prompt(prompt); await executor.Infer(); // Create the root node of the tree diff --git a/LLama.Examples/Examples/BatchedExecutorGuidance.cs b/LLama.Examples/Examples/BatchedExecutorGuidance.cs index de78085a..29f6aa33 100644 --- a/LLama.Examples/Examples/BatchedExecutorGuidance.cs +++ b/LLama.Examples/Examples/BatchedExecutorGuidance.cs @@ -33,8 +33,10 @@ public class BatchedExecutorGuidance Console.WriteLine($"Created executor with model: {name}"); // Load the two prompts into two conversations - using var guided = executor.Prompt(positivePrompt); - using var guidance = executor.Prompt(negativePrompt); + using var guided = executor.Create(); + guided.Prompt(positivePrompt); + using var guidance = executor.Create(); + guidance.Prompt(negativePrompt); // Run inference to evaluate prompts await AnsiConsole diff --git a/LLama.Examples/Examples/BatchedExecutorRewind.cs b/LLama.Examples/Examples/BatchedExecutorRewind.cs index 4007dd50..8aae92af 100644 --- a/LLama.Examples/Examples/BatchedExecutorRewind.cs +++ b/LLama.Examples/Examples/BatchedExecutorRewind.cs @@ -32,7 +32,8 @@ public class BatchedExecutorRewind Console.WriteLine($"Created executor with model: {name}"); // Evaluate the initial prompt to create one conversation - using var conversation = executor.Prompt(prompt); + using var conversation = executor.Create(); + conversation.Prompt(prompt); // Create the start node wrapping the conversation var node = new Node(executor.Context); diff --git a/LLama/Batched/BatchedExecutor.cs b/LLama/Batched/BatchedExecutor.cs index 8ee58a7f..b15d445e 100644 --- a/LLama/Batched/BatchedExecutor.cs +++ b/LLama/Batched/BatchedExecutor.cs @@ -68,17 +68,30 @@ public sealed class BatchedExecutor /// /// /// + [Obsolete("Use BatchedExecutor.Create instead")] public Conversation Prompt(string prompt) { if (IsDisposed) throw new ObjectDisposedException(nameof(BatchedExecutor)); - var conversation = new Conversation(this, GetNextSequenceId(), 0); + var conversation = Create(); conversation.Prompt(prompt); return conversation; } + /// + /// Start a new + /// + /// + public Conversation Create() + { + if (IsDisposed) + throw new ObjectDisposedException(nameof(BatchedExecutor)); + + return new Conversation(this, GetNextSequenceId(), 0); + } + /// /// Run inference for all conversations in the batch which have pending tokens. /// diff --git a/LLama/LLamaContext.cs b/LLama/LLamaContext.cs index 253fe4c4..0db9e19c 100644 --- a/LLama/LLamaContext.cs +++ b/LLama/LLamaContext.cs @@ -85,6 +85,11 @@ namespace LLama } } + /// + /// Get the maximum batch size for this context + /// + public uint BatchSize => NativeHandle.BatchSize; + /// /// Create a new LLamaContext for the given LLamaWeights ///