diff --git a/LLama.Examples/NewVersion/ChatSessionStripRoleName.cs b/LLama.Examples/NewVersion/ChatSessionStripRoleName.cs index 6402e360..230118e5 100644 --- a/LLama.Examples/NewVersion/ChatSessionStripRoleName.cs +++ b/LLama.Examples/NewVersion/ChatSessionStripRoleName.cs @@ -1,9 +1,5 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -12,15 +8,22 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - InteractiveExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); - ChatSession session = new ChatSession(ex).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); + + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var executor = new InteractiveExecutor(context); + + var session = new ChatSession(executor).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The chat session has started. The role names won't be printed."); Console.ForegroundColor = ConsoleColor.White; + // show the prompt + Console.Write(prompt); while (true) { foreach (var text in session.Chat(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" } })) diff --git a/LLama.Examples/NewVersion/ChatSessionWithRoleName.cs b/LLama.Examples/NewVersion/ChatSessionWithRoleName.cs index d1cbf34b..a3609388 100644 --- a/LLama.Examples/NewVersion/ChatSessionWithRoleName.cs +++ b/LLama.Examples/NewVersion/ChatSessionWithRoleName.cs @@ -1,9 +1,5 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -12,10 +8,15 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - InteractiveExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); - ChatSession session = new ChatSession(ex); // The only change is to remove the transform for the output text stream. + + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var executor = new InteractiveExecutor(context); + + var session = new ChatSession(executor); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result."); diff --git a/LLama.Examples/NewVersion/GetEmbeddings.cs b/LLama.Examples/NewVersion/GetEmbeddings.cs index ed12f868..516d2da7 100644 --- a/LLama.Examples/NewVersion/GetEmbeddings.cs +++ b/LLama.Examples/NewVersion/GetEmbeddings.cs @@ -1,9 +1,4 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -12,7 +7,7 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); var embedder = new LLamaEmbedder(new ModelParams(modelPath)); while (true) diff --git a/LLama.Examples/NewVersion/InstructModeExecute.cs b/LLama.Examples/NewVersion/InstructModeExecute.cs index f81f2f58..0a384062 100644 --- a/LLama.Examples/NewVersion/InstructModeExecute.cs +++ b/LLama.Examples/NewVersion/InstructModeExecute.cs @@ -1,9 +1,5 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -12,10 +8,13 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); var prompt = File.ReadAllText("Assets/dan.txt").Trim(); - InstructExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 1024))); + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var executor = new InstructExecutor(context); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions. For example, you can input \"Write a story about a fox who want to " + @@ -26,7 +25,7 @@ namespace LLama.Examples.NewVersion while (true) { - foreach (var text in ex.Infer(prompt, inferenceParams)) + foreach (var text in executor.Infer(prompt, inferenceParams)) { Console.Write(text); } diff --git a/LLama.Examples/NewVersion/InteractiveModeExecute.cs b/LLama.Examples/NewVersion/InteractiveModeExecute.cs index aaacabbe..9fee007f 100644 --- a/LLama.Examples/NewVersion/InteractiveModeExecute.cs +++ b/LLama.Examples/NewVersion/InteractiveModeExecute.cs @@ -1,21 +1,20 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { public class InteractiveModeExecute { - public async static Task Run() + public static async Task Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); - var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); + var modelPath = Console.ReadLine(); + var prompt = (await File.ReadAllTextAsync("Assets/chat-with-bob.txt")).Trim(); - InteractiveExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 256))); + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var ex = new InteractiveExecutor(context); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to 128 and the context size is 256. (an example for small scale usage)"); diff --git a/LLama.Examples/NewVersion/LoadAndSaveSession.cs b/LLama.Examples/NewVersion/LoadAndSaveSession.cs index cbed9179..5e5c4252 100644 --- a/LLama.Examples/NewVersion/LoadAndSaveSession.cs +++ b/LLama.Examples/NewVersion/LoadAndSaveSession.cs @@ -1,10 +1,5 @@ using LLama.Common; -using LLama.OldVersion; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -13,10 +8,15 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - InteractiveExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); - ChatSession session = new ChatSession(ex); // The only change is to remove the transform for the output text stream. + + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var ex = new InteractiveExecutor(context); + + var session = new ChatSession(ex); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result. Input \"save\" to save and reload the session."); diff --git a/LLama.Examples/NewVersion/LoadAndSaveState.cs b/LLama.Examples/NewVersion/LoadAndSaveState.cs index 15f2f815..1a1c0d88 100644 --- a/LLama.Examples/NewVersion/LoadAndSaveState.cs +++ b/LLama.Examples/NewVersion/LoadAndSaveState.cs @@ -1,9 +1,5 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -12,10 +8,13 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); - InteractiveExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 256))); + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var ex = new InteractiveExecutor(context); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to 64 and the context size is 256. (an example for small scale usage)"); @@ -47,9 +46,9 @@ namespace LLama.Examples.NewVersion Console.WriteLine("All states saved!"); Console.ForegroundColor = ConsoleColor.White; - var model = ex.Context; - model.LoadState(modelStatePath); - ex = new InteractiveExecutor(model); + var ctx = ex.Context; + ctx.LoadState(modelStatePath); + ex = new InteractiveExecutor(ctx); ex.LoadState(executorStatePath); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Loaded state!"); diff --git a/LLama.Examples/NewVersion/QuantizeModel.cs b/LLama.Examples/NewVersion/QuantizeModel.cs index a5ad81d8..71966af8 100644 --- a/LLama.Examples/NewVersion/QuantizeModel.cs +++ b/LLama.Examples/NewVersion/QuantizeModel.cs @@ -1,11 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace LLama.Examples.NewVersion +namespace LLama.Examples.NewVersion { public class QuantizeModel { @@ -13,13 +6,16 @@ namespace LLama.Examples.NewVersion { Console.Write("Please input your original model path: "); var inputPath = Console.ReadLine(); + Console.Write("Please input your output model path: "); var outputPath = Console.ReadLine(); + Console.Write("Please input the quantize type (one of q4_0, q4_1, q5_0, q5_1, q8_0): "); var quantizeType = Console.ReadLine(); + if (LLamaQuantizer.Quantize(inputPath, outputPath, quantizeType)) { - Console.WriteLine("Quantization succeed!"); + Console.WriteLine("Quantization succeeded!"); } else { diff --git a/LLama.Examples/NewVersion/StatelessModeExecute.cs b/LLama.Examples/NewVersion/StatelessModeExecute.cs index 8ff2c0a1..dadaf70a 100644 --- a/LLama.Examples/NewVersion/StatelessModeExecute.cs +++ b/LLama.Examples/NewVersion/StatelessModeExecute.cs @@ -1,9 +1,5 @@ using LLama.Common; -using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; namespace LLama.Examples.NewVersion { @@ -12,9 +8,12 @@ namespace LLama.Examples.NewVersion public static void Run() { Console.Write("Please input your model path: "); - string modelPath = Console.ReadLine(); + var modelPath = Console.ReadLine(); - StatelessExecutor ex = new(new LLamaContext(new ModelParams(modelPath, contextSize: 256))); + var parameters = new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5); + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters, Encoding.UTF8); + var ex = new StatelessExecutor(context); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The executor has been enabled. In this example, the inference is an one-time job. That says, the previous input and response has " + diff --git a/LLama.Examples/NewVersion/TestRunner.cs b/LLama.Examples/NewVersion/TestRunner.cs index c90bc78d..6cc3f3da 100644 --- a/LLama.Examples/NewVersion/TestRunner.cs +++ b/LLama.Examples/NewVersion/TestRunner.cs @@ -1,10 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace LLama.Examples.NewVersion +namespace LLama.Examples.NewVersion { public class NewVersionTestRunner {