You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

LoadAndSaveState.md 2.9 kB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576
  1. # Executor - save/load state
  2. ```cs
  3. using LLama.Common;
  4. namespace LLama.Examples.Examples
  5. {
  6. // This example shows how to save/load state of the executor.
  7. public class LoadAndSaveState
  8. {
  9. public static async Task Run()
  10. {
  11. string modelPath = UserSettings.GetModelPath();
  12. var prompt = (await File.ReadAllTextAsync("Assets/chat-with-bob.txt")).Trim();
  13. var parameters = new ModelParams(modelPath)
  14. {
  15. ContextSize = 1024,
  16. Seed = 1337,
  17. GpuLayerCount = 5
  18. };
  19. using var model = LLamaWeights.LoadFromFile(parameters);
  20. using var context = model.CreateContext(parameters);
  21. var ex = new InteractiveExecutor(context);
  22. Console.ForegroundColor = ConsoleColor.Yellow;
  23. Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, " +
  24. "the maximum tokens is set to 64 and the context size is 256. (an example for small scale usage)");
  25. Console.ForegroundColor = ConsoleColor.White;
  26. Console.Write(prompt);
  27. var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "User:" } };
  28. while (true)
  29. {
  30. await foreach (var text in ex.InferAsync(prompt, inferenceParams))
  31. {
  32. Console.Write(text);
  33. }
  34. prompt = Console.ReadLine();
  35. if (prompt == "save")
  36. {
  37. Console.Write("Your path to save model state: ");
  38. var modelStatePath = Console.ReadLine();
  39. ex.Context.SaveState(modelStatePath);
  40. Console.Write("Your path to save executor state: ");
  41. var executorStatePath = Console.ReadLine();
  42. await ex.SaveState(executorStatePath);
  43. Console.ForegroundColor = ConsoleColor.Yellow;
  44. Console.WriteLine("All states saved!");
  45. Console.ForegroundColor = ConsoleColor.White;
  46. var ctx = ex.Context;
  47. ctx.LoadState(modelStatePath);
  48. ex = new InteractiveExecutor(ctx);
  49. await ex.LoadState(executorStatePath);
  50. Console.ForegroundColor = ConsoleColor.Yellow;
  51. Console.WriteLine("Loaded state!");
  52. Console.ForegroundColor = ConsoleColor.White;
  53. Console.Write("Now you can continue your session: ");
  54. Console.ForegroundColor = ConsoleColor.Green;
  55. prompt = Console.ReadLine();
  56. Console.ForegroundColor = ConsoleColor.White;
  57. }
  58. }
  59. }
  60. }
  61. }
  62. ```