diff --git a/LLama.Examples/Assets/json.gbnf b/LLama.Examples/Assets/json.gbnf
new file mode 100644
index 00000000..a01c4efd
--- /dev/null
+++ b/LLama.Examples/Assets/json.gbnf
@@ -0,0 +1,27 @@
+# https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/grammars/json.gbnf
+
+root ::= object
+value ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+ "{" ws (
+ string ":" ws value
+ ("," ws string ":" ws value)*
+ )? "}" ws
+
+array ::=
+ "[" ws (
+ value
+ ("," ws value)*
+ )? "]" ws
+
+string ::=
+ "\"" (
+ [^"\\] |
+ "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+ )* "\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
\ No newline at end of file
diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj
index ef7ac437..6a1685ed 100644
--- a/LLama.Examples/LLama.Examples.csproj
+++ b/LLama.Examples/LLama.Examples.csproj
@@ -49,6 +49,9 @@
PreserveNewest
+
+ PreserveNewest
+
PreserveNewest
diff --git a/LLama.Examples/NewVersion/GrammarJsonResponse.cs b/LLama.Examples/NewVersion/GrammarJsonResponse.cs
new file mode 100644
index 00000000..926aa82b
--- /dev/null
+++ b/LLama.Examples/NewVersion/GrammarJsonResponse.cs
@@ -0,0 +1,55 @@
+using LLama.Common;
+using LLama.Grammar;
+using LLama.Native;
+
+namespace LLama.Examples.NewVersion
+{
+ public class GrammarJsonResponse
+ {
+ public static void Run()
+ {
+ var grammarBytes = File.ReadAllText("Assets/json.gbnf").Trim();
+ var parsedGrammar = new GrammarParser();
+
+ Console.Write("Please input your model path: ");
+ var modelPath = Console.ReadLine();
+
+ var parameters = new ModelParams(modelPath)
+ {
+ ContextSize = 1024,
+ Seed = 1337,
+ GpuLayerCount = 5
+ };
+ using var model = LLamaWeights.LoadFromFile(parameters);
+ var ex = new StatelessExecutor(model, parameters);
+ ParseState state = parsedGrammar.Parse(grammarBytes);
+ using var grammar = SafeLLamaGrammarHandle.Create(state.Rules, 0);
+
+ Console.ForegroundColor = ConsoleColor.Yellow;
+ Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions and always respond in a JSON format. For example, you can input \"Tell me the attributes of a good dish\"");
+ Console.ForegroundColor = ConsoleColor.White;
+
+ var inferenceParams = new InferenceParams()
+ {
+ Temperature = 0.6f,
+ AntiPrompts = new List { "Question:", "#", "Question: ", ".\n" },
+ MaxTokens = 50,
+ Grammar = grammar
+ };
+
+ while (true)
+ {
+ Console.Write("\nQuestion: ");
+ Console.ForegroundColor = ConsoleColor.Green;
+ var prompt = Console.ReadLine();
+ Console.ForegroundColor = ConsoleColor.White;
+ Console.Write("Answer: ");
+ prompt = $"Question: {prompt?.Trim()} Answer: ";
+ foreach (var text in ex.Infer(prompt, inferenceParams))
+ {
+ Console.Write(text);
+ }
+ }
+ }
+ }
+}
diff --git a/LLama.Examples/NewVersion/TestRunner.cs b/LLama.Examples/NewVersion/TestRunner.cs
index 6cc3f3da..f5a10ef4 100644
--- a/LLama.Examples/NewVersion/TestRunner.cs
+++ b/LLama.Examples/NewVersion/TestRunner.cs
@@ -17,6 +17,7 @@
Console.WriteLine("7: Get embeddings from LLama model.");
Console.WriteLine("8: Quantize the model.");
Console.WriteLine("9: Automatic conversation.");
+ Console.WriteLine("10: Constrain response to json format using grammar.");
while (true)
{
@@ -63,6 +64,10 @@
{
await TalkToYourself.Run();
}
+ else if (choice == 10)
+ {
+ GrammarJsonResponse.Run();
+ }
else
{
Console.WriteLine("Cannot parse your choice. Please select again.");