Browse Source

Fix Tokenize of new line, Remove space inserts

tags/v0.5.1
sa_ddam213 2 years ago
parent
commit
a5d742b72c
Failed to extract signature
3 changed files with 1 additions and 7 deletions
  1. +0
    -4
      LLama/LLamaEmbedder.cs
  2. +0
    -1
      LLama/LLamaInstructExecutor.cs
  3. +1
    -2
      LLama/LLamaInteractExecutor.cs

+ 0
- 4
LLama/LLamaEmbedder.cs View File

@@ -65,10 +65,6 @@ namespace LLama
/// <exception cref="RuntimeError"></exception>
public float[] GetEmbeddings(string text, bool addBos)
{
if (addBos)
{
text = text.Insert(0, " ");
}

var embed_inp_array = _ctx.Tokenize(text, addBos);



+ 0
- 1
LLama/LLamaInstructExecutor.cs View File

@@ -114,7 +114,6 @@ namespace LLama
if (_is_prompt_run)
{
// When running the first input (prompt) in inteactive mode, we should specially process it.
text = " " + text;
_embed_inps = Context.Tokenize(text, true).ToList();
}
else


+ 1
- 2
LLama/LLamaInteractExecutor.cs View File

@@ -26,7 +26,7 @@ namespace LLama
/// <param name="context"></param>
public InteractiveExecutor(LLamaContext context) : base(context)
{
_llama_token_newline = Context.NativeHandle.Tokenize("\n", false, Context.Encoding);
_llama_token_newline = new [] { NativeApi.llama_token_nl(Context.NativeHandle) };
}

/// <inheritdoc />
@@ -104,7 +104,6 @@ namespace LLama
if (_is_prompt_run)
{
// When running the first input (prompt) in inteactive mode, we should specially process it.
text = " " + text;
_embed_inps = Context.Tokenize(text, true).ToList();
}
else


Loading…
Cancel
Save