Browse Source

Merge pull request #499 from martindevans/better_batch_processing

Using `AddRange` in `LLamaEmbedder`
tags/v0.10.0
Martin Evans GitHub 1 year ago
parent
commit
a8f9262b7f
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
  1. +2
    -2
      LLama/LLamaEmbedder.cs

+ 2
- 2
LLama/LLamaEmbedder.cs View File

@@ -75,8 +75,8 @@ namespace LLama
n_eval = batchSize; n_eval = batchSize;


batch.Clear(); batch.Clear();
for (var j = 0; j < n_eval; j++)
batch.Add(tokens[i + j], n_past++, LLamaSeqId.Zero, false);
batch.AddRange(tokens.AsSpan(i, n_eval), n_past, LLamaSeqId.Zero, false);
n_past += n_eval;


var returnCode = await Context.DecodeAsync(batch, cancellationToken); var returnCode = await Context.DecodeAsync(batch, cancellationToken);
if (returnCode != 0) if (returnCode != 0)


Loading…
Cancel
Save