LLamaSharp/LLama/Extensions/EncodingExtensions.cs

using System;
using System.Collections.Generic;
using System.Text;

namespace LLama.Extensions;

internal static class EncodingExtensions
{
#if NETSTANDARD2_0
    public static int GetChars(this Encoding encoding, ReadOnlySpan<byte> bytes, Span<char> output)
    {
        return GetCharsImpl(encoding, bytes, output);
    }

    public static int GetCharCount(this Encoding encoding, ReadOnlySpan<byte> bytes)
    {
        return GetCharCountImpl(encoding, bytes);
    }
#endif

    internal static int GetCharsImpl(Encoding encoding, ReadOnlySpan<byte> bytes, Span<char> output)
    {
        if (bytes.Length == 0)
            return 0;

        unsafe
        {
            fixed (byte* bytePtr = bytes)
            fixed (char* charPtr = output)
            {
                return encoding.GetChars(bytePtr, bytes.Length, charPtr, output.Length);
            }
        }
    }

    internal static int GetCharCountImpl(Encoding encoding, ReadOnlySpan<byte> bytes)
    {
        unsafe
        {
            fixed (byte* bytePtr = bytes)
            {
                return encoding.GetCharCount(bytePtr, bytes.Length);
            }
        }
    }
}