using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using LLama;
using LLama.Exceptions;
namespace LLama.Native
{
///
/// A reference to a set of llava model weights
///
public sealed class SafeLlavaModelHandle
: SafeLLamaHandleBase
{
private SafeLlavaModelHandle(IntPtr handle)
: base(handle, true)
{
}
private SafeLlavaModelHandle()
{}
///
protected override bool ReleaseHandle()
{
clip_free(DangerousGetHandle());
SetHandle(IntPtr.Zero);
return true;
}
///
/// Load a model from the given file path into memory
///
///
///
///
///
public static SafeLlavaModelHandle LoadFromFile(string modelPath, int verbosity )
{
// Try to open the model file, this will check:
// - File exists (automatically throws FileNotFoundException)
// - File is readable (explicit check)
// This provides better error messages that llama.cpp, which would throw an access violation exception in both cases.
using (var fs = new FileStream(modelPath, FileMode.Open))
if (!fs.CanRead)
throw new InvalidOperationException($"Llava MMP Model file '{modelPath}' is not readable");
return clip_model_load(modelPath, verbosity)
?? throw new RuntimeError($"Failed to load LLaVa model {modelPath}.");
}
///
/// Embed the image from file in llama context
///
///
///
///
///
public bool EmbedImage(LLamaContext ctxLlama, string image, ref int n_past)
{
var ImageEmbed = SafeLlavaImageEmbedHandle.CreateFromFileName(this, ctxLlama, image);
bool result = NativeApi.llava_eval_image_embed(ctxLlama.NativeHandle, ImageEmbed, (int)ctxLlama.Params.BatchSize, ref n_past );
return result;
}
///
/// Embed the image from binary in llama context
///
///
/// jpeg image
///
///
public bool EmbedImage(LLamaContext ctxLlama, Byte[] image, ref int n_past )
{
var ImageEmbed = SafeLlavaImageEmbedHandle.CreateFromMemory(this, ctxLlama, image );
bool result = NativeApi.llava_eval_image_embed(ctxLlama.NativeHandle, ImageEmbed, (int)ctxLlama.Params.BatchSize, ref n_past );
return result;
}
///
/// Load MULTI MODAL PROJECTIONS model / Clip Model
///
/// Model path/file
/// Verbosity level
/// SafeLlavaModelHandle
[DllImport(NativeApi.llavaLibraryName, EntryPoint = "clip_model_load", CallingConvention = CallingConvention.Cdecl)]
private static extern SafeLlavaModelHandle clip_model_load(string mmProj, int verbosity);
///
/// Frees MULTI MODAL PROJECTIONS model / Clip Model
///
///
[DllImport(NativeApi.llavaLibraryName, EntryPoint = "clip_free", CallingConvention = CallingConvention.Cdecl)]
private static extern void clip_free(IntPtr ctx);
}
}