using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Text; using LLama; using LLama.Exceptions; namespace LLama.Native { /// /// A reference to a set of llava model weights /// public sealed class SafeLlavaModelHandle : SafeLLamaHandleBase { private SafeLlavaModelHandle(IntPtr handle) : base(handle, true) { } private SafeLlavaModelHandle() {} /// protected override bool ReleaseHandle() { clip_free(DangerousGetHandle()); SetHandle(IntPtr.Zero); return true; } /// /// Load a model from the given file path into memory /// /// /// /// /// public static SafeLlavaModelHandle LoadFromFile(string modelPath, int verbosity ) { // Try to open the model file, this will check: // - File exists (automatically throws FileNotFoundException) // - File is readable (explicit check) // This provides better error messages that llama.cpp, which would throw an access violation exception in both cases. using (var fs = new FileStream(modelPath, FileMode.Open)) if (!fs.CanRead) throw new InvalidOperationException($"Llava MMP Model file '{modelPath}' is not readable"); return clip_model_load(modelPath, verbosity) ?? throw new RuntimeError($"Failed to load LLaVa model {modelPath}."); } /// /// Embed the image from file in llama context /// /// /// /// /// public bool EmbedImage(LLamaContext ctxLlama, string image, ref int n_past) { var ImageEmbed = SafeLlavaImageEmbedHandle.CreateFromFileName(this, ctxLlama, image); bool result = NativeApi.llava_eval_image_embed(ctxLlama.NativeHandle, ImageEmbed, (int)ctxLlama.Params.BatchSize, ref n_past ); return result; } /// /// Embed the image from binary in llama context /// /// /// jpeg image /// /// public bool EmbedImage(LLamaContext ctxLlama, Byte[] image, ref int n_past ) { var ImageEmbed = SafeLlavaImageEmbedHandle.CreateFromMemory(this, ctxLlama, image ); bool result = NativeApi.llava_eval_image_embed(ctxLlama.NativeHandle, ImageEmbed, (int)ctxLlama.Params.BatchSize, ref n_past ); return result; } /// /// Load MULTI MODAL PROJECTIONS model / Clip Model /// /// Model path/file /// Verbosity level /// SafeLlavaModelHandle [DllImport(NativeApi.llavaLibraryName, EntryPoint = "clip_model_load", CallingConvention = CallingConvention.Cdecl)] private static extern SafeLlavaModelHandle clip_model_load(string mmProj, int verbosity); /// /// Frees MULTI MODAL PROJECTIONS model / Clip Model /// /// [DllImport(NativeApi.llavaLibraryName, EntryPoint = "clip_free", CallingConvention = CallingConvention.Cdecl)] private static extern void clip_free(IntPtr ctx); } }