scisharp
/
LLamaSharp


<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
        <link rel="next" href="GetStarted/">
      
      <link rel="icon" href="assets/images/favicon.png">
      <meta name="generator" content="mkdocs-1.4.3, mkdocs-material-9.1.16">
    
    
        <title>LLamaSharp Documentation</title>
      
    
      <link rel="stylesheet" href="assets/stylesheets/main.26e3688c.min.css">
      
      
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      
  </head>
  
  
    <body dir="ltr">
  
    
      <script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#overview" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
      <div data-md-color-scheme="default" data-md-component="outdated" hidden>
        
      </div>
    
    
<header class="md-header md-header--shadow" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="." title="LLamaSharp Documentation" class="md-header__button md-logo" aria-label="LLamaSharp Documentation" data-md-component="logo">
      
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"/></svg>

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            LLamaSharp Documentation
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              Overview
            
          </span>
        </div>
      </div>
    </div>
    
    
      <label class="md-header__button md-icon" for="__search">
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
        </button>
      </nav>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="." title="LLamaSharp Documentation" class="md-nav__button md-logo" aria-label="LLamaSharp Documentation" data-md-component="logo">
      
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"/></svg>

    </a>
    LLamaSharp Documentation
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          Overview
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="." class="md-nav__link md-nav__link--active">
        Overview
      </a>
      
        
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#main-features" class="md-nav__link">
    Main features
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#essential-insights-for-novice-learners" class="md-nav__link">
    Essential insights for novice learners
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#welcome-to-join-the-development" class="md-nav__link">
    Welcome to join the development!
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

    <li class="md-nav__item">
      <a href="GetStarted/" class="md-nav__link">
        Get Started
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Architecture/" class="md-nav__link">
        Architecture
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Tricks/" class="md-nav__link">
        Tricks for FAQ
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="ContributingGuide/" class="md-nav__link">
        Contributing Guide
      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
      
      
        <label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
          LLamaContext
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_6">
          <span class="md-nav__icon md-icon"></span>
          LLamaContext
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="LLamaContext/parameters.md" class="md-nav__link">
        Context Parameters
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaContext/tokenization.md" class="md-nav__link">
        Tokenization
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaContext/embeddings.md" class="md-nav__link">
        Get Embeddings
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaContext/quantization.md" class="md-nav__link">
        Quantization
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaContext/save-load-state.md" class="md-nav__link">
        Save/Load State
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" >
      
      
        <label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
          LLamaExecutors
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_7">
          <span class="md-nav__icon md-icon"></span>
          LLamaExecutors
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="LLamaExecutors/parameters/" class="md-nav__link">
        Inference Parameters
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaExecutors/text-to-text-apis/" class="md-nav__link">
        Text-to-Text APIs
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaExecutors/save-load-state/" class="md-nav__link">
        Save/Load State
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="LLamaExecutors/differences/" class="md-nav__link">
        Differences of Executors
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
      
      
        <label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
          ChatSession
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_8">
          <span class="md-nav__icon md-icon"></span>
          ChatSession
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="ChatSession/basic-usages/" class="md-nav__link">
        Basic Usages
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="ChatSession/transforms/" class="md-nav__link">
        Transoforms
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="ChatSession/save-load-session/" class="md-nav__link">
        Save/Load Session
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
      
      
        <label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="0">
          Non-English Usages
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_9">
          <span class="md-nav__icon md-icon"></span>
          Non-English Usages
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="NonEnglishUsage/Chinese/" class="md-nav__link">
        Chinese
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
      
      
        <label class="md-nav__link" for="__nav_10" id="__nav_10_label" tabindex="0">
          High-level Applications
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_10">
          <span class="md-nav__icon md-icon"></span>
          High-level Applications
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="HighLevelApps/bot-sharp/" class="md-nav__link">
        BotSharp
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="HighLevelApps/semantic-kernel/" class="md-nav__link">
        semantic-kernel
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
      
      
        <label class="md-nav__link" for="__nav_11" id="__nav_11_label" tabindex="0">
          More
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_11">
          <span class="md-nav__icon md-icon"></span>
          More
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="More/log/" class="md-nav__link">
        Logger
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_12" >
      
      
        <label class="md-nav__link" for="__nav_12" id="__nav_12_label" tabindex="0">
          Examples
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_12_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_12">
          <span class="md-nav__icon md-icon"></span>
          Examples
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="Examples/ChatSessionStripRoleName/" class="md-nav__link">
        Chat session 1
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/ChatSessionWithRoleName/" class="md-nav__link">
        Chat session 2
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/GetEmbeddings/" class="md-nav__link">
        Get embeddings
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/InstructModeExecute/" class="md-nav__link">
        Instruct executor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/InteractiveModeExecute/" class="md-nav__link">
        Interactive executor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/StatelessModeExecute/" class="md-nav__link">
        Stateless exeutor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/LoadAndSaveSession/" class="md-nav__link">
        Load/Save session
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/LoadAndSaveState/" class="md-nav__link">
        Load/Save state
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="Examples/QuantizeModel/" class="md-nav__link">
        Quantize model
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
      
      <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_13" >
      
      
        <label class="md-nav__link" for="__nav_13" id="__nav_13_label" tabindex="0">
          API Reference
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_13_label" aria-expanded="false">
        <label class="md-nav__title" for="__nav_13">
          <span class="md-nav__icon md-icon"></span>
          API Reference
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
    <li class="md-nav__item">
      <a href="xmldocs/" class="md-nav__link">
        index
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.abstractions.ihistorytransform/" class="md-nav__link">
        llama.abstractions.ihistorytransform
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.abstractions.iinferenceparams/" class="md-nav__link">
        llama.abstractions.iinferenceparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.abstractions.illamaexecutor/" class="md-nav__link">
        llama.abstractions.illamaexecutor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.abstractions.imodelparams/" class="md-nav__link">
        llama.abstractions.imodelparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.abstractions.itextstreamtransform/" class="md-nav__link">
        llama.abstractions.itextstreamtransform
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.abstractions.itexttransform/" class="md-nav__link">
        llama.abstractions.itexttransform
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.chatsession/" class="md-nav__link">
        llama.chatsession
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.authorrole/" class="md-nav__link">
        llama.common.authorrole
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.chathistory/" class="md-nav__link">
        llama.common.chathistory
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.fixedsizequeue-1/" class="md-nav__link">
        llama.common.fixedsizequeue-1
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.illamalogger/" class="md-nav__link">
        llama.common.illamalogger
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.inferenceparams/" class="md-nav__link">
        llama.common.inferenceparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.llamadefaultlogger/" class="md-nav__link">
        llama.common.llamadefaultlogger
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.mirostattype/" class="md-nav__link">
        llama.common.mirostattype
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.common.modelparams/" class="md-nav__link">
        llama.common.modelparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarexpectedname/" class="md-nav__link">
        llama.exceptions.grammarexpectedname
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarexpectednext/" class="md-nav__link">
        llama.exceptions.grammarexpectednext
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarexpectedprevious/" class="md-nav__link">
        llama.exceptions.grammarexpectedprevious
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarformatexception/" class="md-nav__link">
        llama.exceptions.grammarformatexception
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarunexpectedcharaltelement/" class="md-nav__link">
        llama.exceptions.grammarunexpectedcharaltelement
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarunexpectedcharrngelement/" class="md-nav__link">
        llama.exceptions.grammarunexpectedcharrngelement
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarunexpectedendelement/" class="md-nav__link">
        llama.exceptions.grammarunexpectedendelement
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarunexpectedendofinput/" class="md-nav__link">
        llama.exceptions.grammarunexpectedendofinput
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarunexpectedhexcharscount/" class="md-nav__link">
        llama.exceptions.grammarunexpectedhexcharscount
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.grammarunknownescapecharacter/" class="md-nav__link">
        llama.exceptions.grammarunknownescapecharacter
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.exceptions.runtimeerror/" class="md-nav__link">
        llama.exceptions.runtimeerror
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.extensions.imodelparamsextensions/" class="md-nav__link">
        llama.extensions.imodelparamsextensions
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.extensions.keyvaluepairextensions/" class="md-nav__link">
        llama.extensions.keyvaluepairextensions
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.grammars.grammar/" class="md-nav__link">
        llama.grammars.grammar
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.grammars.grammarrule/" class="md-nav__link">
        llama.grammars.grammarrule
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.instructexecutor/" class="md-nav__link">
        llama.instructexecutor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.interactiveexecutor/" class="md-nav__link">
        llama.interactiveexecutor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.llamacontext/" class="md-nav__link">
        llama.llamacontext
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.llamaembedder/" class="md-nav__link">
        llama.llamaembedder
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.llamaquantizer/" class="md-nav__link">
        llama.llamaquantizer
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.llamatransforms/" class="md-nav__link">
        llama.llamatransforms
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.llamaweights/" class="md-nav__link">
        llama.llamaweights
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamacontextparams/" class="md-nav__link">
        llama.native.llamacontextparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamaftype/" class="md-nav__link">
        llama.native.llamaftype
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamagrammarelement/" class="md-nav__link">
        llama.native.llamagrammarelement
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamagrammarelementtype/" class="md-nav__link">
        llama.native.llamagrammarelementtype
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamamodelquantizeparams/" class="md-nav__link">
        llama.native.llamamodelquantizeparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamatokendata/" class="md-nav__link">
        llama.native.llamatokendata
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamatokendataarray/" class="md-nav__link">
        llama.native.llamatokendataarray
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.llamatokendataarraynative/" class="md-nav__link">
        llama.native.llamatokendataarraynative
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.nativeapi/" class="md-nav__link">
        llama.native.nativeapi
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.safellamacontexthandle/" class="md-nav__link">
        llama.native.safellamacontexthandle
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.safellamagrammarhandle/" class="md-nav__link">
        llama.native.safellamagrammarhandle
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.safellamahandlebase/" class="md-nav__link">
        llama.native.safellamahandlebase
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.safellamamodelhandle/" class="md-nav__link">
        llama.native.safellamamodelhandle
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.native.samplingapi/" class="md-nav__link">
        llama.native.samplingapi
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatcompletion/" class="md-nav__link">
        llama.oldversion.chatcompletion
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatcompletionchoice/" class="md-nav__link">
        llama.oldversion.chatcompletionchoice
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatcompletionchunk/" class="md-nav__link">
        llama.oldversion.chatcompletionchunk
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatcompletionchunkchoice/" class="md-nav__link">
        llama.oldversion.chatcompletionchunkchoice
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatcompletionchunkdelta/" class="md-nav__link">
        llama.oldversion.chatcompletionchunkdelta
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatcompletionmessage/" class="md-nav__link">
        llama.oldversion.chatcompletionmessage
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatmessagerecord/" class="md-nav__link">
        llama.oldversion.chatmessagerecord
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatrole/" class="md-nav__link">
        llama.oldversion.chatrole
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.chatsession-1/" class="md-nav__link">
        llama.oldversion.chatsession-1
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.completion/" class="md-nav__link">
        llama.oldversion.completion
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.completionchoice/" class="md-nav__link">
        llama.oldversion.completionchoice
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.completionchunk/" class="md-nav__link">
        llama.oldversion.completionchunk
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.completionlogprobs/" class="md-nav__link">
        llama.oldversion.completionlogprobs
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.completionusage/" class="md-nav__link">
        llama.oldversion.completionusage
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.embedding/" class="md-nav__link">
        llama.oldversion.embedding
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.embeddingdata/" class="md-nav__link">
        llama.oldversion.embeddingdata
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.embeddingusage/" class="md-nav__link">
        llama.oldversion.embeddingusage
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.ichatmodel/" class="md-nav__link">
        llama.oldversion.ichatmodel
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.llamaembedder/" class="md-nav__link">
        llama.oldversion.llamaembedder
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.llamamodel/" class="md-nav__link">
        llama.oldversion.llamamodel
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.oldversion.llamaparams/" class="md-nav__link">
        llama.oldversion.llamaparams
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.statefulexecutorbase/" class="md-nav__link">
        llama.statefulexecutorbase
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.statelessexecutor/" class="md-nav__link">
        llama.statelessexecutor
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="xmldocs/llama.utils/" class="md-nav__link">
        llama.utils
      </a>
    </li>
  

        </ul>
      </nav>
    </li>
  

  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#main-features" class="md-nav__link">
    Main features
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#essential-insights-for-novice-learners" class="md-nav__link">
    Essential insights for novice learners
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#welcome-to-join-the-development" class="md-nav__link">
    Welcome to join the development!
  </a>
  
</li>
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
            <div class="md-content" data-md-component="content">
              <article class="md-content__inner md-typeset">
                
                  
<h1 id="overview">Overview</h1>
<p><img alt="logo" src="media/LLamaSharpLogo.png" /></p>
<p>LLamaSharp is the C#/.NET binding of <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a>. It provides APIs to inference the LLaMa Models and deploy it on native environment or Web. It could help C# developers to deploy the LLM (Large Language Model) locally and integrate with C# apps.</p>
<h2 id="main-features">Main features</h2>
<ul>
<li>Model inference</li>
<li>Model quantization</li>
<li>Generating embeddings</li>
<li>Grammar parse</li>
<li>Interactive/Instruct/Stateless executor mode</li>
<li>Chat session APIs</li>
<li>Save/load the state</li>
<li>Integration with other applications like BotSharp and semantic-kernel</li>
</ul>
<h2 id="essential-insights-for-novice-learners">Essential insights for novice learners</h2>
<p>If you are new to LLM, here're some tips for you to help you to get start with <code>LLamaSharp</code>. If you are experienced in this field, we'd still recommend you to take a few minutes to read it because some things perform differently compared to cpp/python.</p>
<ol>
<li>The main ability of LLamaSharp is to provide an efficient way to run inference of LLM (Large Language Model) locally (and fine-tune model in the future). The model weights, however, need to be downloaded from other resources such as <a href="https://huggingface.co">huggingface</a>.</li>
<li>Since LLamaSharp supports multiple platforms, The nuget package is split into <code>LLamaSharp</code> and <code>LLama.Backend</code>. After installing <code>LLamaSharp</code>, please install one of <code>LLama.Backend.Cpu</code>, <code>LLama.Backend.Cuda11</code> or <code>LLama.Backend.Cuda12</code>. If you use the source code, dynamic libraries can be found in <code>LLama/Runtimes</code>. Rename the one you want to use to <code>libllama.dll</code>.</li>
<li><code>LLaMa</code> originally refers to the weights released by Meta (Facebook Research). After that, many models are fine-tuned based on it, such as <code>Vicuna</code>, <code>GPT4All</code>, and <code>Pyglion</code>. Though all of these models are supported by LLamaSharp, some steps are necessary with different file formats. There're mainly three kinds of files, which are <code>.pth</code>, <code>.bin (ggml)</code>, <code>.bin (quantized)</code>. If you have the <code>.bin (quantized)</code> file, it could be used directly by LLamaSharp. If you have the <code>.bin (ggml)</code> file, you could use it directly but get higher inference speed after the quantization. If you have the <code>.pth</code> file, you need to follow <a href="https://github.com/ggerganov/llama.cpp#prepare-data--run">the instructions in llama.cpp</a> to convert it to <code>.bin (ggml)</code> file at first.</li>
<li>LLamaSharp supports GPU acceleration, but it requires cuda installation. Please install cuda 11 or cuda 12 on your system before using LLamaSharp to enable GPU. If you have another cuda version, you could compile llama.cpp from source to get the dll. For building from source, please refer to <a href="https://github.com/SciSharp/LLamaSharp/issues/5">issue #5</a>.</li>
</ol>
<h2 id="welcome-to-join-the-development">Welcome to join the development!</h2>
<p>Community effort is always one of the most important things in open-source projects. Any contribution in any way is welcomed here. For example, the following things mean a lot for LLamaSharp:</p>
<ol>
<li>Open an issue when you find something wrong.</li>
<li>Open an PR if you've fixed something. Even if just correcting a typo, it also makes great sense.</li>
<li>Help to optimize the documentation. </li>
<li>Write an example or blog about how to integrate LLamaSharp with your APPs.</li>
<li>Ask for a missed feature and discuss with other developers.</li>
</ol>
<p>If you'd like to get deeply involved in development, please touch us in discord channel or send email to <code>AsakusaRinne@gmail.com</code>. :)</p>


              </article>
            </div>
          
          
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    <script id="__config" type="application/json">{"base": ".", "features": [], "search": "assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": {"provider": "mike"}}</script>
    
    
      <script src="assets/javascripts/bundle.a51614de.min.js"></script>
      
    
  </body>
</html>