<li class="md-nav__item">

<a href="#llama_cpp.llama.Llama.generate" class="md-nav__link">

generate()

</a>

<li class="md-nav__item">

<a href="#llama_cpp.llama.Llama.generate" class="md-nav__link">

generate()

</a>

<h3 id="llama_cpp.llama.Llama.generate" class="doc doc-heading">

<code class="highlight language-python"><span class="n">generate</span><span class="p">(</span><span class="n">tokens</span><span class="p">,</span> <span class="n">top_k</span><span class="p">,</span> <span class="n">top_p</span><span class="p">,</span> <span class="n">temp</span><span class="p">,</span> <span class="n">repeat_penalty</span><span class="p">)</span></code>

<div class="doc doc-contents ">

<details class="quote">

<summary>Source code in <code>llama_cpp/llama.py</code></summary>

<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-118">118</a></span>

<span class="normal"><a href="#__codelineno-0-119">119</a></span>

<span class="normal"><a href="#__codelineno-0-120">120</a></span>

<span class="normal"><a href="#__codelineno-0-121">121</a></span>

<span class="normal"><a href="#__codelineno-0-122">122</a></span>

<span class="normal"><a href="#__codelineno-0-123">123</a></span>

<span class="normal"><a href="#__codelineno-0-124">124</a></span>

<span class="normal"><a href="#__codelineno-0-125">125</a></span>

<span class="normal"><a href="#__codelineno-0-126">126</a></span>

<span class="normal"><a href="#__codelineno-0-127">127</a></span>

<span class="normal"><a href="#__codelineno-0-128">128</a></span>

<span class="normal"><a href="#__codelineno-0-129">129</a></span>

<span class="normal"><a href="#__codelineno-0-130">130</a></span>

<span class="normal"><a href="#__codelineno-0-131">131</a></span>

<span class="normal"><a href="#__codelineno-0-132">132</a></span>

<span class="normal"><a href="#__codelineno-0-133">133</a></span>

<span class="normal"><a href="#__codelineno-0-134">134</a></span>

<span class="normal"><a href="#__codelineno-0-135">135</a></span>

<span class="normal"><a href="#__codelineno-0-136">136</a></span>

<span class="normal"><a href="#__codelineno-0-137">137</a></span>

<span class="normal"><a href="#__codelineno-0-138">138</a></span>

<span class="normal"><a href="#__codelineno-0-139">139</a></span>

<span class="normal"><a href="#__codelineno-0-140">140</a></span>

<span class="normal"><a href="#__codelineno-0-141">141</a></span>

<span class="normal"><a href="#__codelineno-0-142">142</a></span>

<span class="normal"><a href="#__codelineno-0-143">143</a></span>

<span class="normal"><a href="#__codelineno-0-144">144</a></span>

<span class="normal"><a href="#__codelineno-0-145">145</a></span>

<span class="normal"><a href="#__codelineno-0-146">146</a></span>

<span class="normal"><a href="#__codelineno-0-147">147</a></span>

<span class="normal"><a href="#__codelineno-0-148">148</a></span>

<span class="normal"><a href="#__codelineno-0-149">149</a></span>

<span class="normal"><a href="#__codelineno-0-150">150</a></span>

<span class="normal"><a href="#__codelineno-0-151">151</a></span>

<span class="normal"><a href="#__codelineno-0-152">152</a></span>

<span class="normal"><a href="#__codelineno-0-153">153</a></span>

<span class="normal"><a href="#__codelineno-0-154">154</a></span>

<span class="normal"><a href="#__codelineno-0-155">155</a></span>

<span class="normal"><a href="#__codelineno-0-156">156</a></span>

<span class="normal"><a href="#__codelineno-0-157">157</a></span>

<span class="normal"><a href="#__codelineno-0-158">158</a></span>

<span class="normal"><a href="#__codelineno-0-159">159</a></span>

<span class="normal"><a href="#__codelineno-0-160">160</a></span>

<span class="normal"><a href="#__codelineno-0-161">161</a></span>

<span class="normal"><a href="#__codelineno-0-162">162</a></span>

<span class="normal"><a href="#__codelineno-0-163">163</a></span>

<span class="normal"><a href="#__codelineno-0-164">164</a></span>

<span class="normal"><a href="#__codelineno-0-165">165</a></span>

<span class="normal"><a href="#__codelineno-0-166">166</a></span>

<span class="normal"><a href="#__codelineno-0-167">167</a></span>

<span class="normal"><a href="#__codelineno-0-168">168</a></span>

<span class="normal"><a href="#__codelineno-0-169">169</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="k">def</span> <span class="nf">generate</span><span class="p">(</span>

</span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119"></a> <span class="bp">self</span><span class="p">,</span>

</span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120"></a> <span class="n">tokens</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_token</span><span class="p">],</span>

</span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121"></a> <span class="n">top_k</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>

</span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122"></a> <span class="n">top_p</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span>

</span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123"></a> <span class="n">temp</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span>

</span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124"></a> <span class="n">repeat_penalty</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span>

</span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Generator</span><span class="p">[</span>

</span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_token</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_token</span><span class="p">]],</span> <span class="kc">None</span>

</span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127"></a><span class="p">]:</span>

</span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="c1"># Temporary workaround for https://github.com/ggerganov/llama.cpp/issues/684</span>

</span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="k">if</span> <span class="n">temp</span> <span class="o">==</span> <span class="mf">0.0</span><span class="p">:</span>

</span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="n">temp</span> <span class="o">=</span> <span class="mf">1.0</span>

</span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="n">top_p</span> <span class="o">=</span> <span class="mf">0.0</span>

</span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132"></a> <span class="n">top_k</span> <span class="o">=</span> <span class="mi">1</span>

</span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">ctx</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>

</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">n_ctx</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_n_ctx</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ctx</span><span class="p">))</span>

</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135"></a> <span class="n">n_tokens</span> <span class="o">=</span> <span class="mi">0</span>

</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">last_n_tokens</span> <span class="o">=</span> <span class="n">deque</span><span class="p">(</span>

</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="p">[</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_token</span><span class="p">(</span><span class="mi">0</span><span class="p">)]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_n_tokens_size</span><span class="p">,</span>

</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">maxlen</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">last_n_tokens_size</span><span class="p">,</span>

</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="p">)</span>

</span><span id="__span-0-140"><a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>

</span><span id="__span-0-141"><a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">tokens</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_batch</span><span class="p">):</span>

</span><span id="__span-0-142"><a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">batch</span> <span class="o">=</span> <span class="n">tokens</span><span class="p">[</span><span class="n">i</span> <span class="p">:</span> <span class="nb">min</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">tokens</span><span class="p">),</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_batch</span><span class="p">)]</span>

</span><span id="__span-0-143"><a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">n_past</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">n_ctx</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="p">),</span> <span class="n">n_tokens</span><span class="p">)</span>

</span><span id="__span-0-144"><a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">return_code</span> <span class="o">=</span> <span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_eval</span><span class="p">(</span>

</span><span id="__span-0-145"><a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="n">ctx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ctx</span><span class="p">,</span>

</span><span id="__span-0-146"><a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="n">tokens</span><span class="o">=</span><span class="p">(</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_token</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="p">))(</span><span class="o">*</span><span class="n">batch</span><span class="p">),</span>

</span><span id="__span-0-147"><a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="n">n_tokens</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_int</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="p">)),</span>

</span><span id="__span-0-148"><a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="n">n_past</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_int</span><span class="p">(</span><span class="n">n_past</span><span class="p">),</span>

</span><span id="__span-0-149"><a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="n">n_threads</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_threads</span><span class="p">),</span>

</span><span id="__span-0-150"><a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="p">)</span>

</span><span id="__span-0-151"><a id="__codelineno-0-151" name="__codelineno-0-151"></a> <span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">return_code</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>

</span><span id="__span-0-152"><a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;llama_eval returned </span><span class="si">{</span><span class="n">return_code</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>

</span><span id="__span-0-153"><a id="__codelineno-0-153" name="__codelineno-0-153"></a> <span class="n">last_n_tokens</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>

</span><span id="__span-0-154"><a id="__codelineno-0-154" name="__codelineno-0-154"></a> <span class="n">n_tokens</span> <span class="o">+=</span> <span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>

</span><span id="__span-0-155"><a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="n">token</span> <span class="o">=</span> <span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_sample_top_p_top_k</span><span class="p">(</span>

</span><span id="__span-0-156"><a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">ctx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ctx</span><span class="p">,</span>

</span><span id="__span-0-157"><a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">last_n_tokens_data</span><span class="o">=</span><span class="p">(</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">llama_token</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_n_tokens_size</span><span class="p">)(</span>

</span><span id="__span-0-158"><a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="o">*</span><span class="n">last_n_tokens</span>

</span><span id="__span-0-159"><a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="p">),</span>

</span><span id="__span-0-160"><a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">last_n_tokens_size</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_n_tokens_size</span><span class="p">),</span>

</span><span id="__span-0-161"><a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="n">top_k</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_int</span><span class="p">(</span><span class="n">top_k</span><span class="p">),</span>

</span><span id="__span-0-162"><a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">top_p</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_float</span><span class="p">(</span><span class="n">top_p</span><span class="p">),</span>

</span><span id="__span-0-163"><a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="n">temp</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_float</span><span class="p">(</span><span class="n">temp</span><span class="p">),</span>

</span><span id="__span-0-164"><a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="n">repeat_penalty</span><span class="o">=</span><span class="n">llama_cpp</span><span class="o">.</span><span class="n">c_float</span><span class="p">(</span><span class="n">repeat_penalty</span><span class="p">),</span>

</span><span id="__span-0-165"><a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="p">)</span>

</span><span id="__span-0-166"><a id="__codelineno-0-166" name="__codelineno-0-166"></a> <span class="n">tokens_or_none</span> <span class="o">=</span> <span class="k">yield</span> <span class="n">token</span>

</span><span id="__span-0-167"><a id="__codelineno-0-167" name="__codelineno-0-167"></a> <span class="n">tokens</span> <span class="o">=</span> <span class="p">[</span><span class="n">token</span><span class="p">]</span>

</span><span id="__span-0-168"><a id="__codelineno-0-168" name="__codelineno-0-168"></a> <span class="k">if</span> <span class="n">tokens_or_none</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>

</span><span id="__span-0-169"><a id="__codelineno-0-169" name="__codelineno-0-169"></a> <span class="n">tokens</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">tokens_or_none</span><span class="p">)</span>

</details>

</div>

<div class="doc doc-object doc-function">