vocab : fix bug (eos -> bos)

ggerganov · ggerganov · commit c89e80854738 · 2025-01-09T22:40:28.000+02:00
ggml-ci
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -426,7 +426,7 @@ struct llm_tokenizer_bpe_session {
     }
 
     bool append_eos(std::vector<llama_token> & output) const {
-        if (vocab.add_bos_token()) {
+        if (vocab.add_eos_token()) {
             GGML_ASSERT(vocab.token_eos() != LLAMA_TOKEN_NULL);
             output.push_back(vocab.token_eos());
             return true;

Original file line number	Diff line number	Diff line change
`@@ -426,7 +426,7 @@ struct llm_tokenizer_bpe_session {`
`426`	`426`	`}`
`427`	`427`
`428`	`428`	`bool append_eos(std::vector<llama_token> & output) const {`
`429`		`- if (vocab.add_bos_token()) {`
	`429`	`+ if (vocab.add_eos_token()) {`
`430`	`430`	`GGML_ASSERT(vocab.token_eos() != LLAMA_TOKEN_NULL);`
`431`	`431`	`output.push_back(vocab.token_eos());`
`432`	`432`	`return true;`