Skip to content

Commit 65e8991

Browse files
committed
cont : fix check for number of cached prompts [no ci]
1 parent 05ba850 commit 65e8991

File tree

1 file changed

+16
-18
lines changed

1 file changed

+16
-18
lines changed

tools/server/server.cpp

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1462,30 +1462,28 @@ struct server_prompt_cache {
14621462
}
14631463

14641464
void update() {
1465-
// always keep at least one state, regardless of the limits
1466-
if (states.size() > 1) {
1467-
if (limit_size > 0) {
1468-
while (size() > limit_size) {
1469-
if (states.empty()) {
1470-
break;
1471-
}
1465+
if (limit_size > 0) {
1466+
// always keep at least one state, regardless of the limits
1467+
while (states.size() > 1 && size() > limit_size) {
1468+
if (states.empty()) {
1469+
break;
1470+
}
14721471

1473-
SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
1472+
SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
14741473

1475-
states.pop_front();
1476-
}
1474+
states.pop_front();
14771475
}
1476+
}
14781477

1479-
if (limit_tokens > 0) {
1480-
while (n_tokens() > limit_tokens) {
1481-
if (states.empty()) {
1482-
break;
1483-
}
1478+
if (limit_tokens > 0) {
1479+
while (states.size() > 1 && n_tokens() > limit_tokens) {
1480+
if (states.empty()) {
1481+
break;
1482+
}
14841483

1485-
SRV_WRN(" - cache token limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
1484+
SRV_WRN(" - cache token limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
14861485

1487-
states.pop_front();
1488-
}
1486+
states.pop_front();
14891487
}
14901488
}
14911489

0 commit comments

Comments
 (0)