niansa/discord_llama
1
0
Fork 0
mirror of https://gitlab.com/niansa/discord_llama.git synced 2025-03-06 20:48:25 +01:00
Code Releases Activity

Added timeout config key

Browse source
This commit is contained in:
niansa/tuxifan 2023-04-27 13:47:18 +02:00
parent 36b34ee5e7
commit 46480005d4
2 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
example_config.txt
View file

@ -16,4 +16,5 @@ persistance true
mlock false
pool_size 2
threads 4
timeout 120
ctx_size 1012

7
main.cpp
View file

@ -113,6 +113,7 @@ public:
models_dir = "models";
unsigned ctx_size = 1012,
pool_size = 2,
timeout = 120,
threads = 4,
persistance = true;
bool mlock = false,
@ -328,7 +329,7 @@ private:
Timer timeout;
bool timeout_exceeded = false;
inference.append(prefix+std::string(llm_translate_to_en(line, channel_cfg.model_config->no_translate))+'\n', [&] (float progress) {
if (timeout.get<std::chrono::minutes>() > 1) {
if (timeout.get<std::chrono::seconds>() > config.timeout) {
std::cerr << "\nWarning: Timeout exceeded processing message" << std::endl;
timeout_exceeded = true;
return false;
@ -363,7 +364,7 @@ private:
msg.content.clear();
auto output = inference.run(channel_cfg.instruct_mode?channel_cfg.model_config->user_prompt:"\n", [&] (std::string_view token) {
std::cout << token << std::flush;
if (timeout.get<std::chrono::minutes>() > 2) {
if (timeout.get<std::chrono::seconds>() > config.timeout) {
timeout_exceeded = true;
std::cerr << "\nWarning: Timeout exceeded generating message";
return false;
@ -658,6 +659,8 @@ int main(int argc, char **argv) {
cfg.pool_size = std::stoi(value);
} else if (key == "threads") {
cfg.threads = std::stoi(value);
} else if (key == "timeout") {
cfg.timeout = std::stoi(value);
} else if (key == "ctx_size") {
cfg.ctx_size = std::stoi(value);
} else if (key == "mlock") {