niansa/libjustlm
1
0
Fork 0
mirror of https://gitlab.com/niansa/libjustlm.git synced 2025-03-06 20:49:17 +01:00
Code Releases Activity

Added GPU support

Browse source
This commit is contained in:
niansa 2023-06-10 00:49:21 +02:00
parent e2f7da65e4
commit 0199db02b7
2 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/justlm.hpp
View file

@ -93,6 +93,7 @@ public:
float mirostat_target_entropy = 5.0f; // mirostat specific
float repeat_penalty = 1.0f;
unsigned n_gpu_layers = 38;
bool use_mlock = true; // llama specific
int prefer_mirostat = 0; // Use given mirostat version if available (see is_mirostat_available()); llama specific
} params;

3
justlm_llama.hpp
View file

@ -32,6 +32,9 @@ class LLaMAInference final : public Inference {
lparams.seed = params.seed;
lparams.n_ctx = params.n_ctx = params.n_ctx>0?params.n_ctx:2024;
lparams.use_mlock = params.use_mlock;
#if LLAMA_DATE >= 230519
lparams.n_gpu_layers = params.n_gpu_layers;
#endif
// Create context
state->ctx = llama_init_from_file(weights_path.c_str(), lparams);