huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-21 23:12:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc(offline): match rework

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-10-31 17:52:18 +01:00
parent d52b4c4978
commit 3af2c6837c
1 changed files with 2 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backends/llamacpp/offline/main.cpp
View File

@ -18,9 +18,7 @@ int main(int argc, char **argv) {
}
spdlog::set_level(spdlog::level::debug);
const auto prompt = "My name is Morgan";
const auto modelPath = absolute(std::filesystem::path(argv[1]));
const auto params = llama_model_default_params();
auto *model = llama_load_model_from_file(modelPath.c_str(), params);
@ -28,7 +26,7 @@ int main(int argc, char **argv) {
auto backend = single_worker_backend_t(model, {});
// generate
const auto promptTokens = {128000, 9906, 856, 836, 374, 23809, 128001};
const auto promptTokens = {128000, 5159, 836, 374, 23809, 11};
const auto out = backend.generate(promptTokens, {.max_new_tokens = 32}, {.top_k = 40});
if (out.has_value())