huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 23:42:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc(offline): match rework

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-10-31 17:52:18 +01:00
parent d52b4c4978
commit 3af2c6837c
1 changed files with 2 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backends/llamacpp/offline/main.cpp
View File

@ -19,8 +19,6 @@ int main(int argc, char **argv) {
spdlog::set_level(spdlog::level::debug); spdlog::set_level(spdlog::level::debug);
const auto prompt = "My name is Morgan";
const auto modelPath = absolute(std::filesystem::path(argv[1])); const auto modelPath = absolute(std::filesystem::path(argv[1]));
const auto params = llama_model_default_params(); const auto params = llama_model_default_params();
auto *model = llama_load_model_from_file(modelPath.c_str(), params); auto *model = llama_load_model_from_file(modelPath.c_str(), params);
@ -28,7 +26,7 @@ int main(int argc, char **argv) {
auto backend = single_worker_backend_t(model, {}); auto backend = single_worker_backend_t(model, {});
// generate // generate
const auto promptTokens = {128000, 9906, 856, 836, 374, 23809, 128001}; const auto promptTokens = {128000, 5159, 836, 374, 23809, 11};
const auto out = backend.generate(promptTokens, {.max_new_tokens = 32}, {.top_k = 40}); const auto out = backend.generate(promptTokens, {.max_new_tokens = 32}, {.top_k = 40});
if (out.has_value()) if (out.has_value())