mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-05-28 01:32:07 +00:00
17367438f3
* test(ctest) enable address sanitizer * feat(trtllm): expose finish reason to Rust * feat(trtllm): fix logits retrieval * misc(ci): enabe building tensorrt-llm * misc(ci): update Rust action toolchain * misc(ci): let's try to build the Dockerfile for trtllm # Conflicts: # Dockerfile_trtllm * misc(ci): provide mecanism to cache inside container * misc(ci): export aws creds as output of step * misc(ci): let's try this way * misc(ci): again * misc(ci): again * misc(ci): add debug profile * misc(ci): add debug profile * misc(ci): lets actually use sccache ... * misc(ci): do not build with ssl enabled * misc(ci): WAT * misc(ci): WAT * misc(ci): WAT * misc(ci): WAT * misc(ci): WAT * misc(backend): test with TGI S3 conf * misc(backend): test with TGI S3 conf * misc(backend): once more? * misc(backend): let's try with GHA * misc(backend): missing env directive * misc(backend): make sure to correctly set IS_GHA_BUILD=true in wf * misc(backend): ok let's debug smtg * misc(backend): WWWWWWWWWWWWWAAAAAAAA * misc(backend): kthxbye retry s3 * misc(backend): use session token * misc(backend): add more info * misc(backend): lets try 1h30 * misc(backend): lets try 1h30 * misc(backend): increase to 2h * misc(backend): lets try... * misc(backend): lets try... * misc(backend): let's build for ci-runtime * misc(backend): let's add some more tooling * misc(backend): add some tags * misc(backend): disable Werror for now * misc(backend): added automatic gha detection * misc(backend): remove leak sanitizer which is included in asan * misc(backend): forward env * misc(backend): forward env * misc(backend): let's try * misc(backend): let's try * misc(backend): again * misc(backend): again * misc(backend): again * misc(backend): again * misc(backend): again * misc(backend): fix sscache -> sccache * misc(backend): fix sscache -> sccache * misc(backend): fix sscache -> sccache * misc(backend): let's actually cache things now * misc(backend): let's actually cache things now * misc(backend): attempt to run the testS? * misc(backend): attempt to run the tests? * misc(backend): attempt to run the tests? * change runner size * fix: Correctly tag docker images (#2878) * fix: Correctly tag docker images * fix: Correctly tag docker images * misc(llamacpp): maybe? * misc(llamacpp): maybe? * misc(llamacpp): maybe? * misc(ci): gogogo * misc(ci): gogogo * misc(ci): gogogo * misc(ci): gogogo * misc(ci): gogogo * misc(ci): gogogo * misc(ci): go * misc(ci): go * misc(ci): go * misc(ci): use bin folder * misc(ci): make the wf callable for reuse * misc(ci): make the wf callable for reuse (bis) * misc(ci): make the wf callable for reuse (bis) * misc(ci): give the wf a name * Create test-trtllm.yml * Update test-trtllm.yml * Create build-trtllm2 * Rename build-trtllm2 to 1-build-trtllm2 * Rename test-trtllm.yml to 1-test-trtllm2.yml * misc(ci): fw secrets * Update 1-test-trtllm2.yml * Rename 1-build-trtllm2 to 1-build-trtllm2.yml * Update 1-test-trtllm2.yml * misc(ci): use ci-build.yaml as main dispatcher * Delete .github/workflows/1-test-trtllm2.yml * Delete .github/workflows/1-build-trtllm2.yml * misc(ci): rights? * misc(ci): rights? * misc(ci): once more? * misc(ci): once more? * misc(ci): baby more time? * misc(ci): baby more time? * misc(ci): try the permission above again? * misc(ci): try the permission above again? * misc(ci): try the permission scoped again? * misc(ci): install tensorrt_llm_executor_static * misc(ci): attempt to rebuild with sccache? * misc(ci):run the tests on GPU instance * misc(ci): let's actually setup sccache in the build.rs * misc(ci): reintroduce variables * misc(ci): enforce sccache * misc(ci): correct right job name dependency * misc(ci): detect dev profile for debug * misc(ci): detect gha build * misc(ci): detect gha build * misc(ci): ok debug * misc(ci): wtf * misc(ci): wtf2 * misc(ci): wtf3 * misc(ci): use commit HEAD instead of merge commit for image id * misc(ci): wtfinfini * misc(ci): wtfinfini * misc(ci): KAMEHAMEHA * Merge TRTLLM in standard CI * misc(ci): remove input machine * misc(ci): missing id-token for AWS auth * misc(ci): missing id-token for AWS auth * misc(ci): missing id-token for AWS auth * misc(ci): again... * misc(ci): again... * misc(ci): again... * misc(ci): again... * misc(ci): missing benchmark * misc(ci): missing backends * misc(ci): missing launcher * misc(ci): give everything aws needs * misc(ci): give everything aws needs * misc(ci): fix warnings * misc(ci): attempt to fix sccache not building trtllm * misc(ci): attempt to fix sccache not building trtllm again --------- Co-authored-by: Guillaume LEGENDRE <glegendre01@gmail.com> Co-authored-by: Hugo Larcher <hugo.larcher@huggingface.co> Co-authored-by: Pauline Bailly-Masson <155966238+paulinebm@users.noreply.github.com>
132 lines
6.2 KiB
CMake
132 lines
6.2 KiB
CMake
cmake_minimum_required(VERSION 3.20)
|
|
|
|
if (NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER)
|
|
find_program(CCACHE_EXECUTABLE "ccache")
|
|
if (CCACHE_EXECUTABLE)
|
|
message(STATUS "Using ccache")
|
|
set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}")
|
|
set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}")
|
|
set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}")
|
|
endif ()
|
|
else ()
|
|
message(STATUS "Using user specified cmake cxx compiler launcher: ${CMAKE_CXX_COMPILER_LAUNCHER}")
|
|
set(CMAKE_C_COMPILER_LAUNCHER "${CMAKE_CXX_COMPILER_LAUNCHER}")
|
|
set(CMAKE_CXX_COMPILER_LAUNCHER "${CMAKE_CXX_COMPILER_LAUNCHER}")
|
|
set(CMAKE_CUDA_COMPILER_LAUNCHER "${CMAKE_CXX_COMPILER_LAUNCHER}")
|
|
endif ()
|
|
|
|
if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
|
|
cmake_policy(SET CMP0135 NEW)
|
|
endif ()
|
|
|
|
project(tgi-trtllm-backend VERSION 1.0.0)
|
|
set(CMAKE_CXX_STANDARD 23)
|
|
|
|
include(FetchContent)
|
|
include(ExternalProject)
|
|
include(CheckCXXCompilerFlag)
|
|
|
|
option(TGI_TRTLLM_BACKEND_BUILD_TESTS "Enable building the unittests suite" OFF)
|
|
option(TGI_TRTLLM_BACKEND_BUILD_EXAMPLES "Enable building the examples suite" OFF)
|
|
option(TGI_TRTLLM_BACKEND_BUILD_USE_LLD "Enable lld linker instead of ld" OFF)
|
|
set(TGI_TRTLLM_BACKEND_TARGET_CUDA_ARCH_LIST "89-real" CACHE STRING "List of CUDA architectures to support")
|
|
set(TGI_TRTLLM_BACKEND_TRT_ROOT "/usr/local/tensorrt" CACHE STRING "Path rgo where TensorRT libraries and headers are located")
|
|
set(TGI_TRTLLM_BACKEND_TRT_INCLUDE_DIR "${TGI_TRTLLM_BACKEND_TRT_ROOT}/include" CACHE STRING "Path where TensorRT headers are located")
|
|
set(TGI_TRTLLM_BACKEND_TRT_LIB_DIR "${TGI_TRTLLM_BACKEND_TRT_ROOT}/lib" CACHE STRING "Path where TensorRT libraries are located")
|
|
|
|
# We are using nvidia-ml to query at runtime device information to enable some architecture-specific features
|
|
find_package(CUDAToolkit 12.6 REQUIRED COMPONENTS CUDA::cudart CUDA::nvml)
|
|
find_package(MPI REQUIRED)
|
|
|
|
#### External dependencies ####
|
|
include(cmake/json.cmake)
|
|
include(cmake/spdlog.cmake)
|
|
include(cmake/trtllm.cmake)
|
|
|
|
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
|
set(TGI_TRTLLM_BACKEND_DEBUG ON)
|
|
add_compile_definitions(TGI_TRTLLM_BACKEND_DEBUG=1)
|
|
add_compile_definitions(SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_TRACE)
|
|
endif ()
|
|
|
|
if (${TGI_TRTLLM_BACKEND_BUILD_USE_LLD})
|
|
message(STATUS "Using lld linker")
|
|
add_link_options("-fuse-ld=lld")
|
|
endif ()
|
|
|
|
# Let's build TRTLLM as part of CMake
|
|
add_subdirectory("${trtllm_SOURCE_DIR}/cpp" "${trtllm_SOURCE_DIR}/..")
|
|
|
|
# Tell CMake to need try to override the RPATH for executorWorker as it has not information on how to do so
|
|
set_target_properties(executorWorker PROPERTIES SKIP_BUILD_RPATH TRUE)
|
|
|
|
# TGI TRTLLM Backend definition
|
|
add_library(tgi_trtllm_backend_impl STATIC csrc/hardware.hpp csrc/backend.hpp csrc/backend.cpp)
|
|
include_directories(${TGI_TRTLLM_BACKEND_TRT_INCLUDE_DIR})
|
|
target_include_directories(tgi_trtllm_backend_impl PRIVATE
|
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/csrc>
|
|
# $<INSTALL_INTERFACE:csrc>
|
|
)
|
|
target_include_directories(tgi_trtllm_backend_impl PUBLIC "${trtllm_SOURCE_DIR}/cpp/include")
|
|
target_link_libraries(tgi_trtllm_backend_impl PRIVATE CUDA::cudart CUDA::nvml)
|
|
target_link_libraries(tgi_trtllm_backend_impl PUBLIC nlohmann_json::nlohmann_json spdlog::spdlog)
|
|
target_link_libraries(tgi_trtllm_backend_impl PRIVATE tensorrt_llm nvinfer_plugin_tensorrt_llm tensorrt_llm_nvrtc_wrapper)
|
|
|
|
# This install all the artifacts in CMAKE_INSTALL_PREFIX under include/ lib/ bin/ to make easy to link / find it back
|
|
install(TARGETS tgi_trtllm_backend_impl)
|
|
install(TARGETS tensorrt_llm nvinfer_plugin_tensorrt_llm decoder_attention executorWorker)
|
|
install(FILES ${TRTLLM_NVRTC_WRAPPER_LIBRARY_PATH} TYPE LIB)
|
|
if (NOT ${TGI_TRTLLM_BACKEND_DEBUG})
|
|
install(FILES ${TRTLLM_EXECUTOR_STATIC_LIBRARY_PATH} TYPE LIB)
|
|
endif ()
|
|
|
|
|
|
#### Unit Tests ####
|
|
if (${TGI_TRTLLM_BACKEND_BUILD_TESTS} AND CMAKE_BUILD_TYPE MATCHES "Debug")
|
|
message(STATUS "Building tests")
|
|
option(TGI_TRTLLM_BACKEND_ENABLE_ASAN "Enable AddressSanitizer")
|
|
option(TGI_TRTLLM_BACKEND_ENABLE_UBSAN "Enable UndefinedSanitizer")
|
|
|
|
FetchContent_Declare(
|
|
Catch2
|
|
URL https://github.com/catchorg/Catch2/archive/refs/tags/v3.7.1.tar.gz
|
|
)
|
|
FetchContent_MakeAvailable(Catch2)
|
|
|
|
# This attempt to detect if the compiler can emit warning if it can't apply return value optimization from a function
|
|
check_cxx_compiler_flag("-Wnrvo" COMPILER_SUPPORT_WARNING_ON_NVRO)
|
|
if (${COMPILER_SUPPORT_WARNING_ON_NVRO})
|
|
message(STATUS "Enabling non-NVRO detection")
|
|
target_compile_options(tgi_trtllm_backend_impl "-Wnvro")
|
|
endif ()
|
|
|
|
cmake_path(GET TRTLLM_NVRTC_WRAPPER_LIBRARY_PATH PARENT_PATH TRTLLM_NVRTC_WRAPPER_PARENT_LIBRARY_PATH)
|
|
message(STATUS "Adding linking path: ${TRTLLM_NVRTC_WRAPPER_PARENT_LIBRARY_PATH}")
|
|
|
|
add_executable(tgi_trtllm_backend_tests tests/test_hardware.cpp tests/test_backend.cpp)
|
|
|
|
# target_compile_options(tgi_trtllm_backend_tests PRIVATE -Werror)
|
|
target_link_directories(tgi_trtllm_backend_tests PRIVATE "${TRTLLM_NVRTC_WRAPPER_PARENT_LIBRARY_PATH}")
|
|
target_include_directories(tgi_trtllm_backend_tests PUBLIC "${trtllm_SOURCE_DIR}/cpp/include")
|
|
target_include_directories(tgi_trtllm_backend_tests PUBLIC "csrc/")
|
|
target_link_libraries(tgi_trtllm_backend_tests PRIVATE ${TRTLLM_LIBS} CUDA::cudart CUDA::nvml)
|
|
target_link_libraries(tgi_trtllm_backend_tests PUBLIC Catch2::Catch2WithMain nlohmann_json::nlohmann_json spdlog::spdlog tgi_trtllm_backend_impl)
|
|
target_link_libraries(tgi_trtllm_backend_tests PRIVATE tensorrt_llm nvinfer_plugin_tensorrt_llm tensorrt_llm_nvrtc_wrapper)
|
|
|
|
if (${TGI_TRTLLM_BACKEND_ENABLE_ASAN})
|
|
message(STATUS "Enabled AddressSanitizer")
|
|
target_link_options(tgi_trtllm_backend_tests BEFORE PUBLIC -fsanitize=address)
|
|
endif ()
|
|
|
|
if (${TGI_TRTLLM_BACKEND_ENABLE_UBSAN})
|
|
message(STATUS "Enabled UndefinedSanitizer")
|
|
target_link_options(tgi_trtllm_backend_tests BEFORE PUBLIC -fsanitize=undefined)
|
|
endif ()
|
|
|
|
install(TARGETS tgi_trtllm_backend_tests)
|
|
|
|
# list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
|
|
# include(CTest)
|
|
# include(Catch)
|
|
# catch_discover_tests(tgi_trtllm_backend_tests)
|
|
endif () |