commit 1010fb3af11b62e6a6826ea1bef40ab49cc503a4 Author: Daniel Verkamp Date: Mon Sep 21 08:52:41 2015 -0700 SPDK: Initial check-in Signed-off-by: Daniel Verkamp diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..05e69e61b --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.a +*.d +*.o +*~ +*.swp +tags diff --git a/CONFIG b/CONFIG new file mode 100644 index 000000000..fb3ff6042 --- /dev/null +++ b/CONFIG @@ -0,0 +1,45 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +# Build with debug logging. Turn off for performance testing and normal usage +CONFIG_DEBUG=y + +# This directory should contain 'include' and 'lib' directories for your DPDK +# installation. Alternatively you can specify this on the command line +# with 'make DPDK_DIR=/path/to/dpdk'. +CONFIG_DPDK_DIR=/path/to/dpdk + +# Header file to use for NVMe implementation specific functions. +# Defaults to depending on DPDK. +CONFIG_NVME_IMPL=nvme_impl.h + diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..47018d3b4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +BSD LICENSE + +Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a606733a1 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(CURDIR) +export SPDK_ROOT_DIR + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y += lib test examples + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +test: lib +examples: lib + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/doc/Doxyfile.nvme b/doc/Doxyfile.nvme new file mode 100644 index 000000000..7b8fdb367 --- /dev/null +++ b/doc/Doxyfile.nvme @@ -0,0 +1,1786 @@ +# Doxyfile 1.7.6.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" "). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or sequence of words) that should +# identify the project. Note that if you do not use Doxywizard you need +# to put quotes around the project name if it contains spaces. + +PROJECT_NAME = "SPDK Userspace NVMe Driver" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer +# a quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify an logo or icon that is +# included in the documentation. The maximum height of the logo should not +# exceed 55 pixels and the maximum width should not exceed 200 pixels. +# Doxygen will copy the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = output.nvme + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful if your file system +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 8 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding +# "class=itcl::class" will allow you to use the command class in the +# itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also makes the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and +# unions are shown inside the group in which they are included (e.g. using +# @ingroup) instead of on a separate page (for HTML and Man pages) or +# section (for LaTeX and RTF). + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and +# unions with only public data fields will be shown inline in the documentation +# of the scope in which they are defined (i.e. file, namespace, or group +# documentation), provided this scope is documented. If set to NO (the default), +# structs, classes, and unions are shown on a separate page (for HTML and Man +# pages) or section (for LaTeX and RTF). + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = NO + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penalty. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will roughly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols. + +SYMBOL_CACHE_SIZE = 0 + +# Similar to the SYMBOL_CACHE_SIZE the size of the symbol lookup cache can be +# set using LOOKUP_CACHE_SIZE. This cache is used to resolve symbols given +# their name and scope. Since this can be an expensive process and often the +# same symbol appear multiple times in the code, doxygen keeps a cache of +# pre-resolved symbols. If the cache is too small doxygen will become slower. +# If the cache is too large, memory is wasted. The cache size is given by this +# formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespaces are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = YES + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to +# do proper type resolution of all parameters of a function it will reject a +# match between the prototype and the implementation of a member function even +# if there is only one candidate or it is obvious which candidate to choose +# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen +# will still accept a match between prototype and implementation in such cases. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or macro consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and macros in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files +# containing the references data. This must be a list of .bib files. The +# .bib extension is automatically appended if omitted. Using this command +# requires the bibtex tool to be installed. See also +# http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style +# of the bibliography can be controlled using LATEX_BIB_STYLE. To use this +# feature you need bibtex and perl available in the search path. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# The WARN_NO_PARAMDOC option can be enabled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ../lib/nvme nvme.index.txt + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh +# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py +# *.f90 *.f *.for *.vhd *.vhdl + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = cap_lo_register \ + cap_hi_register \ + aqa_register \ + cc_register \ + nvme_registers + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = images + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty or if +# non of the patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) +# and it is also possible to disable source filtering for a specific pattern +# using *.ext= (so without naming a filter). This option only has effect when +# FILTER_SOURCE_FILES is enabled. + +FILTER_SOURCE_PATTERNS = + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. Note that when using a custom header you are responsible +# for the proper inclusion of any scripts and style sheets that doxygen +# needs, which is dependent on the configuration options used. +# It is advised to generate a default header using "doxygen -w html +# header.html footer.html stylesheet.css YourConfigFile" and then modify +# that header. Note that the header is subject to change so you typically +# have to redo this when upgrading to a newer version of doxygen or when +# changing the value of configuration settings such as GENERATE_TREEVIEW! + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# style sheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that +# the files will be copied as-is; there are no commands or markers available. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the style sheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# +# Qt Help Project / Custom Filters. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# +# Qt Help Project / Filter Attributes. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) +# at top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. Since the tabs have the same information as the +# navigation tree you can set this option to NO if you already set +# GENERATE_TREEVIEW to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. +# Since the tree basically has the same information as the tab index you +# could consider to set DISABLE_INDEX to NO when enabling this option. + +GENERATE_TREEVIEW = NO + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values +# (range [0,1..20]) that doxygen will group on one line in the generated HTML +# documentation. Note that a value of 0 will completely suppress the enum +# values from appearing in the overview section. + +ENUM_VALUES_PER_LINE = 4 + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax +# (see http://www.mathjax.org) which uses client side Javascript for the +# rendering instead of using prerendered bitmaps. Use this if you do not +# have LaTeX installed or if you want to formulas look prettier in the HTML +# output. When enabled you also need to install MathJax separately and +# configure the path to it using the MATHJAX_RELPATH option. + +USE_MATHJAX = NO + +# When MathJax is enabled you need to specify the location relative to the +# HTML output directory using the MATHJAX_RELPATH option. The destination +# directory should contain the MathJax.js script. For instance, if the mathjax +# directory is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the +# mathjax.org site, so you can quickly see the result without installing +# MathJax, but it is strongly recommended to install a local copy of MathJax +# before deployment. + +MATHJAX_RELPATH = http://www.mathjax.org/mathjax + +# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension +# names that should be enabled during MathJax rendering. + +MATHJAX_EXTENSIONS = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvantages are that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4 + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for +# the generated latex document. The footer should contain everything after +# the last chapter. If it is left blank doxygen will generate a +# standard footer. Notice: only use this tag if you know what you are doing! + +LATEX_FOOTER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +# The LATEX_BIB_STYLE tag can be used to specify the style to use for the +# bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See +# http://en.wikipedia.org/wiki/BibTeX for more info. + +LATEX_BIB_STYLE = plain + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load style sheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# pointed to by INCLUDE_PATH will be searched when a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition that +# overrules the definition found in the source code. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all references to function-like macros +# that are alone on a line, have an all uppercase name, and do not end with a +# semicolon, because these will confuse the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option also works with HAVE_DOT disabled, but it is recommended to +# install and use dot, since it yields more powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = YES + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will use the Helvetica font for all dot files that +# doxygen generates. When you want a differently looking font you can specify +# the font name using DOT_FONTNAME. You need to make sure dot is able to find +# the font, which can be done by putting it in a standard location or by setting +# the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the Helvetica font. +# If you specify a different font using DOT_FONTNAME you can use DOT_FONTPATH to +# set the path where dot can find it. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will generate a graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are svg, png, jpg, or gif. +# If left blank png will be used. If you choose svg you need to set +# HTML_FILE_EXTENSION to xhtml in order to make the SVG files +# visible in IE 9+ (other browsers do not have this requirement). + +DOT_IMAGE_FORMAT = png + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# Note that this requires a modern browser other than Internet Explorer. +# Tested and working are Firefox, Chrome, Safari, and Opera. For IE 9+ you +# need to set HTML_FILE_EXTENSION to xhtml in order to make the SVG files +# visible. Older versions of IE do not have SVG support. + +INTERACTIVE_SVG = NO + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the +# \mscfile command). + +MSCFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 2 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..348e7f09b --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,15 @@ +DOXYFILES = Doxyfile.nvme +OUTPUT_DIRS = $(patsubst Doxyfile.%,output.%,$(DOXYFILES)) + +all: doc + +.PHONY: all doc clean + +doc: $(OUTPUT_DIRS) + +output.%: Doxyfile.% + rm -rf $@ + doxygen $^ + +clean: + rm -rf $(OUTPUT_DIRS) diff --git a/doc/nvme.index.txt b/doc/nvme.index.txt new file mode 100644 index 000000000..bad1953e0 --- /dev/null +++ b/doc/nvme.index.txt @@ -0,0 +1,58 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/*! + +\mainpage SPDK Userspace NVMe Driver + +\section interface Public Interface + +- nvme.h + +\section key_functions Key Functions + +- nvme_attach() \copybrief nvme_attach() +- nvme_ns_cmd_read() \copybrief nvme_ns_cmd_read() +- nvme_ns_cmd_write() \copybrief nvme_ns_cmd_write() +- nvme_ns_cmd_deallocate() \copybrief nvme_ns_cmd_deallocate() +- nvme_ns_cmd_flush() \copybrief nvme_ns_cmd_flush() +- nvme_ctrlr_process_io_completions() \copybrief nvme_ctrlr_process_io_completions() + +\section key_concepts Key Concepts + +- \ref nvme_driver_integration +- \ref nvme_initialization +- \ref nvme_io_submission +- \ref nvme_async_completion + +*/ diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 000000000..a5fb367aa --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,43 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y += nvme + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/examples/nvme/Makefile b/examples/nvme/Makefile new file mode 100644 index 000000000..2d7a96dc6 --- /dev/null +++ b/examples/nvme/Makefile @@ -0,0 +1,43 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y += identify perf + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/examples/nvme/identify/.gitignore b/examples/nvme/identify/.gitignore new file mode 100644 index 000000000..5c5444c1e --- /dev/null +++ b/examples/nvme/identify/.gitignore @@ -0,0 +1 @@ +identify diff --git a/examples/nvme/identify/Makefile b/examples/nvme/identify/Makefile new file mode 100644 index 000000000..5bd342731 --- /dev/null +++ b/examples/nvme/identify/Makefile @@ -0,0 +1,62 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = identify + +C_SRCS := identify.c + +CFLAGS += -I. $(DPDK_INC) + +SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \ + $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \ + $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a + +LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt + +OBJS = $(C_SRCS:.c=.o) + +all : $(APP) + +objs : $(OBJS) + +$(APP) : $(OBJS) $(SPDK_LIBS) + @echo " LINK $@" + $(Q)$(CC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + +clean : + $(Q)rm -f $(OBJS) *.d $(APP) + + + diff --git a/examples/nvme/identify/identify.c b/examples/nvme/identify/identify.c new file mode 100644 index 000000000..65d5e500c --- /dev/null +++ b/examples/nvme/identify/identify.c @@ -0,0 +1,497 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include + +#include +#include +#include +#include + +#include "spdk/nvme.h" +#include "spdk/pci.h" + +struct rte_mempool *request_mempool; + +static int outstanding_commands; + +struct feature { + uint32_t result; + bool valid; +}; + +static struct feature features[256]; + +static struct nvme_health_information_page *health_page; + +static void +get_feature_completion(void *cb_arg, const struct nvme_completion *cpl) +{ + struct feature *feature = cb_arg; + int fid = feature - features; + if (nvme_completion_is_error(cpl)) { + printf("get_feature(0x%02X) failed\n", fid); + } else { + feature->result = cpl->cdw0; + feature->valid = true; + } + outstanding_commands--; +} + +static void +get_log_page_completion(void *cb_arg, const struct nvme_completion *cpl) +{ + if (nvme_completion_is_error(cpl)) { + printf("get log page failed\n"); + } + outstanding_commands--; +} + +static int +get_feature(struct nvme_controller *ctrlr, uint8_t fid) +{ + struct nvme_command cmd = {0}; + + cmd.opc = NVME_OPC_GET_FEATURES; + cmd.cdw10 = fid; + + return nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, get_feature_completion, &features[fid]); +} + +static void +get_features(struct nvme_controller *ctrlr) +{ + int i; + + uint8_t features_to_get[] = { + NVME_FEAT_ARBITRATION, + NVME_FEAT_POWER_MANAGEMENT, + NVME_FEAT_TEMPERATURE_THRESHOLD, + NVME_FEAT_ERROR_RECOVERY, + }; + + /* Submit several GET FEATURES commands and wait for them to complete */ + outstanding_commands = 0; + for (i = 0; i < sizeof(features_to_get) / sizeof(*features_to_get); i++) { + if (get_feature(ctrlr, features_to_get[i]) == 0) { + outstanding_commands++; + } else { + printf("get_feature(0x%02X) failed to submit command\n", features_to_get[i]); + } + } + + while (outstanding_commands) { + nvme_ctrlr_process_admin_completions(ctrlr); + } +} + +static int +get_health_log_page(struct nvme_controller *ctrlr) +{ + struct nvme_command cmd = {0}; + + if (health_page == NULL) { + health_page = rte_zmalloc("nvme health", sizeof(*health_page), 4096); + } + if (health_page == NULL) { + printf("Allocation error (health page)\n"); + exit(1); + } + + cmd.opc = NVME_OPC_GET_LOG_PAGE; + cmd.cdw10 = NVME_LOG_HEALTH_INFORMATION; + cmd.cdw10 |= (sizeof(*health_page) / 4) << 16; // number of dwords + cmd.nsid = NVME_GLOBAL_NAMESPACE_TAG; + + return nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, health_page, sizeof(*health_page), + get_log_page_completion, NULL); +} + +static void +get_log_pages(struct nvme_controller *ctrlr) +{ + outstanding_commands = 0; + + if (get_health_log_page(ctrlr) == 0) { + outstanding_commands++; + } else { + printf("Get Log Page (SMART/health) failed\n"); + } + + while (outstanding_commands) { + nvme_ctrlr_process_admin_completions(ctrlr); + } +} + +static void +cleanup(void) +{ + if (health_page) { + rte_free(health_page); + health_page = NULL; + } +} + +static void +print_uint128_hex(uint64_t *v) +{ + unsigned long long lo = v[0], hi = v[1]; + if (hi) { + printf("0x%llX%016llX", hi, lo); + } else { + printf("0x%llX", lo); + } +} + +static void +print_uint128_dec(uint64_t *v) +{ + unsigned long long lo = v[0], hi = v[1]; + if (hi) { + /* can't handle large (>64-bit) decimal values for now, so fall back to hex */ + print_uint128_hex(v); + } else { + printf("%llu", (unsigned long long)lo); + } +} + +static void +print_namespace(struct nvme_namespace *ns) +{ + const struct nvme_namespace_data *nsdata; + uint32_t i; + uint32_t flags; + + nsdata = nvme_ns_get_data(ns); + flags = nvme_ns_get_flags(ns); + + printf("Namespace ID:%d\n", nvme_ns_get_id(ns)); + printf("Deallocate: %s\n", + (flags & NVME_NS_DEALLOCATE_SUPPORTED) ? "Supported" : "Not Supported"); + printf("Flush: %s\n", + (flags & NVME_NS_FLUSH_SUPPORTED) ? "Supported" : "Not Supported"); + printf("Size (in LBAs): %lld (%lldM)\n", + (long long)nsdata->nsze, + (long long)nsdata->nsze / 1024 / 1024); + printf("Capacity (in LBAs): %lld (%lldM)\n", + (long long)nsdata->ncap, + (long long)nsdata->ncap / 1024 / 1024); + printf("Utilization (in LBAs): %lld (%lldM)\n", + (long long)nsdata->nuse, + (long long)nsdata->nuse / 1024 / 1024); + printf("Thin Provisioning: %s\n", + nsdata->nsfeat.thin_prov ? "Supported" : "Not Supported"); + printf("Number of LBA Formats: %d\n", nsdata->nlbaf + 1); + printf("Current LBA Format: LBA Format #%02d\n", + nsdata->flbas.format); + for (i = 0; i <= nsdata->nlbaf; i++) + printf("LBA Format #%02d: Data Size: %5d Metadata Size: %5d\n", + i, 1 << nsdata->lbaf[i].lbads, nsdata->lbaf[i].ms); + printf("\n"); +} + +static void +print_controller(struct nvme_controller *ctrlr, struct pci_device *pci_dev) +{ + const struct nvme_controller_data *cdata; + uint8_t str[128]; + uint32_t i; + + get_features(ctrlr); + get_log_pages(ctrlr); + + cdata = nvme_ctrlr_get_data(ctrlr); + + printf("=====================================================\n"); + printf("NVMe Controller at PCI bus %d, device %d, function %d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + printf("=====================================================\n"); + printf("Controller Capabilities/Features\n"); + printf("================================\n"); + printf("Vendor ID: %04x\n", cdata->vid); + printf("Subsystem Vendor ID: %04x\n", cdata->ssvid); + snprintf(str, sizeof(cdata->sn) + 1, "%s", cdata->sn); + printf("Serial Number: %s\n", str); + snprintf(str, sizeof(cdata->mn) + 1, "%s", cdata->mn); + printf("Model Number: %s\n", str); + snprintf(str, sizeof(cdata->fr) + 1, "%s", cdata->fr); + printf("Firmware Version: %s\n", str); + printf("Recommended Arb Burst: %d\n", cdata->rab); + printf("IEEE OUI Identifier: %02x %02x %02x\n", + cdata->ieee[0], cdata->ieee[1], cdata->ieee[2]); + printf("Multi-Interface Cap: %02x\n", cdata->mic); + /* TODO: Use CAP.MPSMIN to determine true memory page size. */ + printf("Max Data Transfer Size: "); + if (cdata->mdts == 0) + printf("Unlimited\n"); + else + printf("%d\n", 4096 * (1 << cdata->mdts)); + if (features[NVME_FEAT_ERROR_RECOVERY].valid) { + unsigned tler = features[NVME_FEAT_ERROR_RECOVERY].result & 0xFFFF; + printf("Error Recovery Timeout: "); + if (tler == 0) { + printf("Unlimited\n"); + } else { + printf("%u milliseconds\n", tler * 100); + } + } + printf("\n"); + + printf("Admin Command Set Attributes\n"); + printf("============================\n"); + printf("Security Send/Receive: %s\n", + cdata->oacs.security ? "Supported" : "Not Supported"); + printf("Format NVM: %s\n", + cdata->oacs.format ? "Supported" : "Not Supported"); + printf("Firmware Activate/Download: %s\n", + cdata->oacs.firmware ? "Supported" : "Not Supported"); + printf("Abort Command Limit: %d\n", cdata->acl + 1); + printf("Async Event Request Limit: %d\n", cdata->aerl + 1); + printf("Number of Firmware Slots: "); + if (cdata->oacs.firmware != 0) + printf("%d\n", cdata->frmw.num_slots); + else + printf("N/A\n"); + printf("Firmware Slot 1 Read-Only: "); + if (cdata->oacs.firmware != 0) + printf("%s\n", cdata->frmw.slot1_ro ? "Yes" : "No"); + else + printf("N/A\n"); + printf("Per-Namespace SMART Log: %s\n", + cdata->lpa.ns_smart ? "Yes" : "No"); + printf("Error Log Page Entries: %d\n", cdata->elpe + 1); + printf("\n"); + + printf("NVM Command Set Attributes\n"); + printf("==========================\n"); + printf("Submission Queue Entry Size\n"); + printf(" Max: %d\n", 1 << cdata->sqes.max); + printf(" Min: %d\n", 1 << cdata->sqes.min); + printf("Completion Queue Entry Size\n"); + printf(" Max: %d\n", 1 << cdata->cqes.max); + printf(" Min: %d\n", 1 << cdata->cqes.min); + printf("Number of Namespaces: %d\n", cdata->nn); + printf("Compare Command: %s\n", + cdata->oncs.compare ? "Supported" : "Not Supported"); + printf("Write Uncorrectable Command: %s\n", + cdata->oncs.write_unc ? "Supported" : "Not Supported"); + printf("Dataset Management Command: %s\n", + cdata->oncs.dsm ? "Supported" : "Not Supported"); + printf("Volatile Write Cache: %s\n", + cdata->vwc.present ? "Present" : "Not Present"); + printf("\n"); + + if (features[NVME_FEAT_ARBITRATION].valid) { + uint32_t arb = features[NVME_FEAT_ARBITRATION].result; + unsigned ab, lpw, mpw, hpw; + + ab = arb & 0x3; + lpw = ((arb >> 8) & 0xFF) + 1; + mpw = ((arb >> 16) & 0xFF) + 1; + hpw = ((arb >> 24) & 0xFF) + 1; + + printf("Arbitration\n"); + printf("===========\n"); + printf("Arbitration Burst: "); + if (ab == 7) { + printf("no limit\n"); + } else { + printf("%u\n", 1u << ab); + } + printf("Low Priority Weight: %u\n", lpw); + printf("Medium Priority Weight: %u\n", mpw); + printf("High Priority Weight: %u\n", hpw); + printf("\n"); + } + + if (features[NVME_FEAT_POWER_MANAGEMENT].valid) { + unsigned ps = features[NVME_FEAT_POWER_MANAGEMENT].result & 0x1F; + printf("Power Management\n"); + printf("================\n"); + printf("Number of Power States: %u\n", cdata->npss + 1); + printf("Current Power State: Power State #%u\n", ps); + for (i = 0; i <= cdata->npss; i++) { + const struct nvme_power_state *psd = &cdata->psd[i]; + printf("Power State #%u: ", i); + if (psd->mps) { + /* MP scale is 0.0001 W */ + printf("Max Power: %u.%04u W\n", + psd->mp / 10000, + psd->mp % 10000); + } else { + /* MP scale is 0.01 W */ + printf("Max Power: %3u.%02u W\n", + psd->mp / 100, + psd->mp % 100); + } + /* TODO: print other power state descriptor fields */ + } + printf("\n"); + } + + if (features[NVME_FEAT_TEMPERATURE_THRESHOLD].valid && health_page) { + printf("Health Information\n"); + printf("==================\n"); + printf("Critical Warnings:\n"); + printf(" Available Spare Space: %s\n", + health_page->critical_warning.bits.available_spare ? "WARNING" : "OK"); + printf(" Temperature: %s\n", + health_page->critical_warning.bits.temperature ? "WARNING" : "OK"); + printf(" Device Reliability: %s\n", + health_page->critical_warning.bits.device_reliability ? "WARNING" : "OK"); + printf(" Read Only: %s\n", + health_page->critical_warning.bits.read_only ? "Yes" : "No"); + printf(" Volatile Memory Backup: %s\n", + health_page->critical_warning.bits.volatile_memory_backup ? "WARNING" : "OK"); + printf("Current Temperature: %u Kelvin (%u Celsius)\n", + health_page->temperature, + health_page->temperature - 273); + printf("Temperature Threshold: %u Kelvin (%u Celsius)\n", + features[NVME_FEAT_TEMPERATURE_THRESHOLD].result, + features[NVME_FEAT_TEMPERATURE_THRESHOLD].result - 273); + printf("Available Spare: %u%%\n", health_page->available_spare); + printf("Life Percentage Used: %u%%\n", health_page->percentage_used); + printf("Data Units Read: "); + print_uint128_dec(health_page->data_units_read); + printf("\n"); + printf("Data Units Written: "); + print_uint128_dec(health_page->data_units_written); + printf("\n"); + printf("Host Read Commands: "); + print_uint128_dec(health_page->host_read_commands); + printf("\n"); + printf("Host Write Commands: "); + print_uint128_dec(health_page->host_write_commands); + printf("\n"); + printf("Controller Busy Time: "); + print_uint128_dec(health_page->controller_busy_time); + printf(" minutes\n"); + printf("Power Cycles: "); + print_uint128_dec(health_page->power_cycles); + printf("\n"); + printf("Power On Hours: "); + print_uint128_dec(health_page->power_on_hours); + printf(" hours\n"); + printf("Unsafe Shutdowns: "); + print_uint128_dec(health_page->unsafe_shutdowns); + printf("\n"); + printf("Unrecoverable Media Errors: "); + print_uint128_dec(health_page->media_errors); + printf("\n"); + printf("Lifetime Error Log Entries: "); + print_uint128_dec(health_page->num_error_info_log_entries); + printf("\n"); + printf("\n"); + } + + for (i = 1; i <= nvme_ctrlr_get_num_ns(ctrlr); i++) { + print_namespace(nvme_ctrlr_get_ns(ctrlr, i)); + } +} + +static const char *ealargs[] = { + "identify", + "-c 0x1", + "-n 4", +}; + +int main(int argc, char **argv) +{ + struct pci_device_iterator *pci_dev_iter; + struct pci_device *pci_dev; + struct pci_id_match match; + int rc; + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), + (char **)(void *)(uintptr_t)ealargs); + + if (rc < 0) { + fprintf(stderr, "could not initialize dpdk\n"); + exit(1); + } + + request_mempool = rte_mempool_create("nvme_request", 8192, + nvme_request_size(), 128, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + + if (request_mempool == NULL) { + fprintf(stderr, "could not initialize request mempool\n"); + exit(1); + } + + pci_system_init(); + + match.vendor_id = PCI_MATCH_ANY; + match.subvendor_id = PCI_MATCH_ANY; + match.subdevice_id = PCI_MATCH_ANY; + match.device_id = PCI_MATCH_ANY; + match.device_class = NVME_CLASS_CODE; + match.device_class_mask = 0xFFFFFF; + + pci_dev_iter = pci_id_match_iterator_create(&match); + + rc = 0; + while ((pci_dev = pci_device_next(pci_dev_iter))) { + struct nvme_controller *ctrlr; + + if (pci_device_has_kernel_driver(pci_dev) && + !pci_device_has_uio_driver(pci_dev)) { + fprintf(stderr, "non-uio kernel driver attached to nvme\n"); + fprintf(stderr, " controller at pci bdf %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + fprintf(stderr, " skipping...\n"); + continue; + } + + pci_device_probe(pci_dev); + + ctrlr = nvme_attach(pci_dev); + if (ctrlr == NULL) { + fprintf(stderr, "failed to attach to NVMe controller at PCI BDF %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + rc = 1; + continue; + } + + print_controller(ctrlr, pci_dev); + nvme_detach(ctrlr); + } + + cleanup(); + + pci_iterator_destroy(pci_dev_iter); + return rc; +} diff --git a/examples/nvme/perf/.gitignore b/examples/nvme/perf/.gitignore new file mode 100644 index 000000000..bd14107d8 --- /dev/null +++ b/examples/nvme/perf/.gitignore @@ -0,0 +1 @@ +perf diff --git a/examples/nvme/perf/Makefile b/examples/nvme/perf/Makefile new file mode 100644 index 000000000..dab315719 --- /dev/null +++ b/examples/nvme/perf/Makefile @@ -0,0 +1,61 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = perf + +C_SRCS := perf.c + +CFLAGS += -I. $(DPDK_INC) + +SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \ + $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \ + $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a + +LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt + +OBJS = $(C_SRCS:.c=.o) + +all : $(APP) + +objs : $(OBJS) + +$(APP) : $(OBJS) $(SPDK_LIBS) + @echo " LINK $@" + $(Q)$(CC) $(CPPFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + +clean : + $(Q)rm -f $(OBJS) *.d $(APP) + + diff --git a/examples/nvme/perf/perf.c b/examples/nvme/perf/perf.c new file mode 100644 index 000000000..4b0924955 --- /dev/null +++ b/examples/nvme/perf/perf.c @@ -0,0 +1,522 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include "spdk/nvme.h" +#include "spdk/pci.h" + +struct ctrlr_entry { + struct nvme_controller *ctrlr; + struct ctrlr_entry *next; +}; + +struct ns_entry { + struct nvme_controller *ctrlr; + struct nvme_namespace *ns; + struct ns_entry *next; + uint32_t io_size_blocks; + int io_completed; + int current_queue_depth; + uint64_t size_in_ios; + uint64_t offset_in_ios; + bool is_draining; + char name[1024]; +}; + +struct perf_task { + struct ns_entry *entry; + void *buf; +}; + +struct rte_mempool *request_mempool; +static struct rte_mempool *task_pool; + +static struct ctrlr_entry *g_controllers = NULL; +static struct ns_entry *g_namespaces = NULL; + +static uint64_t g_tsc_rate; + +static int g_io_size_bytes; +static int g_rw_percentage; +static int g_is_random; +static int g_queue_depth; +static int g_time_in_sec; + + +static void +register_ns(struct nvme_controller *ctrlr, struct pci_device *pci_dev, struct nvme_namespace *ns) +{ + struct ns_entry *entry = malloc(sizeof(struct ns_entry)); + const struct nvme_controller_data *cdata = nvme_ctrlr_get_data(ctrlr); + + entry->ctrlr = ctrlr; + entry->ns = ns; + entry->next = g_namespaces; + entry->io_completed = 0; + entry->current_queue_depth = 0; + entry->offset_in_ios = 0; + entry->size_in_ios = nvme_ns_get_size(ns) / + g_io_size_bytes; + entry->io_size_blocks = g_io_size_bytes / nvme_ns_get_sector_size(ns); + entry->is_draining = false; + + snprintf(entry->name, sizeof(cdata->mn), "%s", cdata->mn); + g_namespaces = entry; +} + +static void +register_ctrlr(struct nvme_controller *ctrlr, struct pci_device *pci_dev) +{ + int nsid, num_ns; + struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); + + entry->ctrlr = ctrlr; + entry->next = g_controllers; + g_controllers = entry; + + num_ns = nvme_ctrlr_get_num_ns(ctrlr); + for (nsid = 1; nsid <= num_ns; nsid++) { + register_ns(ctrlr, pci_dev, nvme_ctrlr_get_ns(ctrlr, nsid)); + } + +} + +void task_ctor(struct rte_mempool *mp, void *arg, void *__task, unsigned id) +{ + struct perf_task *task = __task; + task->buf = rte_malloc(NULL, g_io_size_bytes, 0x200); +} + +static void io_complete(void *ctx, const struct nvme_completion *completion); + +static unsigned int __thread seed = 0; + +static void +submit_single_io(struct ns_entry *entry) +{ + struct perf_task *task = NULL; + uint64_t offset_in_ios; + int rc; + + rte_mempool_get(task_pool, (void **)&task); + + task->entry = entry; + + if (g_is_random) { + offset_in_ios = rand_r(&seed) % entry->size_in_ios; + } else { + offset_in_ios = entry->offset_in_ios++; + if (entry->offset_in_ios == entry->size_in_ios) { + entry->offset_in_ios = 0; + } + } + + if ((g_rw_percentage == 100) || + (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) { + rc = nvme_ns_cmd_read(entry->ns, task->buf, offset_in_ios * entry->io_size_blocks, + entry->io_size_blocks, io_complete, task); + } else { + rc = nvme_ns_cmd_write(entry->ns, task->buf, offset_in_ios * entry->io_size_blocks, + entry->io_size_blocks, io_complete, task); + } + + if (rc != 0) { + fprintf(stderr, "starting I/O failed\n"); + } + + entry->current_queue_depth++; +} + +static void +io_complete(void *ctx, const struct nvme_completion *completion) +{ + struct perf_task *task; + struct ns_entry *entry; + + task = (struct perf_task *)ctx; + + entry = task->entry; + entry->current_queue_depth--; + entry->io_completed++; + + rte_mempool_put(task_pool, task); + + /* + * is_draining indicates when time has expired for the test run + * and we are just waiting for the previously submitted I/O + * to complete. In this case, do not submit a new I/O to replace + * the one just completed. + */ + if (!entry->is_draining) { + submit_single_io(entry); + } +} + +static void +check_io(struct ns_entry *entry) +{ + nvme_ctrlr_process_io_completions(entry->ctrlr); +} + +static void +submit_io(struct ns_entry *entry, int queue_depth) +{ + while (queue_depth-- > 0) { + submit_single_io(entry); + } +} + +static void +drain_io(struct ns_entry *entry) +{ + entry->is_draining = true; + while (entry->current_queue_depth > 0) { + check_io(entry); + } +} + +static int +work_fn(void *arg) +{ + uint64_t tsc_end = rte_get_timer_cycles() + g_time_in_sec * g_tsc_rate; + struct ns_entry *entry = (struct ns_entry *)arg; + + nvme_register_io_thread(); + + /* Submit initial I/O for each namespace. */ + while (entry != NULL) { + submit_io(entry, g_queue_depth); + entry = entry->next; + } + + while (1) { + /* + * Check for completed I/O for each controller. A new + * I/O will be submitted in the io_complete callback + * to replace each I/O that is completed. + */ + entry = (struct ns_entry *)arg; + while (entry != NULL) { + check_io(entry); + entry = entry->next; + } + + rte_delay_us(1); + + if (rte_get_timer_cycles() > tsc_end) { + break; + } + } + + entry = (struct ns_entry *)arg; + while (entry != NULL) { + drain_io(entry); + entry = entry->next; + } + + nvme_unregister_io_thread(); + + return 0; +} + +static void usage(char *program_name) +{ + printf("%s options\n", program_name); + printf("\t[-q io depth]\n"); + printf("\t[-s io size in bytes]\n"); + printf("\t[-w io pattern type, must be one of\n"); + printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); + printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); + printf("\t[-t time in seconds]\n"); +} + +static void +print_stats(void) +{ + float io_per_second, mb_per_second; + float total_io_per_second, total_mb_per_second; + + total_io_per_second = 0; + total_mb_per_second = 0; + + struct ns_entry *entry = g_namespaces; + while (entry != NULL) { + io_per_second = (float)entry->io_completed / + g_time_in_sec; + mb_per_second = io_per_second * g_io_size_bytes / + (1024 * 1024); + printf("%-.20s: %10.2f IO/s %10.2f MB/s\n", + entry->name, io_per_second, + mb_per_second); + total_io_per_second += io_per_second; + total_mb_per_second += mb_per_second; + entry = entry->next; + } + + printf("=====================================================\n"); + printf("%-20s: %10.2f IO/s %10.2f MB/s\n", + "Total", total_io_per_second, total_mb_per_second); +} + +static int +parse_args(int argc, char **argv) +{ + const char *workload_type; + int op; + bool mix_specified = false; + + /* default value*/ + g_queue_depth = 0; + g_io_size_bytes = 0; + workload_type = NULL; + g_time_in_sec = 0; + g_rw_percentage = -1; + + while ((op = getopt(argc, argv, "q:s:t:w:M:")) != -1) { + switch (op) { + case 'q': + g_queue_depth = atoi(optarg); + break; + case 's': + g_io_size_bytes = atoi(optarg); + break; + case 't': + g_time_in_sec = atoi(optarg); + break; + case 'w': + workload_type = optarg; + break; + case 'M': + g_rw_percentage = atoi(optarg); + mix_specified = true; + break; + default: + usage(argv[0]); + return 1; + } + } + + if (!g_queue_depth) { + usage(argv[0]); + return 1; + } + if (!g_io_size_bytes) { + usage(argv[0]); + return 1; + } + if (!workload_type) { + usage(argv[0]); + return 1; + } + if (!g_time_in_sec) { + usage(argv[0]); + return 1; + } + + if (strcmp(workload_type, "read") && + strcmp(workload_type, "write") && + strcmp(workload_type, "randread") && + strcmp(workload_type, "randwrite") && + strcmp(workload_type, "rw") && + strcmp(workload_type, "randrw")) { + fprintf(stderr, + "io pattern type must be one of\n" + "(read, write, randread, randwrite, rw, randrw)\n"); + return 1; + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "randread")) { + g_rw_percentage = 100; + } + + if (!strcmp(workload_type, "write") || + !strcmp(workload_type, "randwrite")) { + g_rw_percentage = 0; + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "randread") || + !strcmp(workload_type, "write") || + !strcmp(workload_type, "randwrite")) { + if (mix_specified) { + fprintf(stderr, "Ignoring -M option... Please use -M option" + " only when using rw or randrw.\n"); + } + } + + if (!strcmp(workload_type, "rw") || + !strcmp(workload_type, "randrw")) { + if (g_rw_percentage < 0 || g_rw_percentage > 100) { + fprintf(stderr, + "-M must be specified to value from 0 to 100 " + "for rw or randrw.\n"); + return 1; + } + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "write") || + !strcmp(workload_type, "rw")) { + g_is_random = 0; + } else { + g_is_random = 1; + } + + optind = 1; + return 0; +} + +static int +register_controllers(void) +{ + struct pci_device_iterator *pci_dev_iter; + struct pci_device *pci_dev; + struct pci_id_match match; + int rc; + + pci_system_init(); + + match.vendor_id = PCI_MATCH_ANY; + match.subvendor_id = PCI_MATCH_ANY; + match.subdevice_id = PCI_MATCH_ANY; + match.device_id = PCI_MATCH_ANY; + match.device_class = NVME_CLASS_CODE; + match.device_class_mask = 0xFFFFFF; + + pci_dev_iter = pci_id_match_iterator_create(&match); + + rc = 0; + while ((pci_dev = pci_device_next(pci_dev_iter))) { + struct nvme_controller *ctrlr; + + if (pci_device_has_kernel_driver(pci_dev) && + !pci_device_has_uio_driver(pci_dev)) { + fprintf(stderr, "non-uio kernel driver attached to nvme\n"); + fprintf(stderr, " controller at pci bdf %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + fprintf(stderr, " skipping...\n"); + continue; + } + + pci_device_probe(pci_dev); + + ctrlr = nvme_attach(pci_dev); + if (ctrlr == NULL) { + fprintf(stderr, "nvme_attach failed for controller at pci bdf %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + rc = 1; + continue; + } + + register_ctrlr(ctrlr, pci_dev); + } + + pci_iterator_destroy(pci_dev_iter); + + return rc; +} + +static void +unregister_controllers(void) +{ + struct ctrlr_entry *entry = g_controllers; + while (entry) { + struct ctrlr_entry *next = entry->next; + nvme_detach(entry->ctrlr); + free(entry); + entry = next; + } +} + +static const char *ealargs[] = { + "perf", + "-c 0x1", + "-n 4", +}; + +int main(int argc, char **argv) +{ + int rc; + + rc = parse_args(argc, argv); + if (rc != 0) { + return rc; + } + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), + (char **)(void *)(uintptr_t)ealargs); + + if (rc < 0) { + fprintf(stderr, "could not initialize dpdk\n"); + return 1; + } + + request_mempool = rte_mempool_create("nvme_request", 8192, + nvme_request_size(), 128, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + + if (request_mempool == NULL) { + fprintf(stderr, "could not initialize request mempool\n"); + return 1; + } + + task_pool = rte_mempool_create("task_pool", 2048, + sizeof(struct perf_task), + 64, 0, NULL, NULL, task_ctor, NULL, + SOCKET_ID_ANY, 0); + + g_tsc_rate = rte_get_timer_hz(); + + rc = register_controllers(); + + work_fn(g_namespaces); + print_stats(); + + unregister_controllers(); + + return rc; +} diff --git a/include/spdk/barrier.h b/include/spdk/barrier.h new file mode 100644 index 000000000..195065ce0 --- /dev/null +++ b/include/spdk/barrier.h @@ -0,0 +1,40 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_BARRIER_H +#define SPDK_BARRIER_H + +#define wmb() __asm volatile("sfence" ::: "memory") +#define mb() __asm volatile("mfence" ::: "memory") + +#endif diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h new file mode 100644 index 000000000..9ba73e831 --- /dev/null +++ b/include/spdk/nvme.h @@ -0,0 +1,372 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_NVME_H +#define SPDK_NVME_H + +#include +#include "nvme_spec.h" + +/** \file + * + */ + +#define NVME_DEFAULT_RETRY_COUNT (4) +extern int32_t nvme_retry_count; + +#ifdef __cplusplus +extern "C" { +#endif + +/** \brief Opaque handle to a controller. Obtained by calling nvme_attach(). */ +struct nvme_controller; + +/** + * \brief Attaches specified device to the NVMe driver. + * + * On success, the nvme_controller handle is valid for other nvme_ctrlr_* functions. + * On failure, the return value will be NULL. + * + * This function should be called from a single thread while no other threads or drivers + * are actively using the NVMe device. + * + * To stop using the the controller and release its associated resources, + * call \ref nvme_detach with the nvme_controller instance returned by this function. + */ +struct nvme_controller *nvme_attach(void *devhandle); + +/** + * \brief Detaches specified device returned by \ref nvme_attach() from the NVMe driver. + * + * On success, the nvme_controller handle is no longer valid. + * + * This function should be called from a single thread while no other threads + * are actively using the NVMe device. + * + */ +int nvme_detach(struct nvme_controller *ctrlr); + +/** + * \brief Perform a full hardware reset of the NVMe controller. + * + * This function should be called from a single thread while no other threads + * are actively using the NVMe device. + * + * Any pointers returned from nvme_ctrlr_get_ns() and nvme_ns_get_data() may be invalidated + * by calling this function. The number of namespaces as returned by nvme_ctrlr_get_num_ns() may + * also change. + */ +int nvme_ctrlr_reset(struct nvme_controller *ctrlr); + +/** + * \brief Get the identify controller data as defined by the NVMe specification. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +const struct nvme_controller_data *nvme_ctrlr_get_data(struct nvme_controller *ctrlr); + +/** + * \brief Get the number of namespaces for the given NVMe controller. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + * This is equivalent to calling nvme_ctrlr_get_data() to get the + * nvme_controller_data and then reading the nn field. + * + */ +uint32_t nvme_ctrlr_get_num_ns(struct nvme_controller *ctrlr); + +/** + * Signature for callback function invoked when a command is completed. + * + * The nvme_completion parameter contains the completion status. + */ +typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *); + +/** + * Signature for callback function invoked when an asynchronous error + * request command is completed. + * + * The aer_cb_arg parameter is set to the context specified by + * nvme_register_aer_callback(). + * The nvme_completion parameter contains the completion status of the + * asynchronous event request that was completed. + */ +typedef void (*nvme_aer_cb_fn_t)(void *aer_cb_arg, + const struct nvme_completion *); + +void nvme_ctrlr_register_aer_callback(struct nvme_controller *ctrlr, + nvme_aer_cb_fn_t aer_cb_fn, + void *aer_cb_arg); + +/** + * \brief Send the given NVM I/O command to the NVMe controller. + * + * This is a low level interface for submitting I/O commands directly. Prefer + * the nvme_ns_cmd_* functions instead. The validity of the command will + * not be checked! + * + * When constructing the nvme_command it is not necessary to fill out the PRP + * list/SGL or the CID. The driver will handle both of those for you. + * + * This function is thread safe and can be called at any point after + * nvme_register_io_thread(). + * + */ +int nvme_ctrlr_cmd_io_raw(struct nvme_controller *ctrlr, + struct nvme_command *cmd, + void *buf, uint32_t len, + nvme_cb_fn_t cb_fn, void *cb_arg); + +/** + * \brief Process any outstanding completions for I/O submitted on the current thread. + * + * This will only process completions for I/O that were submitted on the same thread + * that this function is called from. This call is also non-blocking, i.e. it only + * processes completions that are ready at the time of this function call. It does not + * wait for outstanding commands to finish + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +void nvme_ctrlr_process_io_completions(struct nvme_controller *ctrlr); + +/** + * \brief Send the given admin command to the NVMe controller. + * + * This is a low level interface for submitting admin commands directly. Prefer + * the nvme_ctrlr_cmd_* functions instead. The validity of the command will + * not be checked! + * + * When constructing the nvme_command it is not necessary to fill out the PRP + * list/SGL or the CID. The driver will handle both of those for you. + * + * This function is thread safe and can be called at any point after + * \ref nvme_attach(). + * + * Call \ref nvme_ctrlr_process_admin_completions() to poll for completion + * of commands submitted through this function. + */ +int nvme_ctrlr_cmd_admin_raw(struct nvme_controller *ctrlr, + struct nvme_command *cmd, + void *buf, uint32_t len, + nvme_cb_fn_t cb_fn, void *cb_arg); + +/** + * \brief Process any outstanding completions for admin commands. + * + * This will process completions for admin commands submitted on any thread. + * + * This call is non-blocking, i.e. it only processes completions that are ready + * at the time of this function call. It does not wait for outstanding commands to + * finish. + * + * This function is thread safe and can be called at any point after nvme_attach(). + */ +void nvme_ctrlr_process_admin_completions(struct nvme_controller *ctrlr); + + +/** \brief Opaque handle to a namespace. Obtained by calling nvme_ctrlr_get_ns(). */ +struct nvme_namespace; + +/** + * \brief Get a handle to a namespace for the given controller. + * + * Namespaces are numbered from 1 to the total number of namespaces. There will never + * be any gaps in the numbering. The number of namespaces is obtained by calling + * nvme_ctrlr_get_num_ns(). + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +struct nvme_namespace *nvme_ctrlr_get_ns(struct nvme_controller *ctrlr, uint32_t ns_id); + +/** + * \brief Get the identify namespace data as defined by the NVMe specification. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +const struct nvme_namespace_data *nvme_ns_get_data(struct nvme_namespace *ns); + +/** + * \brief Get the namespace id (index number) from the given namespace handle. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +uint32_t nvme_ns_get_id(struct nvme_namespace *ns); + +/** + * \brief Get the maximum transfer size, in bytes, for an I/O sent to the given namespace. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +uint32_t nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns); + +/** + * \brief Get the sector size, in bytes, of the given namespace. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +uint32_t nvme_ns_get_sector_size(struct nvme_namespace *ns); + +/** + * \brief Get the number of sectors for the given namespace. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +uint64_t nvme_ns_get_num_sectors(struct nvme_namespace *ns); + +/** + * \brief Get the size, in bytes, of the given namespace. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +uint64_t nvme_ns_get_size(struct nvme_namespace *ns); + +enum nvme_namespace_flags { + NVME_NS_DEALLOCATE_SUPPORTED = 0x1, + NVME_NS_FLUSH_SUPPORTED = 0x2, +}; + +/** + * \brief Get the flags for the given namespace. + * + * See nvme_namespace_flags for the possible flags returned. + * + * This function is thread safe and can be called at any point after nvme_attach(). + * + */ +uint32_t nvme_ns_get_flags(struct nvme_namespace *ns); + +/** + * \brief Submits a write I/O to the specified NVMe namespace. + * + * \param ns NVMe namespace to submit the write I/O + * \param payload virtual address pointer to the data payload + * \param lba starting LBA to write the data + * \param lba_count length (in sectors) for the write operation + * \param cb_fn callback function to invoke when the I/O is completed + * \param cb_arg argument to pass to the callback function + * + * \return 0 if successfully submitted, ENOMEM if an nvme_request + * structure cannot be allocated for the I/O request + * + * This function is thread safe and can be called at any point after + * nvme_register_io_thread(). + */ +int nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, + uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, + void *cb_arg); + +/** + * \brief Submits a read I/O to the specified NVMe namespace. + * + * \param ns NVMe namespace to submit the read I/O + * \param payload virtual address pointer to the data payload + * \param lba starting LBA to read the data + * \param lba_count length (in sectors) for the read operation + * \param cb_fn callback function to invoke when the I/O is completed + * \param cb_arg argument to pass to the callback function + * + * \return 0 if successfully submitted, ENOMEM if an nvme_request + * structure cannot be allocated for the I/O request + * + * This function is thread safe and can be called at any point after + * nvme_register_io_thread(). + */ +int nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, + uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, + void *cb_arg); + +/** + * \brief Submits a deallocation request to the specified NVMe namespace. + * + * \param ns NVMe namespace to submit the deallocation request + * \param payload virtual address pointer to the list of LBA ranges to + * deallocate + * \param num_ranges number of ranges in the list pointed to by payload + * \param cb_fn callback function to invoke when the I/O is completed + * \param cb_arg argument to pass to the callback function + * + * \return 0 if successfully submitted, ENOMEM if an nvme_request + * structure cannot be allocated for the I/O request + * + * This function is thread safe and can be called at any point after + * nvme_register_io_thread(). + */ +int nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload, + uint8_t num_ranges, nvme_cb_fn_t cb_fn, + void *cb_arg); + +/** + * \brief Submits a flush request to the specified NVMe namespace. + * + * \param ns NVMe namespace to submit the flush request + * \param cb_fn callback function to invoke when the I/O is completed + * \param cb_arg argument to pass to the callback function + * + * \return 0 if successfully submitted, ENOMEM if an nvme_request + * structure cannot be allocated for the I/O request + * + * This function is thread safe and can be called at any point after + * nvme_register_io_thread(). + */ +int nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, + void *cb_arg); + +/** + * \brief Get the size, in bytes, of an nvme_request. + * + * This is the size of the request objects that need to be allocated by the + * nvme_alloc_request macro in nvme_impl.h + * + * This function is thread safe and can be called at any time. + * + */ +size_t nvme_request_size(void); + +int nvme_register_io_thread(void); +void nvme_unregister_io_thread(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/spdk/nvme_spec.h b/include/spdk/nvme_spec.h new file mode 100644 index 000000000..8780f7b1b --- /dev/null +++ b/include/spdk/nvme_spec.h @@ -0,0 +1,948 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_NVME_SPEC_H +#define SPDK_NVME_SPEC_H + +#include +#include + +/** + * \file + * + */ + +/** + * PCI class code for NVMe devices. + * + * Base class code 01h: mass storage + * Subclass code 08h: non-volatile memory + * Programming interface 02h: NVM Express + */ +#define NVME_CLASS_CODE 0x10802 + +/** + * Use to mark a command to apply to all namespaces, or to retrieve global + * log pages. + */ +#define NVME_GLOBAL_NAMESPACE_TAG ((uint32_t)0xFFFFFFFF) + +#define NVME_MAX_IO_QUEUES (1 << 16) + +union nvme_cap_lo_register { + uint32_t raw; + struct { + /** maximum queue entries supported */ + uint32_t mqes : 16; + + /** contiguous queues required */ + uint32_t cqr : 1; + + /** arbitration mechanism supported */ + uint32_t ams : 2; + + uint32_t reserved1 : 5; + + /** timeout */ + uint32_t to : 8; + } bits; +}; +_Static_assert(sizeof(union nvme_cap_lo_register) == 4, "Incorrect size"); + +union nvme_cap_hi_register { + uint32_t raw; + struct { + /** doorbell stride */ + uint32_t dstrd : 4; + + uint32_t reserved3 : 1; + + /** command sets supported */ + uint32_t css_nvm : 1; + + uint32_t css_reserved : 3; + uint32_t reserved2 : 7; + + /** memory page size minimum */ + uint32_t mpsmin : 4; + + /** memory page size maximum */ + uint32_t mpsmax : 4; + + uint32_t reserved1 : 8; + } bits; +}; +_Static_assert(sizeof(union nvme_cap_hi_register) == 4, "Incorrect size"); + +union nvme_cc_register { + uint32_t raw; + struct { + /** enable */ + uint32_t en : 1; + + uint32_t reserved1 : 3; + + /** i/o command set selected */ + uint32_t css : 3; + + /** memory page size */ + uint32_t mps : 4; + + /** arbitration mechanism selected */ + uint32_t ams : 3; + + /** shutdown notification */ + uint32_t shn : 2; + + /** i/o submission queue entry size */ + uint32_t iosqes : 4; + + /** i/o completion queue entry size */ + uint32_t iocqes : 4; + + uint32_t reserved2 : 8; + } bits; +}; +_Static_assert(sizeof(union nvme_cc_register) == 4, "Incorrect size"); + +enum nvme_shn_value { + NVME_SHN_NORMAL = 0x1, + NVME_SHN_ABRUPT = 0x2, +}; + +union nvme_csts_register { + uint32_t raw; + struct { + /** ready */ + uint32_t rdy : 1; + + /** controller fatal status */ + uint32_t cfs : 1; + + /** shutdown status */ + uint32_t shst : 2; + + uint32_t reserved1 : 28; + } bits; +}; +_Static_assert(sizeof(union nvme_csts_register) == 4, "Incorrect size"); + +enum nvme_shst_value { + NVME_SHST_NORMAL = 0x0, + NVME_SHST_OCCURRING = 0x1, + NVME_SHST_COMPLETE = 0x2, +}; + +union nvme_aqa_register { + uint32_t raw; + struct { + /** admin submission queue size */ + uint32_t asqs : 12; + + uint32_t reserved1 : 4; + + /** admin completion queue size */ + uint32_t acqs : 12; + + uint32_t reserved2 : 4; + } bits; +}; +_Static_assert(sizeof(union nvme_aqa_register) == 4, "Incorrect size"); + +struct nvme_registers { + /** controller capabilities */ + union nvme_cap_lo_register cap_lo; + union nvme_cap_hi_register cap_hi; + + uint32_t vs; /* version */ + uint32_t intms; /* interrupt mask set */ + uint32_t intmc; /* interrupt mask clear */ + + /** controller configuration */ + union nvme_cc_register cc; + + uint32_t reserved1; + uint32_t csts; /* controller status */ + uint32_t nssr; /* NVM subsystem reset */ + + /** admin queue attributes */ + union nvme_aqa_register aqa; + + uint64_t asq; /* admin submission queue base addr */ + uint64_t acq; /* admin completion queue base addr */ + uint32_t reserved3[0x3f2]; + + struct { + uint32_t sq_tdbl; /* submission queue tail doorbell */ + uint32_t cq_hdbl; /* completion queue head doorbell */ + } doorbell[1]; +}; + +/* NVMe controller register space offsets */ +_Static_assert(0x00 == offsetof(struct nvme_registers, cap_lo), "Incorrect register offset"); +_Static_assert(0x08 == offsetof(struct nvme_registers, vs), "Incorrect register offset"); +_Static_assert(0x0C == offsetof(struct nvme_registers, intms), "Incorrect register offset"); +_Static_assert(0x10 == offsetof(struct nvme_registers, intmc), "Incorrect register offset"); +_Static_assert(0x14 == offsetof(struct nvme_registers, cc), "Incorrect register offset"); +_Static_assert(0x1C == offsetof(struct nvme_registers, csts), "Incorrect register offset"); +_Static_assert(0x20 == offsetof(struct nvme_registers, nssr), "Incorrect register offset"); +_Static_assert(0x24 == offsetof(struct nvme_registers, aqa), "Incorrect register offset"); +_Static_assert(0x28 == offsetof(struct nvme_registers, asq), "Incorrect register offset"); +_Static_assert(0x30 == offsetof(struct nvme_registers, acq), "Incorrect register offset"); + +enum nvme_sgl_descriptor_type { + NVME_SGL_TYPE_DATA_BLOCK = 0x0, + NVME_SGL_TYPE_BIT_BUCKET = 0x1, + NVME_SGL_TYPE_SEGMENT = 0x2, + NVME_SGL_TYPE_LAST_SEGMENT = 0x3, + /* 0x4 - 0xe reserved */ + NVME_SGL_TYPE_VENDOR_SPECIFIC = 0xf +}; + +struct __attribute__((packed)) nvme_sgl_descriptor { + uint64_t address; + uint32_t length; + uint8_t reserved[3]; + + /** SGL descriptor type */ + uint8_t type : 4; + + /** SGL descriptor type specific */ + uint8_t type_specific : 4; +}; +_Static_assert(sizeof(struct nvme_sgl_descriptor) == 16, "Incorrect size"); + +enum nvme_psdt_value { + NVME_PSDT_PRP = 0x0, + NVME_PSDT_SGL_MPTR_CONTIG = 0x1, + NVME_PSDT_SGL_MPTR_SGL = 0x2, + NVME_PSDT_RESERVED = 0x3 +}; + +struct nvme_command { + /* dword 0 */ + uint16_t opc : 8; /* opcode */ + uint16_t fuse : 2; /* fused operation */ + uint16_t rsvd1 : 4; + uint16_t psdt : 2; + uint16_t cid; /* command identifier */ + + /* dword 1 */ + uint32_t nsid; /* namespace identifier */ + + /* dword 2-3 */ + uint32_t rsvd2; + uint32_t rsvd3; + + /* dword 4-5 */ + uint64_t mptr; /* metadata pointer */ + + /* dword 6-9: data pointer */ + union { + struct { + uint64_t prp1; /* prp entry 1 */ + uint64_t prp2; /* prp entry 2 */ + } prp; + + struct nvme_sgl_descriptor sgl1; + } dptr; + + /* dword 10-15 */ + uint32_t cdw10; /* command-specific */ + uint32_t cdw11; /* command-specific */ + uint32_t cdw12; /* command-specific */ + uint32_t cdw13; /* command-specific */ + uint32_t cdw14; /* command-specific */ + uint32_t cdw15; /* command-specific */ +}; +_Static_assert(sizeof(struct nvme_command) == 64, "Incorrect size"); + +struct nvme_status { + uint16_t p : 1; /* phase tag */ + uint16_t sc : 8; /* status code */ + uint16_t sct : 3; /* status code type */ + uint16_t rsvd2 : 2; + uint16_t m : 1; /* more */ + uint16_t dnr : 1; /* do not retry */ +}; +_Static_assert(sizeof(struct nvme_status) == 2, "Incorrect size"); + +struct nvme_completion { + /* dword 0 */ + uint32_t cdw0; /* command-specific */ + + /* dword 1 */ + uint32_t rsvd1; + + /* dword 2 */ + uint16_t sqhd; /* submission queue head pointer */ + uint16_t sqid; /* submission queue identifier */ + + /* dword 3 */ + uint16_t cid; /* command identifier */ + struct nvme_status status; +}; +_Static_assert(sizeof(struct nvme_completion) == 16, "Incorrect size"); + +struct nvme_dsm_range { + uint32_t attributes; + uint32_t length; + uint64_t starting_lba; +}; +_Static_assert(sizeof(struct nvme_dsm_range) == 16, "Incorrect size"); + +/* status code types */ +enum nvme_status_code_type { + NVME_SCT_GENERIC = 0x0, + NVME_SCT_COMMAND_SPECIFIC = 0x1, + NVME_SCT_MEDIA_ERROR = 0x2, + /* 0x3-0x6 - reserved */ + NVME_SCT_VENDOR_SPECIFIC = 0x7, +}; + +/* generic command status codes */ +enum nvme_generic_command_status_code { + NVME_SC_SUCCESS = 0x00, + NVME_SC_INVALID_OPCODE = 0x01, + NVME_SC_INVALID_FIELD = 0x02, + NVME_SC_COMMAND_ID_CONFLICT = 0x03, + NVME_SC_DATA_TRANSFER_ERROR = 0x04, + NVME_SC_ABORTED_POWER_LOSS = 0x05, + NVME_SC_INTERNAL_DEVICE_ERROR = 0x06, + NVME_SC_ABORTED_BY_REQUEST = 0x07, + NVME_SC_ABORTED_SQ_DELETION = 0x08, + NVME_SC_ABORTED_FAILED_FUSED = 0x09, + NVME_SC_ABORTED_MISSING_FUSED = 0x0a, + NVME_SC_INVALID_NAMESPACE_OR_FORMAT = 0x0b, + NVME_SC_COMMAND_SEQUENCE_ERROR = 0x0c, + + NVME_SC_LBA_OUT_OF_RANGE = 0x80, + NVME_SC_CAPACITY_EXCEEDED = 0x81, + NVME_SC_NAMESPACE_NOT_READY = 0x82, +}; + +/* command specific status codes */ +enum nvme_command_specific_status_code { + NVME_SC_COMPLETION_QUEUE_INVALID = 0x00, + NVME_SC_INVALID_QUEUE_IDENTIFIER = 0x01, + NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED = 0x02, + NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED = 0x03, + /* 0x04 - reserved */ + NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05, + NVME_SC_INVALID_FIRMWARE_SLOT = 0x06, + NVME_SC_INVALID_FIRMWARE_IMAGE = 0x07, + NVME_SC_INVALID_INTERRUPT_VECTOR = 0x08, + NVME_SC_INVALID_LOG_PAGE = 0x09, + NVME_SC_INVALID_FORMAT = 0x0a, + NVME_SC_FIRMWARE_REQUIRES_RESET = 0x0b, + + NVME_SC_CONFLICTING_ATTRIBUTES = 0x80, + NVME_SC_INVALID_PROTECTION_INFO = 0x81, + NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE = 0x82, +}; + +/* media error status codes */ +enum nvme_media_error_status_code { + NVME_SC_WRITE_FAULTS = 0x80, + NVME_SC_UNRECOVERED_READ_ERROR = 0x81, + NVME_SC_GUARD_CHECK_ERROR = 0x82, + NVME_SC_APPLICATION_TAG_CHECK_ERROR = 0x83, + NVME_SC_REFERENCE_TAG_CHECK_ERROR = 0x84, + NVME_SC_COMPARE_FAILURE = 0x85, + NVME_SC_ACCESS_DENIED = 0x86, +}; + +/* admin opcodes */ +enum nvme_admin_opcode { + NVME_OPC_DELETE_IO_SQ = 0x00, + NVME_OPC_CREATE_IO_SQ = 0x01, + NVME_OPC_GET_LOG_PAGE = 0x02, + /* 0x03 - reserved */ + NVME_OPC_DELETE_IO_CQ = 0x04, + NVME_OPC_CREATE_IO_CQ = 0x05, + NVME_OPC_IDENTIFY = 0x06, + /* 0x07 - reserved */ + NVME_OPC_ABORT = 0x08, + NVME_OPC_SET_FEATURES = 0x09, + NVME_OPC_GET_FEATURES = 0x0a, + /* 0x0b - reserved */ + NVME_OPC_ASYNC_EVENT_REQUEST = 0x0c, + NVME_OPC_NAMESPACE_MANAGEMENT = 0x0d, + /* 0x0e-0x0f - reserved */ + NVME_OPC_FIRMWARE_COMMIT = 0x10, + NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD = 0x11, + + NVME_OPC_NAMESPACE_ATTACHMENT = 0x15, + + NVME_OPC_FORMAT_NVM = 0x80, + NVME_OPC_SECURITY_SEND = 0x81, + NVME_OPC_SECURITY_RECEIVE = 0x82, +}; + +/* nvme nvm opcodes */ +enum nvme_nvm_opcode { + NVME_OPC_FLUSH = 0x00, + NVME_OPC_WRITE = 0x01, + NVME_OPC_READ = 0x02, + /* 0x03 - reserved */ + NVME_OPC_WRITE_UNCORRECTABLE = 0x04, + NVME_OPC_COMPARE = 0x05, + /* 0x06-0x07 - reserved */ + NVME_OPC_WRITE_ZEROES = 0x08, + NVME_OPC_DATASET_MANAGEMENT = 0x09, + + NVME_OPC_RESERVATION_REGISTER = 0x0d, + NVME_OPC_RESERVATION_REPORT = 0x0e, + + NVME_OPC_RESERVATION_ACQUIRE = 0x11, + NVME_OPC_RESERVATION_RELEASE = 0x15, +}; + +enum nvme_feature { + /* 0x00 - reserved */ + NVME_FEAT_ARBITRATION = 0x01, + NVME_FEAT_POWER_MANAGEMENT = 0x02, + NVME_FEAT_LBA_RANGE_TYPE = 0x03, + NVME_FEAT_TEMPERATURE_THRESHOLD = 0x04, + NVME_FEAT_ERROR_RECOVERY = 0x05, + NVME_FEAT_VOLATILE_WRITE_CACHE = 0x06, + NVME_FEAT_NUMBER_OF_QUEUES = 0x07, + NVME_FEAT_INTERRUPT_COALESCING = 0x08, + NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09, + NVME_FEAT_WRITE_ATOMICITY = 0x0A, + NVME_FEAT_ASYNC_EVENT_CONFIGURATION = 0x0B, + /* 0x0C-0x7F - reserved */ + NVME_FEAT_SOFTWARE_PROGRESS_MARKER = 0x80, + /* 0x81-0xBF - command set specific (reserved) */ + /* 0xC0-0xFF - vendor specific */ +}; + +enum nvme_dsm_attribute { + NVME_DSM_ATTR_INTEGRAL_READ = 0x1, + NVME_DSM_ATTR_INTEGRAL_WRITE = 0x2, + NVME_DSM_ATTR_DEALLOCATE = 0x4, +}; + +struct nvme_power_state { + uint16_t mp; /* bits 15:00: maximum power */ + + uint8_t reserved1; + + uint8_t mps : 1; /* bit 24: max power scale */ + uint8_t nops : 1; /* bit 25: non-operational state */ + uint8_t reserved2 : 6; + + uint32_t enlat; /* bits 63:32: entry latency in microseconds */ + uint32_t exlat; /* bits 95:64: exit latency in microseconds */ + + uint8_t rrt : 5; /* bits 100:96: relative read throughput */ + uint8_t reserved3 : 3; + + uint8_t rrl : 5; /* bits 108:104: relative read latency */ + uint8_t reserved4 : 3; + + uint8_t rwt : 5; /* bits 116:112: relative write throughput */ + uint8_t reserved5 : 3; + + uint8_t rwl : 5; /* bits 124:120: relative write latency */ + uint8_t reserved6 : 3; + + uint8_t reserved7[16]; +}; +_Static_assert(sizeof(struct nvme_power_state) == 32, "Incorrect size"); + +struct __attribute__((packed)) nvme_controller_data { + /* bytes 0-255: controller capabilities and features */ + + /** pci vendor id */ + uint16_t vid; + + /** pci subsystem vendor id */ + uint16_t ssvid; + + /** serial number */ + int8_t sn[20]; + + /** model number */ + int8_t mn[40]; + + /** firmware revision */ + uint8_t fr[8]; + + /** recommended arbitration burst */ + uint8_t rab; + + /** ieee oui identifier */ + uint8_t ieee[3]; + + /** multi-interface capabilities */ + uint8_t mic; + + /** maximum data transfer size */ + uint8_t mdts; + + /** controller id */ + uint16_t cntlid; + + /** version */ + uint32_t ver; + + /** RTD3 resume latency */ + uint32_t rtd3r; + + /** RTD3 entry latency */ + uint32_t rtd3e; + + /** optional asynchronous events supported */ + uint32_t oaes; + + uint8_t reserved1[160]; + + /* bytes 256-511: admin command set attributes */ + + /** optional admin command support */ + struct { + /* supports security send/receive commands */ + uint16_t security : 1; + + /* supports format nvm command */ + uint16_t format : 1; + + /* supports firmware activate/download commands */ + uint16_t firmware : 1; + + uint16_t oacs_rsvd : 13; + } oacs; + + /** abort command limit */ + uint8_t acl; + + /** asynchronous event request limit */ + uint8_t aerl; + + /** firmware updates */ + struct { + /* first slot is read-only */ + uint8_t slot1_ro : 1; + + /* number of firmware slots */ + uint8_t num_slots : 3; + + uint8_t frmw_rsvd : 4; + } frmw; + + /** log page attributes */ + struct { + /* per namespace smart/health log page */ + uint8_t ns_smart : 1; + + uint8_t lpa_rsvd : 7; + } lpa; + + /** error log page entries */ + uint8_t elpe; + + /** number of power states supported */ + uint8_t npss; + + /** admin vendor specific command configuration */ + struct { + /* admin vendor specific commands use disk format */ + uint8_t spec_format : 1; + + uint8_t avscc_rsvd : 7; + } avscc; + + /** autonomous power state transition attributes */ + struct { + /** controller supports autonomous power state transitions */ + uint8_t supported : 1; + + uint8_t apsta_rsvd : 7; + } apsta; + + /** warning composite temperature threshold */ + uint16_t wctemp; + + /** critical composite temperature threshold */ + uint16_t cctemp; + + /** maximum time for firmware activation */ + uint16_t mtfa; + + /** host memory buffer preferred size */ + uint32_t hmpre; + + /** host memory buffer minimum size */ + uint32_t hmmin; + + /** total NVM capacity */ + uint64_t tnvmcap[2]; + + /** unallocated NVM capacity */ + uint64_t unvmcap[2]; + + /** replay protected memory block support */ + struct { + uint8_t num_rpmb_units : 3; + uint8_t auth_method : 3; + uint8_t reserved1 : 2; + + uint8_t reserved2; + + uint8_t total_size; + uint8_t access_size; + } rpmbs; + + uint8_t reserved2[196]; + + /* bytes 512-703: nvm command set attributes */ + + /** submission queue entry size */ + struct { + uint8_t min : 4; + uint8_t max : 4; + } sqes; + + /** completion queue entry size */ + struct { + uint8_t min : 4; + uint8_t max : 4; + } cqes; + + uint8_t reserved3[2]; + + /** number of namespaces */ + uint32_t nn; + + /** optional nvm command support */ + struct { + uint16_t compare : 1; + uint16_t write_unc : 1; + uint16_t dsm: 1; + uint16_t reserved: 13; + } oncs; + + /** fused operation support */ + uint16_t fuses; + + /** format nvm attributes */ + uint8_t fna; + + /** volatile write cache */ + struct { + uint8_t present : 1; + uint8_t reserved : 7; + } vwc; + + /** atomic write unit normal */ + uint16_t awun; + + /** atomic write unit power fail */ + uint16_t awupf; + + /** NVM vendor specific command configuration */ + uint8_t nvscc; + + uint8_t reserved531; + + /** atomic compare & write unit */ + uint16_t acwu; + + uint16_t reserved534; + + /** SGL support */ + struct { + uint32_t supported : 1; + uint32_t reserved : 15; + uint32_t bit_bucket_descriptor_supported : 1; + uint32_t metadata_pointer_supported : 1; + uint32_t oversized_sgl_supported : 1; + } sgls; + + uint8_t reserved4[164]; + + /* bytes 704-2047: i/o command set attributes */ + uint8_t reserved5[1344]; + + /* bytes 2048-3071: power state descriptors */ + struct nvme_power_state psd[32]; + + /* bytes 3072-4095: vendor specific */ + uint8_t vs[1024]; +}; +_Static_assert(sizeof(struct nvme_controller_data) == 4096, "Incorrect size"); + +struct nvme_namespace_data { + /** namespace size */ + uint64_t nsze; + + /** namespace capacity */ + uint64_t ncap; + + /** namespace utilization */ + uint64_t nuse; + + /** namespace features */ + struct { + /** thin provisioning */ + uint8_t thin_prov : 1; + uint8_t reserved1 : 7; + } nsfeat; + + /** number of lba formats */ + uint8_t nlbaf; + + /** formatted lba size */ + struct { + uint8_t format : 4; + uint8_t extended : 1; + uint8_t reserved2 : 3; + } flbas; + + /** metadata capabilities */ + struct { + /** metadata can be transferred as part of data prp list */ + uint8_t extended : 1; + + /** metadata can be transferred with separate metadata pointer */ + uint8_t pointer : 1; + + /** reserved */ + uint8_t reserved3 : 6; + } mc; + + /** end-to-end data protection capabilities */ + struct { + /** protection information type 1 */ + uint8_t pit1 : 1; + + /** protection information type 2 */ + uint8_t pit2 : 1; + + /** protection information type 3 */ + uint8_t pit3 : 1; + + /** first eight bytes of metadata */ + uint8_t md_start : 1; + + /** last eight bytes of metadata */ + uint8_t md_end : 1; + } dpc; + + /** end-to-end data protection type settings */ + struct { + /** protection information type */ + uint8_t pit : 3; + + /** 1 == protection info transferred at start of metadata */ + /** 0 == protection info transferred at end of metadata */ + uint8_t md_start : 1; + + uint8_t reserved4 : 4; + } dps; + + /** namespace multi-path I/O and namespace sharing capabilities */ + struct { + uint8_t can_share : 1; + uint8_t reserved : 7; + } nmic; + + /** reservation capabilities */ + struct { + /** supports persist through power loss */ + uint8_t persist : 1; + + /** supports write exclusive */ + uint8_t write_exclusive : 1; + + /** supports exclusive access */ + uint8_t exclusive_access : 1; + + /** supports write exclusive - registrants only */ + uint8_t write_exclusive_reg_only : 1; + + /** supports exclusive access - registrants only */ + uint8_t exclusive_access_reg_only : 1; + + /** supports write exclusive - all registrants */ + uint8_t write_exclusive_all_reg : 1; + + /** supports exclusive access - all registrants */ + uint8_t exclusive_access_all_reg : 1; + + uint8_t reserved : 1; + } rescap; + + /** format progress indicator */ + uint8_t fpi; + + uint8_t reserved33; + + /** namespace atomic write unit normal */ + uint16_t nawun; + + /** namespace atomic write unit power fail */ + uint16_t nawupf; + + /** namespace atomic compare & write unit */ + uint16_t nacwu; + + /** namespace atomic boundary size normal */ + uint16_t nabsn; + + /** namespace atomic boundary offset */ + uint16_t nabo; + + /** namespace atomic boundary size power fail */ + uint16_t nabspf; + + uint16_t reserved46; + + /** NVM capacity */ + uint64_t nvmcap[2]; + + uint8_t reserved64[40]; + + /** namespace globally unique identifier */ + uint8_t nguid[16]; + + /** IEEE extended unique identifier */ + uint64_t eui64; + + /** lba format support */ + struct { + /** metadata size */ + uint32_t ms : 16; + + /** lba data size */ + uint32_t lbads : 8; + + /** relative performance */ + uint32_t rp : 2; + + uint32_t reserved6 : 6; + } lbaf[16]; + + uint8_t reserved6[192]; + + uint8_t vendor_specific[3712]; +}; +_Static_assert(sizeof(struct nvme_namespace_data) == 4096, "Incorrect size"); + +enum nvme_log_page { + /* 0x00 - reserved */ + NVME_LOG_ERROR = 0x01, + NVME_LOG_HEALTH_INFORMATION = 0x02, + NVME_LOG_FIRMWARE_SLOT = 0x03, + /* 0x04-0x7F - reserved */ + /* 0x80-0xBF - I/O command set specific */ + /* 0xC0-0xFF - vendor specific */ +}; + +struct nvme_error_information_entry { + uint64_t error_count; + uint16_t sqid; + uint16_t cid; + struct nvme_status status; + uint16_t error_location; + uint64_t lba; + uint32_t nsid; + uint8_t vendor_specific; + uint8_t reserved[35]; +}; +_Static_assert(sizeof(struct nvme_error_information_entry) == 64, "Incorrect size"); + +union nvme_critical_warning_state { + uint8_t raw; + + struct { + uint8_t available_spare : 1; + uint8_t temperature : 1; + uint8_t device_reliability : 1; + uint8_t read_only : 1; + uint8_t volatile_memory_backup : 1; + uint8_t reserved : 3; + } bits; +}; +_Static_assert(sizeof(union nvme_critical_warning_state) == 1, "Incorrect size"); + +struct __attribute__((packed)) nvme_health_information_page { + union nvme_critical_warning_state critical_warning; + + uint16_t temperature; + uint8_t available_spare; + uint8_t available_spare_threshold; + uint8_t percentage_used; + + uint8_t reserved[26]; + + /* + * Note that the following are 128-bit values, but are + * defined as an array of 2 64-bit values. + */ + /* Data Units Read is always in 512-byte units. */ + uint64_t data_units_read[2]; + /* Data Units Written is always in 512-byte units. */ + uint64_t data_units_written[2]; + /* For NVM command set, this includes Compare commands. */ + uint64_t host_read_commands[2]; + uint64_t host_write_commands[2]; + /* Controller Busy Time is reported in minutes. */ + uint64_t controller_busy_time[2]; + uint64_t power_cycles[2]; + uint64_t power_on_hours[2]; + uint64_t unsafe_shutdowns[2]; + uint64_t media_errors[2]; + uint64_t num_error_info_log_entries[2]; + + uint8_t reserved2[320]; +}; +_Static_assert(sizeof(struct nvme_health_information_page) == 512, "Incorrect size"); + +struct nvme_firmware_page { + struct { + uint8_t slot : 3; /* slot for current FW */ + uint8_t reserved : 5; + } afi; + + uint8_t reserved[7]; + uint64_t revision[7]; /* revisions for 7 slots */ + uint8_t reserved2[448]; +}; +_Static_assert(sizeof(struct nvme_firmware_page) == 512, "Incorrect size"); + +#define nvme_completion_is_error(cpl) \ + ((cpl)->status.sc != 0 || (cpl)->status.sct != 0) + +#endif diff --git a/include/spdk/pci.h b/include/spdk/pci.h new file mode 100644 index 000000000..c3af859c5 --- /dev/null +++ b/include/spdk/pci.h @@ -0,0 +1,48 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_PCI_H +#define SPDK_PCI_H + +#define PCI_CFG_SIZE 256 +#define PCI_EXT_CAP_ID_SN 0x03 +#define PCI_UIO_DRIVER "uio_pci_generic" + +int pci_device_get_serial_number(struct pci_device *dev, char *sn, int len); +int pci_device_has_uio_driver(struct pci_device *dev); +int pci_device_unbind_kernel_driver(struct pci_device *dev); +int pci_device_bind_uio_driver(struct pci_device *dev, char *driver_name); +int pci_device_switch_to_uio_driver(struct pci_device *pci_dev); +int pci_device_claim(struct pci_device *dev); + +#endif diff --git a/include/spdk/queue.h b/include/spdk/queue.h new file mode 100644 index 000000000..bd314fd55 --- /dev/null +++ b/include/spdk/queue.h @@ -0,0 +1,344 @@ +/*- + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + * $FreeBSD$ + */ + +#ifndef SPDK_QUEUE_H +#define SPDK_QUEUE_H + +#include +#include + +/* + * This file defines four types of data structures: singly-linked lists, + * singly-linked tail queues, lists and tail queues. + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A singly-linked tail queue is headed by a pair of pointers, one to the + * head of the list and the other to the tail of the list. The elements are + * singly linked for minimum space and pointer manipulation overhead at the + * expense of O(n) removal for arbitrary elements. New elements can be added + * to the list after an existing element, at the head of the list, or at the + * end of the list. Elements being removed from the head of the tail queue + * should use the explicit macro for this purpose for optimum efficiency. + * A singly-linked tail queue may only be traversed in the forward direction. + * Singly-linked tail queues are ideal for applications with large datasets + * and few or no removals or for implementing a FIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may be traversed in either direction. + * + * A tail queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or + * after an existing element, at the head of the list, or at the end of + * the list. A tail queue may be traversed in either direction. + * + * For details on the use of these macros, see the queue(3) manual page. + * + * + * SLIST LIST STAILQ TAILQ + * _HEAD + + + + + * _HEAD_INITIALIZER + + + + + * _ENTRY + + + + + * _INIT + + + + + * _EMPTY + + + + + * _FIRST + + + + + * _NEXT + + + + + * _PREV - + - + + * _LAST - - + + + * _FOREACH + + + + + * _FOREACH_FROM + + + + + * _FOREACH_SAFE + + + + + * _FOREACH_FROM_SAFE + + + + + * _FOREACH_REVERSE - - - + + * _FOREACH_REVERSE_FROM - - - + + * _FOREACH_REVERSE_SAFE - - - + + * _FOREACH_REVERSE_FROM_SAFE - - - + + * _INSERT_HEAD + + + + + * _INSERT_BEFORE - + - + + * _INSERT_AFTER + + + + + * _INSERT_TAIL - - + + + * _CONCAT - - + + + * _REMOVE_AFTER + - + - + * _REMOVE_HEAD + - + - + * _REMOVE + + + + + * _SWAP + + + + + * + */ + +/* + * Singly-linked Tail queue declarations. + */ +#define STAILQ_HEAD(name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ + struct type **stqh_last;/* addr of last next element */ \ +} + +#define STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).stqh_first } + +/* + * Singly-linked Tail queue functions. + */ +#define STAILQ_EMPTY(head) ((head)->stqh_first == NULL) + +#define STAILQ_FIRST(head) ((head)->stqh_first) + +#define STAILQ_FOREACH_FROM(var, head, field) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = STAILQ_FIRST((head)); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_FOREACH_FROM_SAFE(var, head, field, tvar) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_LAST(head, type, field) \ + (STAILQ_EMPTY((head)) ? NULL : \ + __containerof((head)->stqh_last, struct type, field.stqe_next)) + +#define STAILQ_NEXT(elm, field) ((elm)->field.stqe_next) + +#define STAILQ_REMOVE_AFTER(head, elm, field) do { \ + if ((STAILQ_NEXT(elm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_SWAP(head1, head2, type) do { \ + struct type *swap_first = STAILQ_FIRST(head1); \ + struct type **swap_last = (head1)->stqh_last; \ + STAILQ_FIRST(head1) = STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &STAILQ_FIRST(head1); \ + if (STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &STAILQ_FIRST(head2); \ +} while (0) + +/* + * List declarations. + */ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +/* + * List functions. + */ + +#if (defined(_KERNEL) && defined(INVARIANTS)) +#define QMD_LIST_CHECK_HEAD(head, field) do { \ + if (LIST_FIRST((head)) != NULL && \ + LIST_FIRST((head))->field.le_prev != \ + &LIST_FIRST((head))) \ + panic("Bad list head %p first->prev != head", (head)); \ +} while (0) + +#define QMD_LIST_CHECK_NEXT(elm, field) do { \ + if (LIST_NEXT((elm), field) != NULL && \ + LIST_NEXT((elm), field)->field.le_prev != \ + &((elm)->field.le_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +#define QMD_LIST_CHECK_PREV(elm, field) do { \ + if (*(elm)->field.le_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else +#define QMD_LIST_CHECK_HEAD(head, field) +#define QMD_LIST_CHECK_NEXT(elm, field) +#define QMD_LIST_CHECK_PREV(elm, field) +#endif /* (_KERNEL && INVARIANTS) */ + +#define LIST_EMPTY(head) ((head)->lh_first == NULL) + +#define LIST_FIRST(head) ((head)->lh_first) + +#define LIST_FOREACH_FROM(var, head, field) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_FOREACH_FROM_SAFE(var, head, field, tvar) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_NEXT(elm, field) ((elm)->field.le_next) + +#define LIST_PREV(elm, head, type, field) \ + ((elm)->field.le_prev == &LIST_FIRST((head)) ? NULL : \ + __containerof((elm)->field.le_prev, struct type, field.le_next)) + +#define LIST_SWAP(head1, head2, type, field) do { \ + struct type *swap_tmp = LIST_FIRST((head1)); \ + LIST_FIRST((head1)) = LIST_FIRST((head2)); \ + LIST_FIRST((head2)) = swap_tmp; \ + if ((swap_tmp = LIST_FIRST((head1))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head1)); \ + if ((swap_tmp = LIST_FIRST((head2))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head2)); \ +} while (0) + +/* + * Tail queue functions. + */ +#if (defined(_KERNEL) && defined(INVARIANTS)) +#define QMD_TAILQ_CHECK_HEAD(head, field) do { \ + if (!TAILQ_EMPTY(head) && \ + TAILQ_FIRST((head))->field.tqe_prev != \ + &TAILQ_FIRST((head))) \ + panic("Bad tailq head %p first->prev != head", (head)); \ +} while (0) + +#define QMD_TAILQ_CHECK_TAIL(head, field) do { \ + if (*(head)->tqh_last != NULL) \ + panic("Bad tailq NEXT(%p->tqh_last) != NULL", (head)); \ +} while (0) + +#define QMD_TAILQ_CHECK_NEXT(elm, field) do { \ + if (TAILQ_NEXT((elm), field) != NULL && \ + TAILQ_NEXT((elm), field)->field.tqe_prev != \ + &((elm)->field.tqe_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +#define QMD_TAILQ_CHECK_PREV(elm, field) do { \ + if (*(elm)->field.tqe_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else +#define QMD_TAILQ_CHECK_HEAD(head, field) +#define QMD_TAILQ_CHECK_TAIL(head, headname) +#define QMD_TAILQ_CHECK_NEXT(elm, field) +#define QMD_TAILQ_CHECK_PREV(elm, field) +#endif /* (_KERNEL && INVARIANTS) */ + +#define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) + +#define TAILQ_FIRST(head) ((head)->tqh_first) + +#define TAILQ_FOREACH_FROM(var, head, field) \ + for ((var) = ((var) ? (var) : TAILQ_FIRST((head))); \ + (var); \ + (var) = TAILQ_NEXT((var), field)) + +#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TAILQ_FIRST((head)); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define TAILQ_FOREACH_FROM_SAFE(var, head, field, tvar) \ + for ((var) = ((var) ? (var) : TAILQ_FIRST((head))); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define TAILQ_FOREACH_REVERSE_FROM(var, head, headname, field) \ + for ((var) = ((var) ? (var) : TAILQ_LAST((head), headname)); \ + (var); \ + (var) = TAILQ_PREV((var), headname, field)) + +#define TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, tvar) \ + for ((var) = TAILQ_LAST((head), headname); \ + (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1); \ + (var) = (tvar)) + +#define TAILQ_FOREACH_REVERSE_FROM_SAFE(var, head, headname, field, tvar) \ + for ((var) = ((var) ? (var) : TAILQ_LAST((head), headname)); \ + (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1); \ + (var) = (tvar)) + +#define TAILQ_LAST(head, headname) \ + (*(((struct headname *)((head)->tqh_last))->tqh_last)) + +#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) + +#define TAILQ_PREV(elm, headname, field) \ + (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) + +#define TAILQ_SWAP(head1, head2, type, field) do { \ + struct type *swap_first = (head1)->tqh_first; \ + struct type **swap_last = (head1)->tqh_last; \ + (head1)->tqh_first = (head2)->tqh_first; \ + (head1)->tqh_last = (head2)->tqh_last; \ + (head2)->tqh_first = swap_first; \ + (head2)->tqh_last = swap_last; \ + if ((swap_first = (head1)->tqh_first) != NULL) \ + swap_first->field.tqe_prev = &(head1)->tqh_first; \ + else \ + (head1)->tqh_last = &(head1)->tqh_first; \ + if ((swap_first = (head2)->tqh_first) != NULL) \ + swap_first->field.tqe_prev = &(head2)->tqh_first; \ + else \ + (head2)->tqh_last = &(head2)->tqh_first; \ +} while (0) + +#endif diff --git a/include/spdk/vtophys.h b/include/spdk/vtophys.h new file mode 100644 index 000000000..c27cd5e79 --- /dev/null +++ b/include/spdk/vtophys.h @@ -0,0 +1,51 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_VTOPHYS_H +#define SPDK_VTOPHYS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define VTOPHYS_ERROR (0xFFFFFFFFFFFFFFFFULL) + +uint64_t vtophys(void *buf); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/Makefile b/lib/Makefile new file mode 100644 index 000000000..495b12c6f --- /dev/null +++ b/lib/Makefile @@ -0,0 +1,43 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y += memory util nvme + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/lib/memory/Makefile b/lib/memory/Makefile new file mode 100644 index 000000000..e03249fef --- /dev/null +++ b/lib/memory/Makefile @@ -0,0 +1,55 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(DPDK_INC) + +C_SRCS = vtophys.c + +C_OBJS := $(patsubst %.c,%.o,$(C_SRCS)) + +OBJS = $(C_OBJS) + +LIB = libspdk_memory.a + +all : $(LIB) + +objs : $(OBJS) + +clean : + $(Q)rm -f $(LIB) $(OBJS) *.d + +$(LIB) : $(OBJS) + $(Q)ar crDs $(LIB) $(OBJS) + diff --git a/lib/memory/vtophys.c b/lib/memory/vtophys.c new file mode 100644 index 000000000..cc1a5baee --- /dev/null +++ b/lib/memory/vtophys.c @@ -0,0 +1,229 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define _LARGEFILE64_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spdk/vtophys.h" + +/* x86-64 userspace virtual addresses use only the low 47 bits [0..46], + * which is enough to cover 128 TB. + */ +#define SHIFT_128TB 47 /* (1 << 47) == 128 TB */ +#define MASK_128TB ((1ULL << SHIFT_128TB) - 1) + +#define SHIFT_1GB 30 /* (1 << 30) == 1 GB */ +#define MASK_1GB ((1ULL << SHIFT_1GB) - 1) + +#define SHIFT_2MB 21 /* (1 << 21) == 2MB */ +#define MASK_2MB ((1ULL << SHIFT_2MB) - 1) + +#define SHIFT_4KB 12 /* (1 << 12) == 4KB */ +#define MASK_4KB ((1ULL << SHIFT_4KB) - 1) + +#define FN_2MB_TO_4KB(fn) (fn << (SHIFT_2MB - SHIFT_4KB)) +#define FN_4KB_TO_2MB(fn) (fn >> (SHIFT_2MB - SHIFT_4KB)) + +#define MAP_128TB_IDX(vfn_2mb) ((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB)) +#define MAP_1GB_IDX(vfn_2mb) ((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB + 1)) - 1)) + +/* Defines related to Linux pagemap. */ +#define PAGEMAP_PFN_MASK 0x007FFFFFFFFFFFFFULL /* bits 54:0 */ + +/* Defines related to Linux kpageflags. */ +#define KPAGEFLAGS_HUGE 17 + +/* Physical page frame number of a single 2MB page. */ +struct map_2mb { + uint64_t pfn_2mb; +}; + +/* Second-level map table indexed by bits [21..29] of the virtual address. + * Each entry contains the 2MB physical page frame number or VTOPHYS_ERROR for entries that haven't + * been retrieved yet. + */ +struct map_1gb { + struct map_2mb map[1ULL << (SHIFT_1GB - SHIFT_2MB + 1)]; +}; + +/* Top-level map table indexed by bits [30..46] of the virtual address. + * Each entry points to a second-level map table or NULL. + */ +struct map_128tb { + struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)]; +}; + +static struct map_128tb vtophys_map_128tb = {}; +static pthread_mutex_t vtophys_mutex = PTHREAD_MUTEX_INITIALIZER; + +static struct map_2mb * +vtophys_get_map(uint64_t vfn_2mb) +{ + struct map_1gb *map_1gb; + struct map_2mb *map_2mb; + uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb); + uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb); + + if (vfn_2mb & ~MASK_128TB) { + printf("invalid usermode virtual address\n"); + return NULL; + } + + map_1gb = vtophys_map_128tb.map[idx_128tb]; + + if (!map_1gb) { + pthread_mutex_lock(&vtophys_mutex); + + /* Recheck to make sure nobody else got the mutex first. */ + map_1gb = vtophys_map_128tb.map[idx_128tb]; + if (!map_1gb) { + map_1gb = malloc(sizeof(struct map_1gb)); + /* initialize all entries to all 0xFF (VTOPHYS_ERROR) */ + memset(map_1gb, 0xFF, sizeof(struct map_1gb)); + vtophys_map_128tb.map[idx_128tb] = map_1gb; + } + + pthread_mutex_unlock(&vtophys_mutex); + + if (!map_1gb) { + printf("allocation failed\n"); + return NULL; + } + } + + map_2mb = &map_1gb->map[idx_1gb]; + return map_2mb; +} + +uint64_t +vtophys_get_pfn_2mb(uint64_t vfn_2mb) +{ + off64_t offset; + uint64_t pfn_4kb, pfn_2mb, pfn_flag_info; + static __thread int fd = -1; + static __thread int kpageflag_fd = -1; + + if (fd == -1) { + fd = open("/proc/self/pagemap", O_RDONLY); + if (fd == -1) { + perror("pagemap open failed"); + return VTOPHYS_ERROR; + } + } + + /* + * pagemap has a separate 8 byte entry for each 4KB + * page. So even though we are only storing a single + * PFN for every 2MB VFN, we have to get the 4KB + * VFN equivalent to lseek into the pagemap. + * + * vfn contains the 2MB virtual frame number, but we + * we need the 4KB equivalent to lseek into the pagemap. + * So do a left shift to convert the 2MB vfn to the + * 4KB vfn equivalent. + */ + offset = FN_2MB_TO_4KB(vfn_2mb) * sizeof(uint64_t); + + if (lseek64(fd, offset, SEEK_SET) < 0) { + perror("pagemap llseek failed"); + return VTOPHYS_ERROR; + } + + if (read(fd, &pfn_4kb, sizeof(pfn_4kb)) != sizeof(pfn_4kb)) { + perror("pagemap read failed"); + return VTOPHYS_ERROR; + } + + pfn_4kb &= PAGEMAP_PFN_MASK; + + if (kpageflag_fd == -1) { + kpageflag_fd = open("/proc/kpageflags", O_RDONLY); + if (kpageflag_fd == -1) { + perror("kpageflags open failed"); + return VTOPHYS_ERROR; + } + } + + offset = pfn_4kb * sizeof(uint64_t); + if (lseek64(kpageflag_fd, offset, SEEK_SET) < 0) { + perror("kpageflags llseek failed"); + return VTOPHYS_ERROR; + } + + if (read(kpageflag_fd, &pfn_flag_info, sizeof(pfn_flag_info)) != sizeof(pfn_flag_info)) { + perror("kpageflags read failed"); + return VTOPHYS_ERROR; + } + + /* check whether it is a huge page */ + if (!(pfn_flag_info & (1ULL << KPAGEFLAGS_HUGE))) { + printf("This is not a huge page\n"); + return VTOPHYS_ERROR; + } + + pfn_2mb = FN_4KB_TO_2MB(pfn_4kb); + return pfn_2mb; +} + +uint64_t +vtophys(void *buf) +{ + struct map_2mb *map_2mb; + uint64_t vfn_2mb, pfn_2mb; + + vfn_2mb = (uint64_t)buf; + vfn_2mb >>= SHIFT_2MB; + + map_2mb = vtophys_get_map(vfn_2mb); + if (!map_2mb) { + return VTOPHYS_ERROR; + } + + pfn_2mb = map_2mb->pfn_2mb; + if (pfn_2mb == VTOPHYS_ERROR) { + pfn_2mb = vtophys_get_pfn_2mb(vfn_2mb); + if (pfn_2mb == VTOPHYS_ERROR) { + return VTOPHYS_ERROR; + } + map_2mb->pfn_2mb = pfn_2mb; + } + + return (pfn_2mb << SHIFT_2MB) | ((uint64_t)buf & MASK_2MB); +} diff --git a/lib/nvme/Makefile b/lib/nvme/Makefile new file mode 100644 index 000000000..77bdf2004 --- /dev/null +++ b/lib/nvme/Makefile @@ -0,0 +1,53 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(DPDK_INC) -include $(CONFIG_NVME_IMPL) + +C_SRCS = nvme_ctrlr_cmd.c nvme_ctrlr.c nvme_ns_cmd.c nvme_ns.c nvme_qpair.c nvme.c + +C_OBJS := $(patsubst %.c,%.o,$(C_SRCS)) + +OBJS = $(C_OBJS) + +all : libspdk_nvme.a + +objs : $(OBJS) + +clean : + $(Q)rm -f libspdk_nvme.a $(OBJS) *.d + +libspdk_nvme.a : $(OBJS) + $(Q)ar crDs libspdk_nvme.a $(OBJS) + diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c new file mode 100644 index 000000000..ffd609482 --- /dev/null +++ b/lib/nvme/nvme.c @@ -0,0 +1,294 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/** \file + * + */ + +struct nvme_driver g_nvme_driver = { + .lock = NVME_MUTEX_INITIALIZER, + .max_io_queues = NVME_MAX_IO_QUEUES +}; + +int32_t nvme_retry_count; +int __thread nvme_thread_ioq_index = -1; + + +void +nvme_dump_command(struct nvme_command *cmd) +{ + printf( + "opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n", + cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid, + cmd->rsvd2, cmd->rsvd3, + (uintmax_t)cmd->mptr, (uintmax_t)cmd->dptr.prp.prp1, (uintmax_t)cmd->dptr.prp.prp2, + cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, + cmd->cdw15); +} + +void +nvme_dump_completion(struct nvme_completion *cpl) +{ + printf("cdw0:%08x sqhd:%04x sqid:%04x " + "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n", + cpl->cdw0, cpl->sqhd, cpl->sqid, + cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct, + cpl->status.m, cpl->status.dnr); +} + +/** + * \page nvme_initialization NVMe Initialization + +\msc + + app [label="Application"], nvme [label="NVMe Driver"]; + app=>nvme [label="nvme_attach(devhandle)"]; + app<nvme [label="nvme_ctrlr_start(nvme_controller ptr)"]; + nvme=>nvme [label="identify controller"]; + nvme=>nvme [label="create queue pairs"]; + nvme=>nvme [label="identify namespace(s)"]; + app=>app [label="create block devices based on controller's namespaces"]; + +\endmsc + + */ + +struct nvme_controller * +nvme_attach(void *devhandle) +{ + struct nvme_controller *ctrlr; + int status; + uint64_t phys_addr = 0; + + ctrlr = nvme_malloc("nvme_ctrlr", sizeof(struct nvme_controller), + 64, &phys_addr); + if (ctrlr == NULL) { + nvme_printf(NULL, "could not allocate ctrlr\n"); + return NULL; + } + + status = nvme_ctrlr_construct(ctrlr, devhandle); + if (status != 0) { + nvme_free(ctrlr); + return NULL; + } + + if (nvme_ctrlr_start(ctrlr) != 0) { + nvme_ctrlr_destruct(ctrlr); + nvme_free(ctrlr); + return NULL; + } + + return ctrlr; +} + +int +nvme_detach(struct nvme_controller *ctrlr) +{ + nvme_ctrlr_destruct(ctrlr); + nvme_free(ctrlr); + return 0; +} + +void +nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl) +{ + struct nvme_completion_poll_status *status = arg; + + /* + * Copy status into the argument passed by the caller, so that + * the caller can check the status to determine if the + * the request passed or failed. + */ + memcpy(&status->cpl, cpl, sizeof(*cpl)); + status->done = true; +} + +size_t +nvme_request_size(void) +{ + return sizeof(struct nvme_request); +} + +struct nvme_request * +nvme_allocate_request(void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req = NULL; + + nvme_alloc_request(&req); + + if (req == NULL) { + return req; + } + + /* + * Only memset up to (but not including) the children + * TAILQ_ENTRY. children, and following members, are + * only used as part of I/O splitting so we avoid + * memsetting them until it is actually needed. + */ + memset(req, 0, offsetof(struct nvme_request, children)); + req->cb_fn = cb_fn; + req->cb_arg = cb_arg; + req->timeout = true; + nvme_assert((payload == NULL && payload_size == 0) || + (payload != NULL && payload_size != 0), + ("Invalid argument combination of payload and payload_size\n")); + if (payload == NULL || payload_size == 0) { + req->u.payload = NULL; + req->payload_size = 0; + } else { + req->u.payload = payload; + req->payload_size = payload_size; + } + + return req; +} + +void +nvme_cb_complete_child(void *child_arg, const struct nvme_completion *cpl) +{ + struct nvme_request *child = child_arg; + struct nvme_request *parent = child->parent; + + parent->num_children--; + TAILQ_REMOVE(&parent->children, child, child_tailq); + + if (nvme_completion_is_error(cpl)) { + memcpy(&parent->parent_status, cpl, sizeof(*cpl)); + } + + if (parent->num_children == 0) { + if (parent->cb_fn) { + parent->cb_fn(parent->cb_arg, &parent->parent_status); + } + nvme_free_request(parent); + } +} + +void +nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child) +{ + if (parent->num_children == 0) { + /* + * Defer initialization of the children TAILQ since it falls + * on a separate cacheline. This ensures we do not touch this + * cacheline except on request splitting cases, which are + * relatively rare. + */ + TAILQ_INIT(&parent->children); + memset(&parent->parent_status, 0, sizeof(struct nvme_completion)); + } + + parent->num_children++; + TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); + child->parent = parent; + child->cb_fn = nvme_cb_complete_child; + child->cb_arg = child; +} + +static int +nvme_allocate_ioq_index(void) +{ + struct nvme_driver *driver = &g_nvme_driver; + uint32_t i; + + nvme_mutex_lock(&driver->lock); + if (driver->ioq_index_pool == NULL) { + driver->ioq_index_pool = + calloc(driver->max_io_queues, sizeof(*driver->ioq_index_pool)); + if (driver->ioq_index_pool) { + for (i = 0; i < driver->max_io_queues; i++) { + driver->ioq_index_pool[i] = i; + } + } else { + nvme_mutex_unlock(&driver->lock); + return -1; + } + driver->ioq_index_pool_next = 0; + } + + if (driver->ioq_index_pool_next < driver->max_io_queues) { + nvme_thread_ioq_index = driver->ioq_index_pool[driver->ioq_index_pool_next]; + driver->ioq_index_pool[driver->ioq_index_pool_next] = -1; + driver->ioq_index_pool_next++; + } else { + nvme_thread_ioq_index = -1; + } + + nvme_mutex_unlock(&driver->lock); + return 0; +} + +static void +nvme_free_ioq_index(void) +{ + struct nvme_driver *driver = &g_nvme_driver; + + nvme_mutex_lock(&driver->lock); + if (nvme_thread_ioq_index >= 0) { + driver->ioq_index_pool_next--; + driver->ioq_index_pool[driver->ioq_index_pool_next] = nvme_thread_ioq_index; + nvme_thread_ioq_index = -1; + } + nvme_mutex_unlock(&driver->lock); +} + +int +nvme_register_io_thread(void) +{ + int rc = 0; + + if (nvme_thread_ioq_index >= 0) { + nvme_printf(NULL, "thread already registered\n"); + return -1; + } + + rc = nvme_allocate_ioq_index(); + if (rc) { + nvme_printf(NULL, "ioq_index_pool alloc failed\n"); + return rc; + } + return (nvme_thread_ioq_index >= 0) ? 0 : -1; +} + +void +nvme_unregister_io_thread(void) +{ + nvme_free_ioq_index(); +} + diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c new file mode 100644 index 000000000..68dd3a0b0 --- /dev/null +++ b/lib/nvme/nvme_ctrlr.c @@ -0,0 +1,787 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/** + * \file + * + */ + +static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr, + struct nvme_async_event_request *aer); + +static int +nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr) +{ + struct nvme_qpair *qpair; + int rc; + + qpair = &ctrlr->adminq; + + /* + * The admin queue's max xfer size is treated differently than the + * max I/O xfer size. 16KB is sufficient here - maybe even less? + */ + rc = nvme_qpair_construct(qpair, + 0, /* qpair ID */ + NVME_ADMIN_ENTRIES, + NVME_ADMIN_TRACKERS, + ctrlr); + return rc; +} + +static int +nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr) +{ + struct nvme_qpair *qpair; + union nvme_cap_lo_register cap_lo; + int i, num_entries, num_trackers, rc; + + rc = 0; + if (ctrlr->ioq != NULL) { + /* + * io_qpairs were already constructed, so just return. + * This typically happens when the controller is + * initialized a second (or subsequent) time after a + * controller reset. + */ + return 0; + } + + /* + * NVMe spec sets a hard limit of 64K max entries, but + * devices may specify a smaller limit, so we need to check + * the MQES field in the capabilities register. + */ + cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo.raw); + num_entries = nvme_min(NVME_IO_ENTRIES, cap_lo.bits.mqes + 1); + + /* + * No need to have more trackers than entries in the submit queue. + * Note also that for a queue size of N, we can only have (N-1) + * commands outstanding, hence the "-1" here. + */ + num_trackers = nvme_min(NVME_IO_TRACKERS, (num_entries - 1)); + + ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; + + ctrlr->ioq = calloc(ctrlr->num_io_queues, sizeof(struct nvme_qpair)); + + if (ctrlr->ioq == NULL) + return -1; + + for (i = 0; i < ctrlr->num_io_queues; i++) { + qpair = &ctrlr->ioq[i]; + + /* + * Admin queue has ID=0. IO queues start at ID=1 - + * hence the 'i+1' here. + * + * For I/O queues, use the controller-wide max_xfer_size + * calculated in nvme_attach(). + */ + rc = nvme_qpair_construct(qpair, + i + 1, /* qpair ID */ + num_entries, + num_trackers, + ctrlr); + if (rc) + return -1; + } + + return 0; +} + +static void +nvme_ctrlr_fail(struct nvme_controller *ctrlr) +{ + int i; + + ctrlr->is_failed = true; + nvme_qpair_fail(&ctrlr->adminq); + for (i = 0; i < ctrlr->num_io_queues; i++) { + nvme_qpair_fail(&ctrlr->ioq[i]); + } +} + +static int +_nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_ready_value) +{ + int ms_waited, ready_timeout_in_ms; + union nvme_csts_register csts; + union nvme_cap_lo_register cap_lo; + + /* Get ready timeout value from controller, in units of 500ms. */ + cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo.raw); + ready_timeout_in_ms = cap_lo.bits.to * 500; + + csts.raw = nvme_mmio_read_4(ctrlr, csts); + + ms_waited = 0; + + while (csts.bits.rdy != desired_ready_value) { + nvme_delay(1000); + if (ms_waited++ > ready_timeout_in_ms) { + nvme_printf(ctrlr, "controller ready did not become %d " + "within %d ms\n", desired_ready_value, ready_timeout_in_ms); + return ENXIO; + } + csts.raw = nvme_mmio_read_4(ctrlr, csts); + } + + return 0; +} + +static int +nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr) +{ + union nvme_cc_register cc; + + cc.raw = nvme_mmio_read_4(ctrlr, cc.raw); + + if (!cc.bits.en) { + nvme_printf(ctrlr, "%s called with cc.en = 0\n", __func__); + return ENXIO; + } + + return _nvme_ctrlr_wait_for_ready(ctrlr, 1); +} + +static void +nvme_ctrlr_disable(struct nvme_controller *ctrlr) +{ + union nvme_cc_register cc; + union nvme_csts_register csts; + + cc.raw = nvme_mmio_read_4(ctrlr, cc.raw); + csts.raw = nvme_mmio_read_4(ctrlr, csts); + + if (cc.bits.en == 1 && csts.bits.rdy == 0) { + _nvme_ctrlr_wait_for_ready(ctrlr, 1); + } + + cc.bits.en = 0; + nvme_mmio_write_4(ctrlr, cc.raw, cc.raw); + nvme_delay(5000); + + _nvme_ctrlr_wait_for_ready(ctrlr, 0); +} + +static void +nvme_ctrlr_shutdown(struct nvme_controller *ctrlr) +{ + union nvme_cc_register cc; + union nvme_csts_register csts; + int ms_waited = 0; + + cc.raw = nvme_mmio_read_4(ctrlr, cc.raw); + cc.bits.shn = NVME_SHN_NORMAL; + nvme_mmio_write_4(ctrlr, cc.raw, cc.raw); + + csts.raw = nvme_mmio_read_4(ctrlr, csts); + /* + * The NVMe spec does not define a timeout period + * for shutdown notification, so we just pick + * 5 seconds as a reasonable amount of time to + * wait before proceeding. + */ + while (csts.bits.shst != NVME_SHST_COMPLETE) { + nvme_delay(1000); + csts.raw = nvme_mmio_read_4(ctrlr, csts); + if (ms_waited++ >= 5000) + break; + } + if (csts.bits.shst != NVME_SHST_COMPLETE) + nvme_printf(ctrlr, "did not shutdown within 5 seconds\n"); +} + +static int +nvme_ctrlr_enable(struct nvme_controller *ctrlr) +{ + union nvme_cc_register cc; + union nvme_csts_register csts; + union nvme_aqa_register aqa; + + cc.raw = nvme_mmio_read_4(ctrlr, cc.raw); + csts.raw = nvme_mmio_read_4(ctrlr, csts); + + if (cc.bits.en == 1) { + if (csts.bits.rdy == 1) { + return 0; + } else { + return nvme_ctrlr_wait_for_ready(ctrlr); + } + } + + nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr); + nvme_delay(5000); + nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr); + nvme_delay(5000); + + aqa.raw = 0; + /* acqs and asqs are 0-based. */ + aqa.bits.acqs = ctrlr->adminq.num_entries - 1; + aqa.bits.asqs = ctrlr->adminq.num_entries - 1; + nvme_mmio_write_4(ctrlr, aqa.raw, aqa.raw); + nvme_delay(5000); + + cc.bits.en = 1; + cc.bits.css = 0; + cc.bits.ams = 0; + cc.bits.shn = 0; + cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ + cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ + + /* Page size is 2 ^ (12 + mps). */ + cc.bits.mps = nvme_u32log2(PAGE_SIZE) - 12; + + nvme_mmio_write_4(ctrlr, cc.raw, cc.raw); + nvme_delay(5000); + + return nvme_ctrlr_wait_for_ready(ctrlr); +} + +int +nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr) +{ + int i, rc; + union nvme_cc_register cc; + + cc.raw = nvme_mmio_read_4(ctrlr, cc.raw); + if (cc.bits.en) { + nvme_qpair_disable(&ctrlr->adminq); + for (i = 0; i < ctrlr->num_io_queues; i++) { + nvme_qpair_disable(&ctrlr->ioq[i]); + } + + nvme_delay(100 * 1000); + } else { + /* + * Ensure we do a transition from cc.en==1 to cc.en==0. + * If we started disabled (cc.en==0), then we have to enable + * first to get a reset. + */ + nvme_ctrlr_enable(ctrlr); + } + + nvme_ctrlr_disable(ctrlr); + rc = nvme_ctrlr_enable(ctrlr); + + nvme_delay(100 * 1000); + + return rc; +} + +int +nvme_ctrlr_reset(struct nvme_controller *ctrlr) +{ + int rc; + + nvme_mutex_lock(&ctrlr->ctrlr_lock); + + if (ctrlr->is_resetting || ctrlr->is_failed) { + /* + * Controller is already resetting or has failed. Return + * immediately since there is no need to kick off another + * reset in these cases. + */ + nvme_mutex_unlock(&ctrlr->ctrlr_lock); + return 0; + } + + ctrlr->is_resetting = 1; + + nvme_printf(ctrlr, "resetting controller\n"); + /* nvme_ctrlr_start() issues a reset as its first step */ + rc = nvme_ctrlr_start(ctrlr); + if (rc) { + nvme_ctrlr_fail(ctrlr); + } + + ctrlr->is_resetting = 0; + + nvme_mutex_unlock(&ctrlr->ctrlr_lock); + + return rc; +} + +static int +nvme_ctrlr_identify(struct nvme_controller *ctrlr) +{ + struct nvme_completion_poll_status status; + + status.done = false; + nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata, + nvme_completion_poll_cb, &status); + while (status.done == false) { + nvme_qpair_process_completions(&ctrlr->adminq); + } + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_identify_controller failed!\n"); + return ENXIO; + } + + /* + * Use MDTS to ensure our default max_xfer_size doesn't exceed what the + * controller supports. + */ + if (ctrlr->cdata.mdts > 0) { + ctrlr->max_xfer_size = nvme_min(ctrlr->max_xfer_size, + ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts))); + } + + return 0; +} + +static int +nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr) +{ + struct nvme_driver *driver = &g_nvme_driver; + struct nvme_completion_poll_status status; + int cq_allocated, sq_allocated; + uint32_t max_io_queues; + + status.done = false; + + nvme_mutex_lock(&driver->lock); + max_io_queues = driver->max_io_queues; + nvme_mutex_unlock(&driver->lock); + + nvme_ctrlr_cmd_set_num_queues(ctrlr, max_io_queues, + nvme_completion_poll_cb, &status); + while (status.done == false) { + nvme_qpair_process_completions(&ctrlr->adminq); + } + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_set_num_queues failed!\n"); + return ENXIO; + } + + /* + * Data in cdw0 is 0-based. + * Lower 16-bits indicate number of submission queues allocated. + * Upper 16-bits indicate number of completion queues allocated. + */ + sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1; + cq_allocated = (status.cpl.cdw0 >> 16) + 1; + + ctrlr->num_io_queues = nvme_min(sq_allocated, cq_allocated); + + nvme_mutex_lock(&driver->lock); + driver->max_io_queues = nvme_min(driver->max_io_queues, ctrlr->num_io_queues); + nvme_mutex_unlock(&driver->lock); + + return 0; +} + +static int +nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr) +{ + struct nvme_completion_poll_status status; + struct nvme_qpair *qpair; + int i; + + if (nvme_ctrlr_construct_io_qpairs(ctrlr)) { + nvme_printf(ctrlr, "nvme_ctrlr_construct_io_qpairs failed!\n"); + return ENXIO; + } + + for (i = 0; i < ctrlr->num_io_queues; i++) { + qpair = &ctrlr->ioq[i]; + + status.done = false; + nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, + nvme_completion_poll_cb, &status); + while (status.done == false) { + nvme_qpair_process_completions(&ctrlr->adminq); + } + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_create_io_cq failed!\n"); + return ENXIO; + } + + status.done = false; + nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, + nvme_completion_poll_cb, &status); + while (status.done == false) { + nvme_qpair_process_completions(&ctrlr->adminq); + } + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_create_io_sq failed!\n"); + return ENXIO; + } + + nvme_qpair_reset(qpair); + } + + return 0; +} + +static void +nvme_ctrlr_destruct_namespaces(struct nvme_controller *ctrlr) +{ + if (ctrlr->ns) { + uint32_t i, num_ns = ctrlr->num_ns; + + for (i = 0; i < num_ns; i++) { + nvme_ns_destruct(&ctrlr->ns[i]); + } + + free(ctrlr->ns); + ctrlr->ns = NULL; + ctrlr->num_ns = 0; + } + + if (ctrlr->nsdata) { + nvme_free(ctrlr->nsdata); + ctrlr->nsdata = NULL; + } +} + +static int +nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr) +{ + uint32_t i, nn = ctrlr->cdata.nn; + uint64_t phys_addr = 0; + + if (nn == 0) { + nvme_printf(ctrlr, "controller has 0 namespaces\n"); + return -1; + } + + /* ctrlr->num_ns may be 0 (startup) or a different number of namespaces (reset), + * so check if we need to reallocate. + */ + if (nn != ctrlr->num_ns) { + nvme_ctrlr_destruct_namespaces(ctrlr); + + ctrlr->ns = calloc(nn, sizeof(struct nvme_namespace)); + if (ctrlr->ns == NULL) { + goto fail; + } + + ctrlr->nsdata = nvme_malloc("nvme_namespaces", + nn * sizeof(struct nvme_namespace_data), 64, + &phys_addr); + if (ctrlr->nsdata == NULL) { + goto fail; + } + + ctrlr->num_ns = nn; + } + + for (i = 0; i < nn; i++) { + struct nvme_namespace *ns = &ctrlr->ns[i]; + uint32_t nsid = i + 1; + + if (nvme_ns_construct(ns, nsid, ctrlr) != 0) { + goto fail; + } + } + + return 0; + +fail: + nvme_ctrlr_destruct_namespaces(ctrlr); + return -1; +} + +static void +nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl) +{ + struct nvme_async_event_request *aer = arg; + struct nvme_controller *ctrlr = aer->ctrlr; + + if (cpl->status.sc == NVME_SC_ABORTED_SQ_DELETION) { + /* + * This is simulated when controller is being shut down, to + * effectively abort outstanding asynchronous event requests + * and make sure all memory is freed. Do not repost the + * request in this case. + */ + return; + } + + if (ctrlr->aer_cb_fn != NULL) { + ctrlr->aer_cb_fn(ctrlr->aer_cb_arg, cpl); + } + + /* + * Repost another asynchronous event request to replace the one + * that just completed. + */ + nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer); +} + +static void +nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr, + struct nvme_async_event_request *aer) +{ + struct nvme_request *req; + + aer->ctrlr = ctrlr; + req = nvme_allocate_request(NULL, 0, nvme_ctrlr_async_event_cb, aer); + aer->req = req; + + /* + * Disable timeout here, since asynchronous event requests should by + * nature never be timed out. + */ + req->timeout = false; + req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST; + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +static int +nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr) +{ + union nvme_critical_warning_state state; + struct nvme_async_event_request *aer; + uint32_t i; + struct nvme_completion_poll_status status; + + status.done = false; + + state.raw = 0xFF; + state.bits.reserved = 0; + nvme_ctrlr_cmd_set_async_event_config(ctrlr, state, nvme_completion_poll_cb, &status); + + while (status.done == false) { + nvme_qpair_process_completions(&ctrlr->adminq); + } + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_ctrlr_cmd_set_async_event_config failed!\n"); + return ENXIO; + } + + /* aerl is a zero-based value, so we need to add 1 here. */ + ctrlr->num_aers = nvme_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1)); + + for (i = 0; i < ctrlr->num_aers; i++) { + aer = &ctrlr->aer[i]; + nvme_ctrlr_construct_and_submit_aer(ctrlr, aer); + } + + return 0; +} + +int +nvme_ctrlr_start(struct nvme_controller *ctrlr) +{ + if (nvme_ctrlr_hw_reset(ctrlr) != 0) { + return -1; + } + + nvme_qpair_reset(&ctrlr->adminq); + + nvme_qpair_enable(&ctrlr->adminq); + + if (nvme_ctrlr_identify(ctrlr) != 0) { + return -1; + } + + if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) { + return -1; + } + + if (nvme_ctrlr_create_qpairs(ctrlr) != 0) { + return -1; + } + + if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) { + return -1; + } + + if (nvme_ctrlr_configure_aer(ctrlr) != 0) { + return -1; + } + + return 0; +} + +static int +nvme_ctrlr_allocate_bars(struct nvme_controller *ctrlr) +{ + int rc; + void *addr; + + rc = nvme_pcicfg_map_bar(ctrlr->devhandle, 0, 0 /* writable */, &addr); + ctrlr->regs = (volatile struct nvme_registers *)addr; + if ((ctrlr->regs == NULL) || (rc != 0)) { + printf("pci_device_map_range failed with error code %d\n", rc); + return -1; + } + + return 0; +} + +static int +nvme_ctrlr_free_bars(struct nvme_controller *ctrlr) +{ + int rc = 0; + void *addr = (void *)ctrlr->regs; + + if (addr) { + rc = nvme_pcicfg_unmap_bar(ctrlr->devhandle, 0, addr); + } + return rc; +} + +int +nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle) +{ + union nvme_cap_hi_register cap_hi; + uint32_t cmd_reg; + int status; + int rc; + + ctrlr->devhandle = devhandle; + + status = nvme_ctrlr_allocate_bars(ctrlr); + if (status != 0) { + return status; + } + + /* Enable PCI busmaster. */ + nvme_pcicfg_read32(devhandle, &cmd_reg, 4); + cmd_reg |= 0x4; + nvme_pcicfg_write32(devhandle, cmd_reg, 4); + + cap_hi.raw = nvme_mmio_read_4(ctrlr, cap_hi.raw); + + /* Doorbell stride is 2 ^ (dstrd + 2), + * but we want multiples of 4, so drop the + 2 */ + ctrlr->doorbell_stride_u32 = 1 << cap_hi.bits.dstrd; + + ctrlr->min_page_size = 1 << (12 + cap_hi.bits.mpsmin); + + rc = nvme_ctrlr_construct_admin_qpair(ctrlr); + if (rc) + return rc; + + ctrlr->is_resetting = 0; + ctrlr->is_failed = false; + + nvme_mutex_init_recursive(&ctrlr->ctrlr_lock); + + return 0; +} + +void +nvme_ctrlr_destruct(struct nvme_controller *ctrlr) +{ + int i; + + nvme_ctrlr_disable(ctrlr); + nvme_ctrlr_shutdown(ctrlr); + + nvme_ctrlr_destruct_namespaces(ctrlr); + + for (i = 0; i < ctrlr->num_io_queues; i++) { + nvme_qpair_destroy(&ctrlr->ioq[i]); + } + + free(ctrlr->ioq); + + nvme_qpair_destroy(&ctrlr->adminq); + + nvme_ctrlr_free_bars(ctrlr); + nvme_mutex_destroy(&ctrlr->ctrlr_lock); +} + +void +nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, + struct nvme_request *req) +{ + nvme_qpair_submit_request(&ctrlr->adminq, req); +} + +void +nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, + struct nvme_request *req) +{ + struct nvme_qpair *qpair; + + nvme_assert(nvme_thread_ioq_index >= 0, ("no ioq_index assigned for thread\n")); + qpair = &ctrlr->ioq[nvme_thread_ioq_index]; + + nvme_qpair_submit_request(qpair, req); +} + +void +nvme_ctrlr_process_io_completions(struct nvme_controller *ctrlr) +{ + nvme_assert(nvme_thread_ioq_index >= 0, ("no ioq_index assigned for thread\n")); + nvme_qpair_process_completions(&ctrlr->ioq[nvme_thread_ioq_index]); +} + +void +nvme_ctrlr_process_admin_completions(struct nvme_controller *ctrlr) +{ + nvme_mutex_lock(&ctrlr->ctrlr_lock); + nvme_qpair_process_completions(&ctrlr->adminq); + nvme_mutex_unlock(&ctrlr->ctrlr_lock); +} + +const struct nvme_controller_data * +nvme_ctrlr_get_data(struct nvme_controller *ctrlr) +{ + + return &ctrlr->cdata; +} + +uint32_t +nvme_ctrlr_get_num_ns(struct nvme_controller *ctrlr) +{ + return ctrlr->num_ns; +} + +struct nvme_namespace * +nvme_ctrlr_get_ns(struct nvme_controller *ctrlr, uint32_t ns_id) +{ + if (ns_id < 1 || ns_id > ctrlr->num_ns) { + return NULL; + } + + return &ctrlr->ns[ns_id - 1]; +} + +void +nvme_ctrlr_register_aer_callback(struct nvme_controller *ctrlr, + nvme_aer_cb_fn_t aer_cb_fn, + void *aer_cb_arg) +{ + ctrlr->aer_cb_fn = aer_cb_fn; + ctrlr->aer_cb_arg = aer_cb_arg; +} diff --git a/lib/nvme/nvme_ctrlr_cmd.c b/lib/nvme/nvme_ctrlr_cmd.c new file mode 100644 index 000000000..bf4232afb --- /dev/null +++ b/lib/nvme/nvme_ctrlr_cmd.c @@ -0,0 +1,310 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +int +nvme_ctrlr_cmd_io_raw(struct nvme_controller *ctrlr, + struct nvme_command *cmd, + void *buf, uint32_t len, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = nvme_allocate_request(buf, len, cb_fn, cb_arg); + + if (req == NULL) { + return ENOMEM; + } + + memcpy(&req->cmd, cmd, sizeof(req->cmd)); + + nvme_ctrlr_submit_io_request(ctrlr, req); + return 0; +} + +int +nvme_ctrlr_cmd_admin_raw(struct nvme_controller *ctrlr, + struct nvme_command *cmd, + void *buf, uint32_t len, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + nvme_mutex_lock(&ctrlr->ctrlr_lock); + req = nvme_allocate_request(buf, len, cb_fn, cb_arg); + if (req == NULL) { + nvme_mutex_unlock(&ctrlr->ctrlr_lock); + return ENOMEM; + } + + memcpy(&req->cmd, cmd, sizeof(req->cmd)); + + nvme_ctrlr_submit_admin_request(ctrlr, req); + + nvme_mutex_unlock(&ctrlr->ctrlr_lock); + return 0; +} + +void +nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(payload, + sizeof(struct nvme_controller_data), + cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_IDENTIFY; + + /* + * TODO: create an identify command data structure, which + * includes this CNS bit in cdw10. + */ + cmd->cdw10 = 1; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, uint16_t nsid, + void *payload, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(payload, + sizeof(struct nvme_namespace_data), + cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_IDENTIFY; + + /* + * TODO: create an identify command data structure + */ + cmd->nsid = nsid; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr, + struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, + void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_CREATE_IO_CQ; + + /* + * TODO: create a create io completion queue command data + * structure. + */ + cmd->cdw10 = ((io_que->num_entries - 1) << 16) | io_que->id; + /* + * 0x2 = interrupts enabled + * 0x1 = physically contiguous + */ + cmd->cdw11 = (io_que->id << 16) | 0x1; + cmd->dptr.prp.prp1 = io_que->cpl_bus_addr; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr, + struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_CREATE_IO_SQ; + + /* + * TODO: create a create io submission queue command data + * structure. + */ + cmd->cdw10 = ((io_que->num_entries - 1) << 16) | io_que->id; + /* 0x1 = physically contiguous */ + cmd->cdw11 = (io_que->id << 16) | 0x1; + cmd->dptr.prp.prp1 = io_que->cmd_bus_addr; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature, + uint32_t cdw11, void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_SET_FEATURES; + cmd->cdw10 = feature; + cmd->cdw11 = cdw11; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature, + uint32_t cdw11, void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_GET_FEATURES; + cmd->cdw10 = feature; + cmd->cdw11 = cdw11; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr, + uint32_t num_queues, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + uint32_t cdw11; + + cdw11 = ((num_queues - 1) << 16) | (num_queues - 1); + nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_NUMBER_OF_QUEUES, cdw11, + NULL, 0, cb_fn, cb_arg); +} + +void +nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr, + union nvme_critical_warning_state state, nvme_cb_fn_t cb_fn, + void *cb_arg) +{ + uint32_t cdw11; + + cdw11 = state.raw; + nvme_ctrlr_cmd_set_feature(ctrlr, + NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, NULL, 0, cb_fn, + cb_arg); +} + +void +nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, uint8_t log_page, + uint32_t nsid, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, + void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(payload, payload_size, cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_GET_LOG_PAGE; + cmd->nsid = nsid; + cmd->cdw10 = ((payload_size / sizeof(uint32_t)) - 1) << 16; + cmd->cdw10 |= log_page; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} + +void +nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr, + struct nvme_error_information_entry *payload, uint32_t num_entries, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + + nvme_assert(num_entries > 0, ("%s called with num_entries==0\n", __func__)); + + /* Controller's error log page entries is 0-based. */ + nvme_assert(num_entries <= (ctrlr->cdata.elpe + 1), + ("%s called with num_entries=%d but (elpe+1)=%d\n", __func__, + num_entries, ctrlr->cdata.elpe + 1)); + + if (num_entries > (ctrlr->cdata.elpe + 1)) + num_entries = ctrlr->cdata.elpe + 1; + + nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_ERROR, + NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload) * num_entries, + cb_fn, cb_arg); +} + +void +nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr, + uint32_t nsid, struct nvme_health_information_page *payload, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + + nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_HEALTH_INFORMATION, + nsid, payload, sizeof(*payload), cb_fn, cb_arg); +} + +void +nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr, + struct nvme_firmware_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + + nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_FIRMWARE_SLOT, + NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload), cb_fn, + cb_arg); +} + +void +nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid, + uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + cmd = &req->cmd; + cmd->opc = NVME_OPC_ABORT; + cmd->cdw10 = (cid << 16) | sqid; + + nvme_ctrlr_submit_admin_request(ctrlr, req); +} diff --git a/lib/nvme/nvme_impl.h b/lib/nvme/nvme_impl.h new file mode 100644 index 000000000..8ece20d51 --- /dev/null +++ b/lib/nvme/nvme_impl.h @@ -0,0 +1,164 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NVME_IMPL_H__ +#define __NVME_IMPL_H__ + +#include "spdk/vtophys.h" +#include +#include +#include +#include +#include + +/** + * \file + * + * This file describes the callback functions required to integrate + * the userspace NVMe driver for a specific implementation. This + * implementation is specific for DPDK for Storage. Users would + * revise it as necessary for their own particular environment if not + * using it within the DPDK for Storage framework. + */ + +/** + * \page nvme_driver_integration NVMe Driver Integration + * + * Users can integrate the userspace NVMe driver into their environment + * by implementing the callbacks in nvme_impl.h. These callbacks + * enable users to specify how to allocate pinned and physically + * contiguous memory, performance virtual to physical address + * translations, log messages, PCI configuration and register mapping, + * and a number of other facilities that may differ depending on the + * environment. + */ + +/** + * Allocate a pinned, physically contiguous memory buffer with the + * given size and alignment. + * Note: these calls are only made during driver initialization. Per + * I/O allocations during driver operation use the nvme_alloc_request + * callback. + */ +static inline void * +nvme_malloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr) +{ + void *buf = rte_zmalloc(tag, size, align); + *phys_addr = rte_malloc_virt2phy(buf); + return buf; +} + +/** + * Free a memory buffer previously allocated with nvme_malloc. + */ +#define nvme_free(buf) rte_free(buf) + +/** + * Log or print a message from the NVMe driver. + */ +#define nvme_printf(ctrlr, fmt, args...) printf(fmt, ##args) + +/** + * Assert a condition and panic/abort as desired. Failures of these + * assertions indicate catastrophic failures within the driver. + */ +#define nvme_assert(check, str) assert(check) + +/** + * Return the physical address for the specified virtual address. + */ +#define nvme_vtophys(buf) vtophys(buf) + +extern struct rte_mempool *request_mempool; + +/** + * Return a buffer for an nvme_request object. These objects are allocated + * for each I/O. They do not need to be pinned nor physically contiguous. + */ +#define nvme_alloc_request(bufp) rte_mempool_get(request_mempool, (void **)(bufp)); + +/** + * Free a buffer previously allocated with nvme_alloc_request(). + */ +#define nvme_free_request(buf) rte_mempool_put(request_mempool, buf) + +/** + * + */ +#define nvme_pcicfg_read32(handle, var, offset) pci_device_cfg_read_u32(handle, var, offset) +#define nvme_pcicfg_write32(handle, var, offset) pci_device_cfg_write_u32(handle, var, offset) + +static inline int +nvme_pcicfg_map_bar(void *devhandle, uint32_t bar, uint32_t read_only, void **mapped_addr) +{ + struct pci_device *dev = devhandle; + uint32_t flags = (read_only ? 0 : PCI_DEV_MAP_FLAG_WRITABLE); + + return pci_device_map_range(dev, dev->regions[bar].base_addr, dev->regions[bar].size, + flags, mapped_addr); +} + +static inline int +nvme_pcicfg_unmap_bar(void *devhandle, uint32_t bar, void *addr) +{ + struct pci_device *dev = devhandle; + + return pci_device_unmap_range(dev, addr, dev->regions[bar].size); +} + +typedef pthread_mutex_t nvme_mutex_t; + +#define nvme_mutex_init(x) pthread_mutex_init((x), NULL) +#define nvme_mutex_destroy(x) pthread_mutex_destroy((x)) +#define nvme_mutex_lock pthread_mutex_lock +#define nvme_mutex_unlock pthread_mutex_unlock +#define NVME_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +static inline int +nvme_mutex_init_recursive(nvme_mutex_t *mtx) +{ + pthread_mutexattr_t attr; + int rc = 0; + + if (pthread_mutexattr_init(&attr)) { + return -1; + } + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) || + pthread_mutex_init(mtx, &attr)) { + rc = -1; + } + pthread_mutexattr_destroy(&attr); + return rc; +} + +#endif /* __NVME_IMPL_H__ */ diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h new file mode 100644 index 000000000..b96559c92 --- /dev/null +++ b/lib/nvme/nvme_internal.h @@ -0,0 +1,442 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NVME_INTERNAL_H__ +#define __NVME_INTERNAL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "spdk/nvme.h" + +#include "spdk/queue.h" +#include "spdk/barrier.h" + +#define NVME_MAX_PRP_LIST_ENTRIES (32) + +/* + * For commands requiring more than 2 PRP entries, one PRP will be + * embedded in the command (prp1), and the rest of the PRP entries + * will be in a list pointed to by the command (prp2). This means + * that real max number of PRP entries we support is 32+1, which + * results in a max xfer size of 32*PAGE_SIZE. + */ +#define NVME_MAX_XFER_SIZE NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE + +#define NVME_ADMIN_TRACKERS (16) +#define NVME_ADMIN_ENTRIES (128) +/* min and max are defined in admin queue attributes section of spec */ +#define NVME_MIN_ADMIN_ENTRIES (2) +#define NVME_MAX_ADMIN_ENTRIES (4096) + +/* + * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion + * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we + * will allow outstanding on an I/O qpair at any time. The only advantage in + * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping + * the contents of the submission and completion queues, it will show a longer + * history of data. + */ +#define NVME_IO_ENTRIES (256) +#define NVME_IO_TRACKERS (128) +#define NVME_MIN_IO_TRACKERS (4) +#define NVME_MAX_IO_TRACKERS (1024) + +/* + * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES + * for each controller. + */ + +#define NVME_MAX_ASYNC_EVENTS (8) + +#define NVME_MIN_TIMEOUT_PERIOD (5) +#define NVME_MAX_TIMEOUT_PERIOD (120) + +/* Maximum log page size to fetch for AERs. */ +#define NVME_MAX_AER_LOG_SIZE (4096) + +struct nvme_request { + struct nvme_command cmd; + + /** + * Points to a parent request if part of a split request, + * NULL otherwise. + */ + struct nvme_request *parent; + union { + void *payload; + } u; + + uint8_t timeout; + uint8_t retries; + + /** + * Number of children requests still outstanding for this + * request which was split into multiple child requests. + */ + uint8_t num_children; + uint32_t payload_size; + nvme_cb_fn_t cb_fn; + void *cb_arg; + STAILQ_ENTRY(nvme_request) stailq; + + /** + * The following members should not be reordered with members + * above. These members are only needed when splitting + * requests which is done rarely, and the driver is careful + * to not touch the following fields until a split operation is + * needed, to avoid touching an extra cacheline. + */ + + /** + * Points to the outstanding child requests for a parent request. + * Only valid if a request was split into multiple children + * requests, and is not initialized for non-split requests. + */ + TAILQ_HEAD(, nvme_request) children; + + /** + * Linked-list pointers for a child request in its parent's list. + */ + TAILQ_ENTRY(nvme_request) child_tailq; + + /** + * Completion status for a parent request. Initialized to all 0's + * (SUCCESS) before child requests are submitted. If a child + * request completes with error, the error status is copied here, + * to ensure that the parent request is also completed with error + * status once all child requests are completed. + */ + struct nvme_completion parent_status; +}; + +struct nvme_completion_poll_status { + struct nvme_completion cpl; + bool done; +}; + +struct nvme_async_event_request { + struct nvme_controller *ctrlr; + struct nvme_request *req; + struct nvme_completion cpl; +}; + +struct nvme_tracker { + LIST_ENTRY(nvme_tracker) list; + + struct nvme_request *req; + uint16_t cid; + + uint64_t prp_bus_addr; + uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES]; +}; + +struct nvme_qpair { + volatile uint32_t *sq_tdbl; + volatile uint32_t *cq_hdbl; + + /** + * Submission queue + */ + struct nvme_command *cmd; + + /** + * Completion queue + */ + struct nvme_completion *cpl; + + LIST_HEAD(, nvme_tracker) free_tr; + LIST_HEAD(, nvme_tracker) outstanding_tr; + + STAILQ_HEAD(, nvme_request) queued_req; + + struct nvme_tracker **act_tr; + + uint16_t id; + + uint16_t num_entries; + uint16_t sq_tail; + uint16_t cq_head; + + uint8_t phase; + + bool is_enabled; + + /* + * Fields below this point should not be touched on the normal I/O happy path. + */ + struct nvme_controller *ctrlr; + + uint64_t cmd_bus_addr; + uint64_t cpl_bus_addr; +}; + +struct nvme_namespace { + struct nvme_controller *ctrlr; + uint32_t stripe_size; + uint32_t sector_size; + uint32_t sectors_per_max_io; + uint32_t sectors_per_stripe; + uint16_t id; + uint16_t flags; +}; + +/* + * One of these per allocated PCI device. + */ +struct nvme_controller { + /* Hot data (accessed in I/O path) starts here. */ + + /** NVMe MMIO register space */ + volatile struct nvme_registers *regs; + + /** I/O queue pairs */ + struct nvme_qpair *ioq; + + uint32_t is_resetting; + + uint32_t num_ns; + + /** Array of namespaces indexed by nsid - 1 */ + struct nvme_namespace *ns; + + + /* Cold data (not accessed in normal I/O path) is after this point. */ + + /* Opaque handle to associated PCI device. */ + void *devhandle; + + uint32_t num_io_queues; + + /** maximum i/o size in bytes */ + uint32_t max_xfer_size; + + /** minimum page size supported by this controller in bytes */ + uint32_t min_page_size; + + /** stride in uint32_t units between doorbell registers (1 = 4 bytes, 2 = 8 bytes, ...) */ + uint32_t doorbell_stride_u32; + + uint32_t num_aers; + struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS]; + nvme_aer_cb_fn_t aer_cb_fn; + void *aer_cb_arg; + + bool is_failed; + + /** guards access to the controller itself, including admin queues */ + nvme_mutex_t ctrlr_lock; + + + struct nvme_qpair adminq; + + /** + * Identify Controller data. + */ + struct nvme_controller_data cdata; + + /** + * Array of Identify Namespace data. + * + * Stored separately from ns since nsdata should not normally be accessed during I/O. + */ + struct nvme_namespace_data *nsdata; +}; + +extern int __thread nvme_thread_ioq_index; + +struct nvme_driver { + nvme_mutex_t lock; + uint16_t *ioq_index_pool; + uint32_t max_io_queues; + uint16_t ioq_index_pool_next; +}; + +extern struct nvme_driver g_nvme_driver; + +#define nvme_min(a,b) (((a)<(b))?(a):(b)) + +#define INTEL_DC_P3X00_DEVID 0x09538086 + +static inline uint32_t +_nvme_mmio_read_4(const volatile uint32_t *addr) +{ + return *addr; +} + +static inline void +_nvme_mmio_write_4(volatile uint32_t *addr, uint32_t val) +{ + *addr = val; +} + +static inline void +_nvme_mmio_write_8(volatile uint64_t *addr, uint64_t val) +{ + *addr = val; +} + +#define nvme_mmio_read_4(sc, reg) \ + _nvme_mmio_read_4(&(sc)->regs->reg) + +#define nvme_mmio_write_4(sc, reg, val) \ + _nvme_mmio_write_4(&(sc)->regs->reg, val) + +#define nvme_mmio_write_8(sc, reg, val) \ + _nvme_mmio_write_8(&(sc)->regs->reg, val) + +#define nvme_delay usleep + +static inline uint32_t +nvme_u32log2(uint32_t x) +{ + if (x == 0) { + /* __builtin_clz(0) is undefined, so just bail */ + return 0; + } + return 31u - __builtin_clz(x); +} + +static inline uint32_t +nvme_align32pow2(uint32_t x) +{ + return 1u << (1 + nvme_u32log2(x - 1)); +} + +/* Admin functions */ +void nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, + uint8_t feature, uint32_t cdw11, + void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, + uint8_t feature, uint32_t cdw11, + void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, + uint8_t log_page, uint32_t nsid, + void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg); + +void nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, + void *payload, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, + uint16_t nsid, void *payload, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr, + struct nvme_error_information_entry *payload, + uint32_t num_entries, /* 0 = max */ + nvme_cb_fn_t cb_fn, + void *cb_arg); +void nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr, + uint32_t nsid, + struct nvme_health_information_page *payload, + nvme_cb_fn_t cb_fn, + void *cb_arg); +void nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr, + struct nvme_firmware_page *payload, + nvme_cb_fn_t cb_fn, + void *cb_arg); +void nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr, + struct nvme_qpair *io_que, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr, + struct nvme_qpair *io_que, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr, + uint32_t num_queues, nvme_cb_fn_t cb_fn, + void *cb_arg); +void nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr, + union nvme_critical_warning_state state, + nvme_cb_fn_t cb_fn, void *cb_arg); +void nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid, + uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg); + +void nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl); + +int nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle); +void nvme_ctrlr_destruct(struct nvme_controller *ctrlr); +int nvme_ctrlr_start(struct nvme_controller *ctrlr); +int nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr); + +void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, + struct nvme_request *req); +void nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, + struct nvme_request *req); +void nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr, + struct nvme_request *req); + +int nvme_qpair_construct(struct nvme_qpair *qpair, uint16_t id, + uint16_t num_entries, + uint16_t num_trackers, + struct nvme_controller *ctrlr); +void nvme_qpair_destroy(struct nvme_qpair *qpair); +void nvme_qpair_enable(struct nvme_qpair *qpair); +void nvme_qpair_disable(struct nvme_qpair *qpair); +void nvme_qpair_submit_tracker(struct nvme_qpair *qpair, + struct nvme_tracker *tr); +void nvme_qpair_process_completions(struct nvme_qpair *qpair); +void nvme_qpair_submit_request(struct nvme_qpair *qpair, + struct nvme_request *req); +void nvme_qpair_reset(struct nvme_qpair *qpair); +void nvme_qpair_fail(struct nvme_qpair *qpair); +void nvme_qpair_manual_complete_request(struct nvme_qpair *qpair, + struct nvme_request *req, + uint32_t sct, uint32_t sc, + bool print_on_error); + +int nvme_ns_construct(struct nvme_namespace *ns, uint16_t id, + struct nvme_controller *ctrlr); +void nvme_ns_destruct(struct nvme_namespace *ns); + +void nvme_dump_command(struct nvme_command *cmd); +void nvme_dump_completion(struct nvme_completion *cpl); + +struct nvme_request * +nvme_allocate_request(void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg); + +void nvme_cb_complete_child(void *child, const struct nvme_completion *cpl); +void nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child); + +#endif /* __NVME_INTERNAL_H__ */ diff --git a/lib/nvme/nvme_ns.c b/lib/nvme/nvme_ns.c new file mode 100644 index 000000000..e9fe1de2b --- /dev/null +++ b/lib/nvme/nvme_ns.c @@ -0,0 +1,135 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +static inline struct nvme_namespace_data * +_nvme_ns_get_data(struct nvme_namespace *ns) +{ + return &ns->ctrlr->nsdata[ns->id - 1]; +} + +uint32_t +nvme_ns_get_id(struct nvme_namespace *ns) +{ + return ns->id; +} + +uint32_t +nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns) +{ + return ns->ctrlr->max_xfer_size; +} + +uint32_t +nvme_ns_get_sector_size(struct nvme_namespace *ns) +{ + return ns->sector_size; +} + +uint64_t +nvme_ns_get_num_sectors(struct nvme_namespace *ns) +{ + return _nvme_ns_get_data(ns)->nsze; +} + +uint64_t +nvme_ns_get_size(struct nvme_namespace *ns) +{ + return nvme_ns_get_num_sectors(ns) * nvme_ns_get_sector_size(ns); +} + +uint32_t +nvme_ns_get_flags(struct nvme_namespace *ns) +{ + return ns->flags; +} + +const struct nvme_namespace_data * +nvme_ns_get_data(struct nvme_namespace *ns) +{ + return _nvme_ns_get_data(ns); +} + +int +nvme_ns_construct(struct nvme_namespace *ns, uint16_t id, + struct nvme_controller *ctrlr) +{ + struct nvme_completion_poll_status status; + struct nvme_namespace_data *nsdata; + uint32_t pci_devid; + + nvme_assert(id > 0, ("invalid namespace id %d", id)); + + ns->ctrlr = ctrlr; + ns->id = id; + ns->stripe_size = 0; + + nvme_pcicfg_read32(ctrlr->devhandle, &pci_devid, 0); + if (pci_devid == INTEL_DC_P3X00_DEVID && ctrlr->cdata.vs[3] != 0) { + ns->stripe_size = (1 << ctrlr->cdata.vs[3]) * ctrlr->min_page_size; + } + + nsdata = _nvme_ns_get_data(ns); + + status.done = false; + nvme_ctrlr_cmd_identify_namespace(ctrlr, id, nsdata, + nvme_completion_poll_cb, &status); + while (status.done == false) { + nvme_qpair_process_completions(&ctrlr->adminq); + } + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_identify_namespace failed\n"); + return ENXIO; + } + + ns->sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads; + + ns->sectors_per_max_io = nvme_ns_get_max_io_xfer_size(ns) / ns->sector_size; + ns->sectors_per_stripe = ns->stripe_size / ns->sector_size; + + if (ctrlr->cdata.oncs.dsm) { + ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED; + } + + if (ctrlr->cdata.vwc.present) { + ns->flags |= NVME_NS_FLUSH_SUPPORTED; + } + + return 0; +} + +void nvme_ns_destruct(struct nvme_namespace *ns) +{ + +} diff --git a/lib/nvme/nvme_ns_cmd.c b/lib/nvme/nvme_ns_cmd.c new file mode 100644 index 000000000..3806f9817 --- /dev/null +++ b/lib/nvme/nvme_ns_cmd.c @@ -0,0 +1,197 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/** + * \file + * + */ + +static struct nvme_request * +_nvme_ns_cmd_rw(struct nvme_namespace *ns, void *payload, uint64_t lba, + uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg, + uint32_t opc) +{ + struct nvme_request *req; + struct nvme_command *cmd; + uint64_t *tmp_lba; + uint32_t sector_size; + uint32_t sectors_per_max_io; + uint32_t sectors_per_stripe; + + sector_size = ns->sector_size; + sectors_per_max_io = ns->sectors_per_max_io; + sectors_per_stripe = ns->sectors_per_stripe; + + req = nvme_allocate_request(payload, lba_count * sector_size, cb_fn, cb_arg); + if (req == NULL) { + return NULL; + } + + /* + * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping. + * If this controller defines a stripe boundary and this I/O spans a stripe + * boundary, split the request into multiple requests and submit each + * separately to hardware. + */ + if (sectors_per_stripe > 0 && + (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) { + uint64_t remaining_lba_count = lba_count; + struct nvme_request *child; + + while (remaining_lba_count > 0) { + lba_count = sectors_per_stripe - (lba & (sectors_per_stripe - 1)); + lba_count = nvme_min(remaining_lba_count, lba_count); + + child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, + cb_arg, opc); + if (child == NULL) { + nvme_free_request(req); + return NULL; + } + nvme_request_add_child(req, child); + remaining_lba_count -= lba_count; + lba += lba_count; + payload = (void *)((uintptr_t)payload + (lba_count * sector_size)); + } + } else if (lba_count > sectors_per_max_io) { + uint64_t remaining_lba_count = lba_count; + struct nvme_request *child; + + while (remaining_lba_count > 0) { + lba_count = nvme_min(remaining_lba_count, sectors_per_max_io); + child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, + cb_arg, opc); + if (child == NULL) { + nvme_free_request(req); + return NULL; + } + nvme_request_add_child(req, child); + remaining_lba_count -= lba_count; + lba += lba_count; + payload = (void *)((uintptr_t)payload + (lba_count * sector_size)); + } + } else { + cmd = &req->cmd; + cmd->opc = opc; + cmd->nsid = ns->id; + + tmp_lba = (uint64_t *)&cmd->cdw10; + *tmp_lba = lba; + cmd->cdw12 = lba_count - 1; + } + + return req; +} + +int +nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba, + uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_READ); + if (req != NULL) { + nvme_ctrlr_submit_io_request(ns->ctrlr, req); + return 0; + } else { + return ENOMEM; + } +} + +int +nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba, + uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_WRITE); + if (req != NULL) { + nvme_ctrlr_submit_io_request(ns->ctrlr, req); + return 0; + } else { + return ENOMEM; + } +} + +int +nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload, + uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + if (num_ranges == 0) { + return EINVAL; + } + + req = nvme_allocate_request(payload, + num_ranges * sizeof(struct nvme_dsm_range), + cb_fn, cb_arg); + if (req == NULL) { + return ENOMEM; + } + + cmd = &req->cmd; + cmd->opc = NVME_OPC_DATASET_MANAGEMENT; + cmd->nsid = ns->id; + + /* TODO: create a delete command data structure */ + cmd->cdw10 = num_ranges - 1; + cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE; + + nvme_ctrlr_submit_io_request(ns->ctrlr, req); + + return 0; +} + +int +nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_command *cmd; + + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + if (req == NULL) { + return ENOMEM; + } + + cmd = &req->cmd; + cmd->opc = NVME_OPC_FLUSH; + cmd->nsid = ns->id; + + nvme_ctrlr_submit_io_request(ns->ctrlr, req); + + return 0; +} diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c new file mode 100644 index 000000000..082aa8817 --- /dev/null +++ b/lib/nvme/nvme_qpair.c @@ -0,0 +1,878 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/** + * \file + * + */ + +static inline bool nvme_qpair_is_admin_queue(struct nvme_qpair *qpair) +{ + return qpair->id == 0; +} + +static inline bool nvme_qpair_is_io_queue(struct nvme_qpair *qpair) +{ + return qpair->id != 0; +} + +struct nvme_string { + uint16_t value; + const char *str; +}; + +static const struct nvme_string admin_opcode[] = { + { NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, + { NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, + { NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, + { NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, + { NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, + { NVME_OPC_IDENTIFY, "IDENTIFY" }, + { NVME_OPC_ABORT, "ABORT" }, + { NVME_OPC_SET_FEATURES, "SET FEATURES" }, + { NVME_OPC_GET_FEATURES, "GET FEATURES" }, + { NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, + { NVME_OPC_NAMESPACE_MANAGEMENT, "NAMESPACE MANAGEMENT" }, + { NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" }, + { NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, + { NVME_OPC_NAMESPACE_ATTACHMENT, "NAMESPACE ATTACHMENT" }, + { NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, + { NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, + { NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, + { 0xFFFF, "ADMIN COMMAND" } +}; + +static const struct nvme_string io_opcode[] = { + { NVME_OPC_FLUSH, "FLUSH" }, + { NVME_OPC_WRITE, "WRITE" }, + { NVME_OPC_READ, "READ" }, + { NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, + { NVME_OPC_COMPARE, "COMPARE" }, + { NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" }, + { NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, + { NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" }, + { NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" }, + { NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" }, + { NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" }, + { 0xFFFF, "IO COMMAND" } +}; + +static const char * +nvme_get_string(const struct nvme_string *strings, uint16_t value) +{ + const struct nvme_string *entry; + + entry = strings; + + while (entry->value != 0xFFFF) { + if (entry->value == value) { + return entry->str; + } + entry++; + } + return entry->str; +} + +static void +nvme_admin_qpair_print_command(struct nvme_qpair *qpair, + struct nvme_command *cmd) +{ + + nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x " + "cdw10:%08x cdw11:%08x\n", + nvme_get_string(admin_opcode, cmd->opc), cmd->opc, qpair->id, cmd->cid, + cmd->nsid, cmd->cdw10, cmd->cdw11); +} + +static void +nvme_io_qpair_print_command(struct nvme_qpair *qpair, + struct nvme_command *cmd) +{ + + switch ((int)cmd->opc) { + case NVME_OPC_WRITE: + case NVME_OPC_READ: + case NVME_OPC_WRITE_UNCORRECTABLE: + case NVME_OPC_COMPARE: + nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d " + "lba:%llu len:%d\n", + nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid, + cmd->nsid, + ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10, + (cmd->cdw12 & 0xFFFF) + 1); + break; + case NVME_OPC_FLUSH: + case NVME_OPC_DATASET_MANAGEMENT: + nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n", + nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid, + cmd->nsid); + break; + default: + nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n", + nvme_get_string(io_opcode, cmd->opc), cmd->opc, qpair->id, + cmd->cid, cmd->nsid); + break; + } +} + +static void +nvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd) +{ + if (nvme_qpair_is_admin_queue(qpair)) { + nvme_admin_qpair_print_command(qpair, cmd); + } else { + nvme_io_qpair_print_command(qpair, cmd); + } +} + +static const struct nvme_string generic_status[] = { + { NVME_SC_SUCCESS, "SUCCESS" }, + { NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, + { NVME_SC_INVALID_FIELD, "INVALID_FIELD" }, + { NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, + { NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, + { NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, + { NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, + { NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, + { NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, + { NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, + { NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, + { NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, + { NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, + { NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, + { NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, + { NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, + { 0xFFFF, "GENERIC" } +}; + +static const struct nvme_string command_specific_status[] = { + { NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, + { NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, + { NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" }, + { NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, + { NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" }, + { NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, + { NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, + { NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, + { NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, + { NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, + { NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" }, + { NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, + { NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, + { NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" }, + { 0xFFFF, "COMMAND SPECIFIC" } +}; + +static const struct nvme_string media_error_status[] = { + { NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, + { NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, + { NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, + { NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, + { NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, + { NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, + { NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, + { 0xFFFF, "MEDIA ERROR" } +}; + +static const char * +get_status_string(uint16_t sct, uint16_t sc) +{ + const struct nvme_string *entry; + + switch (sct) { + case NVME_SCT_GENERIC: + entry = generic_status; + break; + case NVME_SCT_COMMAND_SPECIFIC: + entry = command_specific_status; + break; + case NVME_SCT_MEDIA_ERROR: + entry = media_error_status; + break; + case NVME_SCT_VENDOR_SPECIFIC: + return "VENDOR SPECIFIC"; + default: + return "RESERVED"; + } + + return nvme_get_string(entry, sc); +} + +static void +nvme_qpair_print_completion(struct nvme_qpair *qpair, + struct nvme_completion *cpl) +{ + nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n", + get_status_string(cpl->status.sct, cpl->status.sc), + cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0); +} + +static bool +nvme_completion_is_retry(const struct nvme_completion *cpl) +{ + /* + * TODO: spec is not clear how commands that are aborted due + * to TLER will be marked. So for now, it seems + * NAMESPACE_NOT_READY is the only case where we should + * look at the DNR bit. + */ + switch ((int)cpl->status.sct) { + case NVME_SCT_GENERIC: + switch ((int)cpl->status.sc) { + case NVME_SC_ABORTED_BY_REQUEST: + case NVME_SC_NAMESPACE_NOT_READY: + if (cpl->status.dnr) { + return false; + } else { + return true; + } + case NVME_SC_INVALID_OPCODE: + case NVME_SC_INVALID_FIELD: + case NVME_SC_COMMAND_ID_CONFLICT: + case NVME_SC_DATA_TRANSFER_ERROR: + case NVME_SC_ABORTED_POWER_LOSS: + case NVME_SC_INTERNAL_DEVICE_ERROR: + case NVME_SC_ABORTED_SQ_DELETION: + case NVME_SC_ABORTED_FAILED_FUSED: + case NVME_SC_ABORTED_MISSING_FUSED: + case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: + case NVME_SC_COMMAND_SEQUENCE_ERROR: + case NVME_SC_LBA_OUT_OF_RANGE: + case NVME_SC_CAPACITY_EXCEEDED: + default: + return false; + } + case NVME_SCT_COMMAND_SPECIFIC: + case NVME_SCT_MEDIA_ERROR: + case NVME_SCT_VENDOR_SPECIFIC: + default: + return false; + } +} + +static void +nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr) +{ + tr->prp_bus_addr = phys_addr + offsetof(struct nvme_tracker, prp); + tr->cid = cid; +} + +static void +nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, + struct nvme_completion *cpl, bool print_on_error) +{ + struct nvme_request *req; + bool retry, error; + + req = tr->req; + error = nvme_completion_is_error(cpl); + retry = error && nvme_completion_is_retry(cpl) && + req->retries < nvme_retry_count; + + if (error && print_on_error) { + nvme_qpair_print_command(qpair, &req->cmd); + nvme_qpair_print_completion(qpair, cpl); + } + + qpair->act_tr[cpl->cid] = NULL; + + nvme_assert(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); + + if (req->cb_fn && !retry) { + req->cb_fn(req->cb_arg, cpl); + } + + if (retry) { + req->retries++; + nvme_qpair_submit_tracker(qpair, tr); + } else { + nvme_free_request(req); + tr->req = NULL; + + LIST_REMOVE(tr, list); + LIST_INSERT_HEAD(&qpair->free_tr, tr, list); + + /* + * If the controller is in the middle of resetting, don't + * try to submit queued requests here - let the reset logic + * handle that instead. + */ + if (!STAILQ_EMPTY(&qpair->queued_req) && + !qpair->ctrlr->is_resetting) { + req = STAILQ_FIRST(&qpair->queued_req); + STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); + nvme_qpair_submit_request(qpair, req); + } + } +} + +static void +nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, + struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr, + bool print_on_error) +{ + struct nvme_completion cpl; + + memset(&cpl, 0, sizeof(cpl)); + cpl.sqid = qpair->id; + cpl.cid = tr->cid; + cpl.status.sct = sct; + cpl.status.sc = sc; + cpl.status.dnr = dnr; + nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); +} + +void +nvme_qpair_manual_complete_request(struct nvme_qpair *qpair, + struct nvme_request *req, uint32_t sct, uint32_t sc, + bool print_on_error) +{ + struct nvme_completion cpl; + bool error; + + memset(&cpl, 0, sizeof(cpl)); + cpl.sqid = qpair->id; + cpl.status.sct = sct; + cpl.status.sc = sc; + + error = nvme_completion_is_error(&cpl); + + if (error && print_on_error) { + nvme_qpair_print_command(qpair, &req->cmd); + nvme_qpair_print_completion(qpair, &cpl); + } + + if (req->cb_fn) { + req->cb_fn(req->cb_arg, &cpl); + } + + nvme_free_request(req); +} + +static inline bool +nvme_qpair_check_enabled(struct nvme_qpair *qpair) +{ + if (!qpair->is_enabled && + !qpair->ctrlr->is_resetting) { + nvme_qpair_enable(qpair); + } + return qpair->is_enabled; +} + +/** + * \page nvme_async_completion NVMe Asynchronous Completion + * + * The userspace NVMe driver follows an asynchronous polled model for + * I/O completion. + * + * \section async_io I/O commands + * + * The application may submit I/O from one or more threads + * and must call nvme_ctrlr_process_io_completions() + * from each thread that submitted I/O. + * + * When the application calls nvme_ctrlr_process_io_completions(), + * if the NVMe driver detects completed I/Os that were submitted on that thread, + * it will invoke the registered callback function + * for each I/O within the context of nvme_ctrlr_process_io_completions(). + * + * \section async_admin Admin commands + * + * The application may submit admin commands from one or more threads + * and must call nvme_ctrlr_process_admin_completions() + * from at least one thread to receive admin command completions. + * The thread that processes admin completions need not be the same thread that submitted the + * admin commands. + * + * When the application calls nvme_ctrlr_process_admin_completions(), + * if the NVMe driver detects completed admin commands submitted from any thread, + * it will invote the registered callback function + * for each command within the context of nvme_ctrlr_process_admin_completions(). + * + * It is the application's responsibility to manage the order of submitted admin commands. + * If certain admin commands must be submitted while no other commands are outstanding, + * it is the application's responsibility to enforce this rule + * using its own synchronization method. + */ + +/** + * \brief Checks for and processes completions on the specified qpair. + * + * For each completed command, the request's callback function will + * be called if specified as non-NULL when the request was submitted. + * + * \sa nvme_cb_fn_t + */ +void +nvme_qpair_process_completions(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + struct nvme_completion *cpl; + + if (!nvme_qpair_check_enabled(qpair)) { + /* + * qpair is not enabled, likely because a controller reset is + * is in progress. Ignore the interrupt - any I/O that was + * associated with this interrupt will get retried when the + * reset is complete. + */ + return; + } + + while (1) { + cpl = &qpair->cpl[qpair->cq_head]; + + if (cpl->status.p != qpair->phase) + break; + + tr = qpair->act_tr[cpl->cid]; + + if (tr != NULL) { + nvme_qpair_complete_tracker(qpair, tr, cpl, true); + } else { + nvme_printf(qpair->ctrlr, + "cpl does not map to outstanding cmd\n"); + nvme_dump_completion(cpl); + nvme_assert(0, ("received completion for unknown cmd\n")); + } + + if (++qpair->cq_head == qpair->num_entries) { + qpair->cq_head = 0; + qpair->phase = !qpair->phase; + } + + _nvme_mmio_write_4(qpair->cq_hdbl, qpair->cq_head); + } +} + +int +nvme_qpair_construct(struct nvme_qpair *qpair, uint16_t id, + uint16_t num_entries, uint16_t num_trackers, + struct nvme_controller *ctrlr) +{ + struct nvme_tracker *tr; + uint16_t i; + volatile uint32_t *doorbell_base; + uint64_t phys_addr = 0; + + qpair->id = id; + qpair->num_entries = num_entries; + + qpair->ctrlr = ctrlr; + + /* cmd and cpl rings must be aligned on 4KB boundaries. */ + qpair->cmd = nvme_malloc("qpair_cmd", + qpair->num_entries * sizeof(struct nvme_command), + 0x1000, + &qpair->cmd_bus_addr); + if (qpair->cmd == NULL) { + nvme_printf(ctrlr, "alloc qpair_cmd failed\n"); + goto fail; + } + qpair->cpl = nvme_malloc("qpair_cpl", + qpair->num_entries * sizeof(struct nvme_completion), + 0x1000, + &qpair->cpl_bus_addr); + if (qpair->cpl == NULL) { + nvme_printf(ctrlr, "alloc qpair_cpl failed\n"); + goto fail; + } + + doorbell_base = &ctrlr->regs->doorbell[0].sq_tdbl; + qpair->sq_tdbl = doorbell_base + (2 * id + 0) * ctrlr->doorbell_stride_u32; + qpair->cq_hdbl = doorbell_base + (2 * id + 1) * ctrlr->doorbell_stride_u32; + + LIST_INIT(&qpair->free_tr); + LIST_INIT(&qpair->outstanding_tr); + STAILQ_INIT(&qpair->queued_req); + + for (i = 0; i < num_trackers; i++) { + /* + * Round alignment up to next power of 2. This ensures the PRP + * list embedded in the nvme_tracker object will not span a + * 4KB boundary. + */ + tr = nvme_malloc("nvme_tr", sizeof(*tr), nvme_align32pow2(sizeof(*tr)), &phys_addr); + if (tr == NULL) { + nvme_printf(ctrlr, "nvme_tr failed\n"); + goto fail; + } + nvme_qpair_construct_tracker(tr, i, phys_addr); + LIST_INSERT_HEAD(&qpair->free_tr, tr, list); + } + + qpair->act_tr = calloc(num_trackers, sizeof(struct nvme_tracker *)); + if (qpair->act_tr == NULL) { + nvme_printf(ctrlr, "alloc nvme_act_tr failed\n"); + goto fail; + } + nvme_qpair_reset(qpair); + return 0; +fail: + nvme_qpair_destroy(qpair); + return -1; +} + +static void +nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + + tr = LIST_FIRST(&qpair->outstanding_tr); + while (tr != NULL) { + if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { + nvme_qpair_manual_complete_tracker(qpair, tr, + NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0, + false); + tr = LIST_FIRST(&qpair->outstanding_tr); + } else { + tr = LIST_NEXT(tr, list); + } + } +} + +static void +_nvme_admin_qpair_destroy(struct nvme_qpair *qpair) +{ + nvme_admin_qpair_abort_aers(qpair); +} + +static void +_nvme_io_qpair_destroy(struct nvme_qpair *qpair) +{ +} + +void +nvme_qpair_destroy(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + + if (nvme_qpair_is_io_queue(qpair)) { + _nvme_io_qpair_destroy(qpair); + } else { + _nvme_admin_qpair_destroy(qpair); + } + if (qpair->cmd) + nvme_free(qpair->cmd); + if (qpair->cpl) + nvme_free(qpair->cpl); + if (qpair->act_tr) + free(qpair->act_tr); + + while (!LIST_EMPTY(&qpair->free_tr)) { + tr = LIST_FIRST(&qpair->free_tr); + LIST_REMOVE(tr, list); + nvme_free(tr); + } +} + +/** + * \page nvme_io_submission NVMe I/O Submission + * + * I/O is submitted to an NVMe namespace using nvme_ns_cmd_xxx functions + * defined in nvme_ns_cmd.c. The NVMe driver submits the I/O request + * as an NVMe submission queue entry on the nvme_qpair associated with + * the logical core that submits the I/O. + * + * \sa nvme_ns_cmd_read, nvme_ns_cmd_write, nvme_ns_cmd_deallocate, + * nvme_ns_cmd_flush, nvme_get_ioq_idx + */ + + +void +nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) +{ + struct nvme_request *req; + + req = tr->req; + qpair->act_tr[tr->cid] = tr; + + /* Copy the command from the tracker to the submission queue. */ + memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); + + if (++qpair->sq_tail == qpair->num_entries) { + qpair->sq_tail = 0; + } + + wmb(); + _nvme_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail); +} + +static void +_nvme_fail_request_bad_vtophys(struct nvme_qpair *qpair, struct nvme_tracker *tr) +{ + /* + * Bad vtophys translation, so abort this request and return + * immediately. + */ + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_INVALID_FIELD, + 1 /* do not retry */, true); +} + +static void +_nvme_fail_request_ctrlr_failed(struct nvme_qpair *qpair, struct nvme_request *req) +{ + nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, true); +} + +void +nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) +{ + struct nvme_tracker *tr; + struct nvme_request *child_req; + uint64_t phys_addr; + void *seg_addr; + uint32_t nseg, cur_nseg, modulo, unaligned; + + nvme_qpair_check_enabled(qpair); + + if (req->num_children) { + /* + * This is a split (parent) request. Submit all of the children but not the parent + * request itself, since the parent is the original unsplit request. + */ + TAILQ_FOREACH(child_req, &req->children, child_tailq) { + nvme_qpair_submit_request(qpair, child_req); + } + return; + } + + tr = LIST_FIRST(&qpair->free_tr); + + if (tr == NULL || !qpair->is_enabled) { + /* + * No tracker is available, or the qpair is disabled due to + * an in-progress controller-level reset or controller + * failure. + */ + + if (qpair->ctrlr->is_failed) { + _nvme_fail_request_ctrlr_failed(qpair, req); + } else { + /* + * Put the request on the qpair's request queue to be + * processed when a tracker frees up via a command + * completion or when the controller reset is + * completed. + */ + STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); + } + return; + } + + LIST_REMOVE(tr, list); /* remove tr from free_tr */ + LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list); + tr->req = req; + req->cmd.cid = tr->cid; + + if (req->payload_size) { + /* + * Build PRP list describing payload buffer. + */ + + phys_addr = nvme_vtophys(req->u.payload); + if (phys_addr == -1) { + _nvme_fail_request_bad_vtophys(qpair, tr); + return; + } + nseg = req->payload_size >> nvme_u32log2(PAGE_SIZE); + modulo = req->payload_size & (PAGE_SIZE - 1); + unaligned = phys_addr & (PAGE_SIZE - 1); + if (modulo || unaligned) { + nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE)); + } + + tr->req->cmd.psdt = NVME_PSDT_PRP; + tr->req->cmd.dptr.prp.prp1 = phys_addr; + if (nseg == 2) { + seg_addr = req->u.payload + PAGE_SIZE - unaligned; + tr->req->cmd.dptr.prp.prp2 = nvme_vtophys(seg_addr); + } else if (nseg > 2) { + cur_nseg = 1; + tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr; + while (cur_nseg < nseg) { + seg_addr = req->u.payload + cur_nseg * PAGE_SIZE - unaligned; + phys_addr = nvme_vtophys(seg_addr); + if (phys_addr == -1) { + _nvme_fail_request_bad_vtophys(qpair, tr); + return; + } + tr->prp[cur_nseg - 1] = phys_addr; + cur_nseg++; + } + } + } + + nvme_qpair_submit_tracker(qpair, tr); +} + +void +nvme_qpair_reset(struct nvme_qpair *qpair) +{ + qpair->sq_tail = qpair->cq_head = 0; + + /* + * First time through the completion queue, HW will set phase + * bit on completions to 1. So set this to 1 here, indicating + * we're looking for a 1 to know which entries have completed. + * we'll toggle the bit each time when the completion queue + * rolls over. + */ + qpair->phase = 1; + + memset(qpair->cmd, 0, + qpair->num_entries * sizeof(struct nvme_command)); + memset(qpair->cpl, 0, + qpair->num_entries * sizeof(struct nvme_completion)); +} + +static void +_nvme_admin_qpair_enable(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + struct nvme_tracker *tr_temp; + + /* + * Manually abort each outstanding admin command. Do not retry + * admin commands found here, since they will be left over from + * a controller reset and its likely the context in which the + * command was issued no longer applies. + */ + LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) { + nvme_printf(qpair->ctrlr, + "aborting outstanding admin command\n"); + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true); + } + + qpair->is_enabled = true; +} + +static void +_nvme_io_qpair_enable(struct nvme_qpair *qpair) +{ + STAILQ_HEAD(, nvme_request) temp; + struct nvme_tracker *tr; + struct nvme_tracker *tr_temp; + struct nvme_request *req; + + /* + * Manually abort each outstanding I/O. This normally results in a + * retry, unless the retry count on the associated request has + * reached its limit. + */ + LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) { + nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n"); + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, 0, true); + } + + qpair->is_enabled = true; + + STAILQ_INIT(&temp); + STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); + + while (!STAILQ_EMPTY(&temp)) { + req = STAILQ_FIRST(&temp); + STAILQ_REMOVE_HEAD(&temp, stailq); + + nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n"); + nvme_qpair_print_command(qpair, &req->cmd); + nvme_qpair_submit_request(qpair, req); + } +} + +void +nvme_qpair_enable(struct nvme_qpair *qpair) +{ + if (nvme_qpair_is_io_queue(qpair)) { + _nvme_io_qpair_enable(qpair); + } else { + _nvme_admin_qpair_enable(qpair); + } +} + +static void +_nvme_admin_qpair_disable(struct nvme_qpair *qpair) +{ + qpair->is_enabled = false; + nvme_admin_qpair_abort_aers(qpair); +} + +static void +_nvme_io_qpair_disable(struct nvme_qpair *qpair) +{ + qpair->is_enabled = false; +} + +void +nvme_qpair_disable(struct nvme_qpair *qpair) +{ + if (nvme_qpair_is_io_queue(qpair)) { + _nvme_io_qpair_disable(qpair); + } else { + _nvme_admin_qpair_disable(qpair); + } +} + +void +nvme_qpair_fail(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + struct nvme_request *req; + + while (!STAILQ_EMPTY(&qpair->queued_req)) { + req = STAILQ_FIRST(&qpair->queued_req); + STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); + nvme_printf(qpair->ctrlr, "failing queued i/o\n"); + nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, true); + } + + /* Manually abort each outstanding I/O. */ + while (!LIST_EMPTY(&qpair->outstanding_tr)) { + tr = LIST_FIRST(&qpair->outstanding_tr); + /* + * Do not remove the tracker. The abort_tracker path will + * do that for us. + */ + nvme_printf(qpair->ctrlr, "failing outstanding i/o\n"); + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true); + } +} + diff --git a/lib/util/Makefile b/lib/util/Makefile new file mode 100644 index 000000000..02d0c5de8 --- /dev/null +++ b/lib/util/Makefile @@ -0,0 +1,53 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(DPDK_INC) + +C_SRCS = pci.c + +C_OBJS := $(patsubst %.c,%.o,$(C_SRCS)) + +OBJS = $(C_OBJS) + +all : libspdk_util.a + +objs : $(OBJS) + +clean : + $(Q)rm -f libspdk_util.a $(OBJS) *.d + +libspdk_util.a : $(OBJS) + $(Q)ar crDs libspdk_util.a $(OBJS) + diff --git a/lib/util/pci.c b/lib/util/pci.c new file mode 100644 index 000000000..47b06f8fd --- /dev/null +++ b/lib/util/pci.c @@ -0,0 +1,293 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "spdk/pci.h" + +#define SYSFS_PCI_DEVICES "/sys/bus/pci/devices" +#define SYSFS_PCI_DRIVERS "/sys/bus/pci/drivers" +#define PCI_PRI_FMT "%04x:%02x:%02x.%1u" + +int +pci_device_get_serial_number(struct pci_device *dev, char *sn, int len) +{ + int err; + uint32_t pos, header = 0; + uint32_t i, buf[2]; + + if (len < 17) + return -1; + + err = pci_device_cfg_read_u32(dev, &header, PCI_CFG_SIZE); + if (err || !header) + return -1; + + pos = PCI_CFG_SIZE; + while (1) { + if ((header & 0x0000ffff) == PCI_EXT_CAP_ID_SN) { + if (pos) { + /*skip the header*/ + pos += 4; + for (i = 0; i < 2; i++) { + err = pci_device_cfg_read_u32(dev, + &buf[i], pos + 4 * i); + if (err) + return -1; + } + sprintf(sn, "%08x%08x", buf[1], buf[0]); + return 0; + } + } + pos = (header >> 20) & 0xffc; + /*0 if no other items exist*/ + if (pos < PCI_CFG_SIZE) + return -1; + err = pci_device_cfg_read_u32(dev, &header, pos); + if (err) + return -1; + } + return -1; +} + +int +pci_device_has_uio_driver(struct pci_device *dev) +{ + struct dirent *e; + DIR *dir; + char dirname[PATH_MAX]; + + snprintf(dirname, sizeof(dirname), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio", + dev->domain, dev->bus, dev->dev, dev->func); + + dir = opendir(dirname); + if (!dir) { + snprintf(dirname, sizeof(dirname), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT, + dev->domain, dev->bus, dev->dev, dev->func); + dir = opendir(dirname); + if (!dir) + return 0; + } + + while ((e = readdir(dir)) != NULL) { + if (strncmp(e->d_name, "uio", 3) == 0) { + break; + } + } + + closedir(dir); + + if (!e) + return 0; + + return 1; +} + +int +pci_device_unbind_kernel_driver(struct pci_device *dev) +{ + int n; + FILE *fd; + char filename[PATH_MAX]; + char buf[256]; + + snprintf(filename, sizeof(filename), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/driver/unbind", + dev->domain, dev->bus, dev->dev, dev->func); + + fd = fopen(filename, "w"); + if (!fd) + return 0; + + n = snprintf(buf, sizeof(buf), PCI_PRI_FMT, + dev->domain, dev->bus, dev->dev, dev->func); + + if (fwrite(buf, n, 1, fd) == 0) + goto error; + + fclose(fd); + return 0; + +error: + fclose(fd); + return -1; +} + +static int +check_modules(char *driver_name) +{ + FILE *fd; + const char *proc_modules = "/proc/modules"; + char buffer[256]; + + fd = fopen(proc_modules, "r"); + if (!fd) + return -1; + + while (fgets(buffer, sizeof(buffer), fd)) { + if (strstr(buffer, driver_name) == NULL) + continue; + else { + fclose(fd); + return 0; + } + } + fclose(fd); + + return -1; +} + +int +pci_device_bind_uio_driver(struct pci_device *dev, char *driver_name) +{ + int err, n; + FILE *fd; + char filename[PATH_MAX]; + char buf[256]; + + err = check_modules(driver_name); + if (err < 0) { + fprintf(stderr, "No %s module loaded\n", driver_name); + return err; + } + + snprintf(filename, sizeof(filename), + SYSFS_PCI_DRIVERS "/" "%s" "/new_id", driver_name); + + fd = fopen(filename, "w"); + if (!fd) { + return -1; + } + + n = snprintf(buf, sizeof(buf), "%04x %04x", + dev->vendor_id, dev->device_id); + + if (fwrite(buf, n, 1, fd) == 0) + goto error; + + fclose(fd); + return 0; + +error: + fclose(fd); + return -1; +} + +int +pci_device_switch_to_uio_driver(struct pci_device *pci_dev) +{ + if (pci_device_unbind_kernel_driver(pci_dev)) { + fprintf(stderr, "Device %s %d:%d:%d unbind from " + "kernel driver failed\n", + pci_device_get_device_name(pci_dev), pci_dev->bus, + pci_dev->dev, pci_dev->func); + return -1; + } + if (pci_device_bind_uio_driver(pci_dev, PCI_UIO_DRIVER)) { + fprintf(stderr, "Device %s %d:%d:%d bind to " + "uio driver failed\n", + pci_device_get_device_name(pci_dev), pci_dev->bus, + pci_dev->dev, pci_dev->func); + return -1; + } + printf("Device %s %d:%d:%d bind to uio driver success\n", + pci_device_get_device_name(pci_dev), pci_dev->bus, + pci_dev->dev, pci_dev->func); + return 0; +} + +int +pci_device_claim(struct pci_device *dev) +{ + int dev_fd; + char shm_name[64]; + int pid; + void *dev_map; + struct flock pcidev_lock = { + .l_type = F_WRLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0, + }; + + sprintf(shm_name, PCI_PRI_FMT, dev->domain, dev->bus, + dev->dev, dev->func); + + dev_fd = shm_open(shm_name, O_RDWR | O_CREAT, 0600); + if (dev_fd == -1) { + fprintf(stderr, "could not shm_open %s\n", shm_name); + return -1; + } + + if (ftruncate(dev_fd, sizeof(int)) != 0) { + fprintf(stderr, "could not truncate shm %s\n", shm_name); + close(dev_fd); + return -1; + } + + dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE, + MAP_SHARED, dev_fd, 0); + if (dev_map == NULL) { + fprintf(stderr, "could not mmap shm %s\n", shm_name); + close(dev_fd); + return -1; + } + + if (fcntl(dev_fd, F_SETLK, &pcidev_lock) != 0) { + pid = *(int *)dev_map; + fprintf(stderr, "Cannot create lock on device %s, probably" + " process %d has claimed it\n", shm_name, pid); + munmap(dev_map, sizeof(int)); + close(dev_fd); + return -1; + } + + *(int *)dev_map = (int)getpid(); + munmap(dev_map, sizeof(int)); + /* Keep dev_fd open to maintain the lock. */ + return 0; +} diff --git a/mk/nvme.unittest.mk b/mk/nvme.unittest.mk new file mode 100644 index 000000000..8a2c906b5 --- /dev/null +++ b/mk/nvme.unittest.mk @@ -0,0 +1,71 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.. +NVME_DIR := $(SPDK_ROOT_DIR)/lib/nvme + +include $(SPDK_ROOT_DIR)/CONFIG + +C_OPT ?= -O2 -fno-omit-frame-pointer +Q ?= @ +S ?= $(notdir $(CURDIR)) + +C_SRCS = $(TEST_FILE) $(OTHER_FILES) + +OBJS = $(C_SRCS:.c=.o) + +CFLAGS += $(C_OPT) -I$(SPDK_ROOT_DIR)/lib -I$(SPDK_ROOT_DIR)/include -include $(SPDK_ROOT_DIR)/test/lib/nvme/unit/nvme_impl.h + +LIBS += -lcunit -lpthread + +UT_APP = $(TEST_FILE:.c=) + +all: $(UT_APP) + +$(UT_APP) : $(OBJS) + @echo " LINK $@" + $(Q)$(CC) $(CFLAGS) -o $@ $(OBJS) $(LIBS) + +clean: + $(Q)rm -f $(UT_APP) $(OBJS) *.d + +%.o: $(NVME_DIR)/%.c + @echo " CC $@" + $(Q)$(CC) $(CFLAGS) -c $< -o $@ + $(Q)$(CC) -MM $(CFLAGS) $(NVME_DIR)/$*.c > $*.d + @mv -f $*.d $*.d.tmp + @sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d + @sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -1 | \ + sed -e 's/^ *//' -e 's/$$/:/' >> $*.d + @rm -f $*.d.tmp + diff --git a/mk/spdk.common.mk b/mk/spdk.common.mk new file mode 100644 index 000000000..f974e1b59 --- /dev/null +++ b/mk/spdk.common.mk @@ -0,0 +1,77 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/CONFIG + +C_OPT ?= -fno-omit-frame-pointer +Q ?= @ +S ?= $(notdir $(CURDIR)) + +ifeq ($(MAKECMDGOALS),) +MAKECMDGOALS=$(.DEFAULT_GOAL) +endif + +COMMON_CFLAGS = -g $(C_OPT) -Wall -Werror -fno-strict-aliasing -march=native -m64 -I$(SPDK_ROOT_DIR)/include + +ifeq ($(CONFIG_DEBUG), y) +COMMON_CFLAGS += -DDEBUG -O0 +else +COMMON_CFLAGS += -O2 +endif + +CFLAGS += $(COMMON_CFLAGS) -Wno-pointer-sign -std=gnu11 + +MAKEFLAGS += --no-print-directory + +.PRECIOUS: $(OBJS) + +-include $(OBJS:.o=.d) + +%.o : %.c + @echo " CC $@" + $(Q)$(CC) $(CFLAGS) -c $< + $(Q)$(CC) -MM $(CFLAGS) $*.c > $*.d + @mv -f $*.d $*.d.tmp + @sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d + @sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -1 | \ + sed -e 's/^ *//' -e 's/$$/:/' >> $*.d + @rm -f $*.d.tmp + +DPDK_DIR ?= $(CONFIG_DPDK_DIR) +DPDK_INC_DIR ?= $(DPDK_DIR)/include +DPDK_LIB_DIR ?= $(DPDK_DIR)/lib + +DPDK_INC = -I$(DPDK_INC_DIR) +# DPDK requires dl library for dlopen/dlclose. +DPDK_LIB = -L$(DPDK_LIB_DIR) -lrte_eal -lrte_malloc -lrte_mempool -lrte_ring -ldl -Wl,-rpath=$(DPDK_LIB_DIR) + diff --git a/mk/spdk.subdirs.mk b/mk/spdk.subdirs.mk new file mode 100644 index 000000000..173912410 --- /dev/null +++ b/mk/spdk.subdirs.mk @@ -0,0 +1,3 @@ +$(DIRS-y) : + @echo "== $S/$@ ($(MAKECMDGOALS))" + $(Q)$(MAKE) -C $@ S=$S/$@ $(MAKECMDGOALS) $(MAKESUBDIRFLAGS) diff --git a/scripts/autotest_common.sh b/scripts/autotest_common.sh new file mode 100755 index 000000000..9fcd8b3d3 --- /dev/null +++ b/scripts/autotest_common.sh @@ -0,0 +1,55 @@ +set -xe +ulimit -c unlimited + +if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then + output_dir=. +else + output_dir=$rootdir/../output +fi + +function timing() { + direction="$1" + testname="$2" + + now=$(date +%s) + + if [ "$direction" = "enter" ]; then + export timing_stack="${timing_stack}/${now}" + export test_stack="${test_stack}/${testname}" + else + start_time=$(echo "$timing_stack" | sed -e 's@^.*/@@') + timing_stack=$(echo "$timing_stack" | sed -e 's@/[^/]*$@@') + + elapsed=$((now - start_time)) + echo "$elapsed $test_stack" >> $output_dir/timing.txt + + test_stack=$(echo "$test_stack" | sed -e 's@/[^/]*$@@') + fi +} + +function timing_enter() { + timing "enter" "$1" +} + +function timing_exit() { + timing "exit" "$1" +} + +function process_core() { + ret=0 + for core in $(find . -type f -name 'core*'); do + exe=$(eu-readelf -n "$core" | grep psargs | awk '{ print $2 }') + echo "exe for $core is $exe" + if [[ ! -z "$exe" ]]; then + if hash gdb; then + gdb -batch -ex "bt" $exe $core + fi + cp $exe $output_dir + fi + mv $core $output_dir + chmod a+r $output_dir/$core + ret=1 + done + return $ret +} + diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 000000000..34aff4861 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,46 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +# These directories contain tests. +TESTDIRS = lib + +DIRS-y = $(TESTDIRS) + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/test/lib/Makefile b/test/lib/Makefile new file mode 100644 index 000000000..64d26b615 --- /dev/null +++ b/test/lib/Makefile @@ -0,0 +1,43 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y = nvme memory + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/test/lib/memory/.gitignore b/test/lib/memory/.gitignore new file mode 100644 index 000000000..a03b46cc9 --- /dev/null +++ b/test/lib/memory/.gitignore @@ -0,0 +1 @@ +vtophys diff --git a/test/lib/memory/Makefile b/test/lib/memory/Makefile new file mode 100644 index 000000000..9612072c9 --- /dev/null +++ b/test/lib/memory/Makefile @@ -0,0 +1,48 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +C_SRCS = vtophys.c + +CFLAGS += $(DPDK_INC) +OBJS = $(SPDK_ROOT_DIR)/lib/memory/vtophys.o + +all: vtophys + +vtophys: vtophys.o $(OBJS) + @echo " LINK $@" + $(Q)$(CC) $(LDFLAGS) -o vtophys vtophys.o $(OBJS) -lpthread $(DPDK_LIB) -lrt + +clean: + $(Q)rm -f vtophys vtophys.o *.d diff --git a/test/lib/memory/memory.sh b/test/lib/memory/memory.sh new file mode 100755 index 000000000..abe76a10c --- /dev/null +++ b/test/lib/memory/memory.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir="$testdir/../../.." +source $rootdir/scripts/autotest_common.sh + +$testdir/vtophys +process_core diff --git a/test/lib/memory/vtophys.c b/test/lib/memory/vtophys.c new file mode 100644 index 000000000..1dfb0740b --- /dev/null +++ b/test/lib/memory/vtophys.c @@ -0,0 +1,136 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "spdk/vtophys.h" + +static const char *ealargs[] = { + "vtophys", + "-c 0x1", + "-n 4", +}; + +static int +vtophys_negative_test() +{ + void *p = NULL; + int i; + unsigned int size = 1; + int rc = 0; + + for (i = 0; i < 31; i++) { + p = malloc(size); + if (p == NULL) + continue; + + if (vtophys(p) != -1) { + rc = -1; + printf("Err: VA=%p is mapped to a huge_page,\n", p); + free(p); + break; + } + + free(p); + size = size << 1; + } + + if (!rc) + printf("vtophys_negative_test passed\n"); + else + printf("vtophys_negative_test failed\n"); + + return rc; +} + +static int +vtophys_positive_test() +{ + void *p = NULL; + int i; + unsigned int size = 1; + int rc = 0; + + for (i = 0; i < 31; i++) { + p = rte_malloc("vtophys_test", size, 512); + if (p == NULL) + continue; + + if (vtophys(p) == -1) { + rc = -1; + printf("Err: VA=%p is not mapped to a huge_page,\n", p); + rte_free(p); + break; + } + + rte_free(p); + size = size << 1; + } + + if (!rc) + printf("vtophys_positive_test passed\n"); + else + printf("vtophys_positive_test failed\n"); + + return rc; +} + + +int +main(int argc, char **argv) +{ + int rc; + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), + (char **)(void *)(uintptr_t)ealargs); + + if (rc < 0) { + fprintf(stderr, "Could not init eal\n"); + exit(1); + } + + rc = vtophys_negative_test(); + if (rc < 0) + return rc; + + rc = vtophys_positive_test(); + return rc; +} diff --git a/test/lib/nvme/Makefile b/test/lib/nvme/Makefile new file mode 100644 index 000000000..675b83aac --- /dev/null +++ b/test/lib/nvme/Makefile @@ -0,0 +1,43 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y = unit aer + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/test/lib/nvme/aer/.gitignore b/test/lib/nvme/aer/.gitignore new file mode 100644 index 000000000..313796176 --- /dev/null +++ b/test/lib/nvme/aer/.gitignore @@ -0,0 +1 @@ +aer diff --git a/test/lib/nvme/aer/Makefile b/test/lib/nvme/aer/Makefile new file mode 100644 index 000000000..688432c07 --- /dev/null +++ b/test/lib/nvme/aer/Makefile @@ -0,0 +1,59 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = aer + +C_SRCS := aer.c + +CFLAGS += -I. $(DPDK_INC) + +SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \ + $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \ + $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a + +LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt + +OBJS = $(C_SRCS:.c=.o) + +all : $(APP) + +objs : $(OBJS) + +$(APP) : $(OBJS) $(SPDK_LIBS) + @echo " LINK $@" + $(Q)$(CC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + +clean : + $(Q)rm -f $(OBJS) *.d $(APP) diff --git a/test/lib/nvme/aer/aer.c b/test/lib/nvme/aer/aer.c new file mode 100644 index 000000000..72e626119 --- /dev/null +++ b/test/lib/nvme/aer/aer.c @@ -0,0 +1,301 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include + +#include +#include +#include +#include + +#include "spdk/nvme.h" +#include "spdk/pci.h" + +struct rte_mempool *request_mempool; + +#define MAX_DEVS 64 + +struct dev { + struct pci_device *pci_dev; + struct nvme_controller *ctrlr; + struct nvme_health_information_page *health_page; + uint32_t orig_temp_threshold; + char name[100]; +}; + +static struct dev devs[MAX_DEVS]; +static int num_devs = 0; + +static int aer_done = 0; + + +#define foreach_dev(iter) \ + for (iter = devs; iter - devs < num_devs; iter++) + + +static int temperature_done = 0; + +static void set_feature_completion(void *arg, const struct nvme_completion *cpl) +{ + /* Admin command completions are synchronized by the NVMe driver, + * so we don't need to do any special locking here. */ + temperature_done++; +} + +static int +set_temp_threshold(struct dev *dev, uint32_t temp) +{ + struct nvme_command cmd = {0}; + + cmd.opc = NVME_OPC_SET_FEATURES; + cmd.cdw10 = NVME_FEAT_TEMPERATURE_THRESHOLD; + cmd.cdw11 = temp; + + return nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, set_feature_completion, dev); +} + +static void +get_feature_completion(void *cb_arg, const struct nvme_completion *cpl) +{ + struct dev *dev = cb_arg; + + if (nvme_completion_is_error(cpl)) { + printf("%s: get feature (temp threshold) failed\n", dev->name); + } else { + dev->orig_temp_threshold = cpl->cdw0; + printf("%s: original temperature threshold: %u Kelvin (%d Celsius)\n", + dev->name, dev->orig_temp_threshold, dev->orig_temp_threshold - 273); + } + + /* Set temperature threshold to a low value so the AER will trigger. */ + set_temp_threshold(dev, 200); +} + +static int +get_temp_threshold(struct dev *dev) +{ + struct nvme_command cmd = {0}; + + cmd.opc = NVME_OPC_GET_FEATURES; + cmd.cdw10 = NVME_FEAT_TEMPERATURE_THRESHOLD; + + return nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, get_feature_completion, dev); +} + +static void +print_health_page(struct dev *dev, struct nvme_health_information_page *hip) +{ + printf("%s: Current Temperature: %u Kelvin (%d Celsius)\n", + dev->name, hip->temperature, hip->temperature - 273); +} + +static void +get_log_page_completion(void *cb_arg, const struct nvme_completion *cpl) +{ + struct dev *dev = cb_arg; + + if (nvme_completion_is_error(cpl)) { + printf("%s: get log page failed\n", dev->name); + } else { + print_health_page(dev, dev->health_page); + } + + aer_done++; +} + +static int +get_health_log_page(struct dev *dev) +{ + struct nvme_command cmd = {0}; + + cmd.opc = NVME_OPC_GET_LOG_PAGE; + cmd.cdw10 = NVME_LOG_HEALTH_INFORMATION; + cmd.cdw10 |= (sizeof(*(dev->health_page)) / 4) << 16; // number of dwords + cmd.nsid = NVME_GLOBAL_NAMESPACE_TAG; + + return nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, dev->health_page, sizeof(*dev->health_page), + get_log_page_completion, dev); +} + +static void +cleanup(void) +{ + struct dev *dev; + + foreach_dev(dev) { + if (dev->health_page) { + rte_free(dev->health_page); + } + } +} + +static void aer_cb(void *arg, const struct nvme_completion *cpl) +{ + uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; + struct dev *dev = arg; + + printf("%s: aer_cb for log page %d\n", dev->name, log_page_id); + + /* Set the temperature threshold back to the original value + * so the AER doesn't trigger again. + */ + set_temp_threshold(dev, dev->orig_temp_threshold); + + get_health_log_page(dev); +} + +static const char *ealargs[] = { + "aer", + "-c 0x1", + "-n 4", +}; + +int main(int argc, char **argv) +{ + struct pci_device_iterator *pci_dev_iter; + struct pci_device *pci_dev; + struct dev *dev; + struct pci_id_match match; + int rc, i; + + printf("Asynchronous Event Request test\n"); + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), + (char **)(void *)(uintptr_t)ealargs); + + if (rc < 0) { + fprintf(stderr, "could not initialize dpdk\n"); + exit(1); + } + + request_mempool = rte_mempool_create("nvme_request", 8192, + nvme_request_size(), 128, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + + if (request_mempool == NULL) { + fprintf(stderr, "could not initialize request mempool\n"); + exit(1); + } + + pci_system_init(); + + match.vendor_id = PCI_MATCH_ANY; + match.subvendor_id = PCI_MATCH_ANY; + match.subdevice_id = PCI_MATCH_ANY; + match.device_id = PCI_MATCH_ANY; + match.device_class = NVME_CLASS_CODE; + match.device_class_mask = 0xFFFFFF; + + pci_dev_iter = pci_id_match_iterator_create(&match); + + rc = 0; + while ((pci_dev = pci_device_next(pci_dev_iter))) { + struct dev *dev; + + if (pci_device_has_kernel_driver(pci_dev) && + !pci_device_has_uio_driver(pci_dev)) { + fprintf(stderr, "non-uio kernel driver attached to nvme\n"); + fprintf(stderr, " controller at pci bdf %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + fprintf(stderr, " skipping...\n"); + continue; + } + + pci_device_probe(pci_dev); + + /* add to dev list */ + dev = &devs[num_devs++]; + + dev->pci_dev = pci_dev; + + snprintf(dev->name, sizeof(dev->name), "%04X:%02X:%02X.%02X", + pci_dev->domain, pci_dev->bus, pci_dev->dev, pci_dev->func); + + printf("%s: attaching NVMe driver...\n", dev->name); + + dev->health_page = rte_zmalloc("nvme health", sizeof(*dev->health_page), 4096); + if (dev->health_page == NULL) { + printf("Allocation error (health page)\n"); + rc = 1; + continue; /* TODO: just abort */ + } + + dev->ctrlr = nvme_attach(pci_dev); + if (dev->ctrlr == NULL) { + fprintf(stderr, "failed to attach to NVMe controller %s\n", dev->name); + rc = 1; + continue; /* TODO: just abort */ + } + } + + printf("Registering asynchronous event callbacks...\n"); + foreach_dev(dev) { + nvme_ctrlr_register_aer_callback(dev->ctrlr, aer_cb, dev); + } + + printf("Setting temperature thresholds...\n"); + foreach_dev(dev) { + /* Get the original temperature threshold and set it to a low value */ + get_temp_threshold(dev); + } + + while (temperature_done < num_devs) { + foreach_dev(dev) { + nvme_ctrlr_process_admin_completions(dev->ctrlr); + } + } + + printf("Waiting for all controllers to trigger AER...\n"); + + while (aer_done < num_devs) { + foreach_dev(dev) { + nvme_ctrlr_process_admin_completions(dev->ctrlr); + } + } + + printf("Cleaning up...\n"); + + for (i = 0; i < num_devs; i++) { + struct dev *dev = &devs[i]; + + nvme_detach(dev->ctrlr); + } + + cleanup(); + + pci_iterator_destroy(pci_dev_iter); + return rc; +} diff --git a/test/lib/nvme/nvme.sh b/test/lib/nvme/nvme.sh new file mode 100755 index 000000000..a5a285b4e --- /dev/null +++ b/test/lib/nvme/nvme.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir="$testdir/../../.." +source $rootdir/scripts/autotest_common.sh + +$testdir/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut +$testdir/unit/nvme_c/nvme_ut +$testdir/unit/nvme_qpair_c/nvme_qpair_ut +$testdir/unit/nvme_ctrlr_c/nvme_ctrlr_ut +$testdir/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut + +$testdir/aer/aer +process_core + +$rootdir/examples/nvme/identify/identify +process_core + +$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 5 +process_core diff --git a/test/lib/nvme/unit/Makefile b/test/lib/nvme/unit/Makefile new file mode 100644 index 000000000..0fe65a3a1 --- /dev/null +++ b/test/lib/nvme/unit/Makefile @@ -0,0 +1,43 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y = nvme_c nvme_ns_cmd_c nvme_qpair_c nvme_ctrlr_c nvme_ctrlr_cmd_c + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/test/lib/nvme/unit/nvme_c/.gitignore b/test/lib/nvme/unit/nvme_c/.gitignore new file mode 100644 index 000000000..90c0c1678 --- /dev/null +++ b/test/lib/nvme/unit/nvme_c/.gitignore @@ -0,0 +1 @@ +nvme_ut diff --git a/test/lib/nvme/unit/nvme_c/Makefile b/test/lib/nvme/unit/nvme_c/Makefile new file mode 100644 index 000000000..184080f0b --- /dev/null +++ b/test/lib/nvme/unit/nvme_c/Makefile @@ -0,0 +1,37 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +TEST_FILE = nvme_ut.c + +include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk + diff --git a/test/lib/nvme/unit/nvme_c/nvme_ut.c b/test/lib/nvme/unit/nvme_c/nvme_ut.c new file mode 100644 index 000000000..e0547009f --- /dev/null +++ b/test/lib/nvme/unit/nvme_c/nvme_ut.c @@ -0,0 +1,251 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "CUnit/Basic.h" + +#include "nvme/nvme.c" + +char outbuf[OUTBUF_SIZE]; + +volatile int sync_start = 0; +volatile int threads_pass = 0; +volatile int threads_fail = 0; + +uint64_t nvme_vtophys(void *buf) +{ + return (uintptr_t)buf; +} + +int +nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr) +{ + return 0; +} + +int +nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle) +{ + return 0; +} + +void +nvme_ctrlr_destruct(struct nvme_controller *ctrlr) +{ +} + +int +nvme_ctrlr_start(struct nvme_controller *ctrlr) +{ + return 0; +} + +void prepare_for_test(uint32_t max_io_queues) +{ + struct nvme_driver *driver = &g_nvme_driver; + + driver->max_io_queues = max_io_queues; + if (driver->ioq_index_pool != NULL) { + free(driver->ioq_index_pool); + driver->ioq_index_pool = NULL; + } + driver->ioq_index_pool_next = 0; + nvme_thread_ioq_index = -1; + + sync_start = 0; + threads_pass = 0; + threads_fail = 0; +} + +void * +nvme_thread(void *arg) +{ + int rc; + + /* Try to synchronize the nvme_register_io_thread() calls + * as much as possible to ensure the mutex locking is tested + * correctly. + */ + while (sync_start == 0) + ; + + rc = nvme_register_io_thread(); + if (rc == 0) { + __sync_fetch_and_add(&threads_pass, 1); + } else { + __sync_fetch_and_add(&threads_fail, 1); + } + + pthread_exit(NULL); +} + +void +test1(void) +{ + struct nvme_driver *driver = &g_nvme_driver; + int rc; + int last_index; + + prepare_for_test(1); + + CU_ASSERT(nvme_thread_ioq_index == -1); + + rc = nvme_register_io_thread(); + CU_ASSERT(rc == 0); + CU_ASSERT(nvme_thread_ioq_index >= 0); + CU_ASSERT(driver->ioq_index_pool_next == 1); + + /* try to register thread again - this should fail */ + last_index = nvme_thread_ioq_index; + rc = nvme_register_io_thread(); + CU_ASSERT(rc != 0); + /* assert that the ioq_index was unchanged */ + CU_ASSERT(nvme_thread_ioq_index == last_index); + + nvme_unregister_io_thread(); + CU_ASSERT(nvme_thread_ioq_index == -1); + CU_ASSERT(driver->ioq_index_pool_next == 0); +} + +void +test2(void) +{ + int num_threads = 16; + int i; + pthread_t td; + + /* + * Start 16 threads, but only simulate a maximum of 12 I/O + * queues. 12 threads should be able to successfully + * register, while the other 4 should fail. + */ + prepare_for_test(12); + + for (i = 0; i < num_threads; i++) { + pthread_create(&td, NULL, nvme_thread, NULL); + } + + sync_start = 1; + + while ((threads_pass + threads_fail) < num_threads) + ; + + CU_ASSERT(threads_pass == 12); + CU_ASSERT(threads_fail == 4); +} + +void +test_nvme_dump_command(void) +{ + struct nvme_command *cmd = NULL; + uint64_t physaddr = 0; + + cmd = nvme_malloc("nvme_command", sizeof(struct nvme_command), + 64, &physaddr); + CU_ASSERT(cmd != NULL); + + cmd->opc = 1; + cmd->fuse = 1; + cmd->rsvd1 = 1; + cmd->cid = 1; + cmd->nsid = 1; + cmd->rsvd2 = 1; + cmd->rsvd3 = 1; + cmd->mptr = 1; + cmd->dptr.prp.prp1 = 1; + cmd->dptr.prp.prp2 = 1; + cmd->cdw10 = 1; + cmd->cdw11 = 1; + cmd->cdw12 = 1; + cmd->cdw13 = 1; + cmd->cdw14 = 1; + cmd->cdw15 = 1; + + nvme_dump_command(cmd); + nvme_free(cmd); +} + +void +test_nvme_dump_completion(void) +{ + struct nvme_completion *cpl = NULL; + uint64_t physaddr = 0; + + cpl = nvme_malloc("nvme_completion", sizeof(struct nvme_completion), + 64, &physaddr); + CU_ASSERT(cpl != NULL); + + cpl->cdw0 = 1; + cpl->sqhd = 1; + cpl->sqid = 1; + cpl->cid = 1; + cpl->status.p = 1; + cpl->status.sc = 1; + cpl->status.sct = 1; + cpl->status.m = 1; + cpl->status.dnr = 1; + + nvme_dump_completion(cpl); + nvme_free(cpl); +} + +int main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + if (CU_initialize_registry() != CUE_SUCCESS) { + return CU_get_error(); + } + + suite = CU_add_suite("nvme", NULL, NULL); + if (suite == NULL) { + CU_cleanup_registry(); + return CU_get_error(); + } + + if ( + CU_add_test(suite, "test1", test1) == NULL + || CU_add_test(suite, "test2", test2) == NULL + || CU_add_test(suite, "nvme_dump_command", test_nvme_dump_command) == NULL + || CU_add_test(suite, "nvme_dump_completion", test_nvme_dump_completion) == NULL + ) { + CU_cleanup_registry(); + return CU_get_error(); + } + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +} diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/.gitignore b/test/lib/nvme/unit/nvme_ctrlr_c/.gitignore new file mode 100644 index 000000000..97a75bee8 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ctrlr_c/.gitignore @@ -0,0 +1 @@ +nvme_ctrlr_ut diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/Makefile b/test/lib/nvme/unit/nvme_ctrlr_c/Makefile new file mode 100644 index 000000000..4387e5eb1 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ctrlr_c/Makefile @@ -0,0 +1,37 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +TEST_FILE = nvme_ctrlr_ut.c + +include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk + diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c new file mode 100644 index 000000000..3bcfbcf78 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c @@ -0,0 +1,207 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme/nvme_internal.h" + +#include "CUnit/Basic.h" + +#include "nvme/nvme_ctrlr.c" + +struct nvme_driver g_nvme_driver = { + .lock = NVME_MUTEX_INITIALIZER, + .max_io_queues = NVME_MAX_IO_QUEUES +}; + +char outbuf[OUTBUF_SIZE]; + +int __thread nvme_thread_ioq_index = -1; + +int nvme_qpair_construct(struct nvme_qpair *qpair, uint16_t id, + uint16_t num_entries, uint16_t num_trackers, + struct nvme_controller *ctrlr) +{ + return 0; +} + +void +nvme_qpair_fail(struct nvme_qpair *qpair) +{ +} + +void +nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) +{ + CU_ASSERT(req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST); +} + +void +nvme_qpair_process_completions(struct nvme_qpair *qpair) +{ +} + +void +nvme_qpair_disable(struct nvme_qpair *qpair) +{ +} + +void +nvme_qpair_destroy(struct nvme_qpair *qpair) +{ +} + +void +nvme_qpair_enable(struct nvme_qpair *qpair) +{ +} + +void +nvme_qpair_reset(struct nvme_qpair *qpair) +{ +} + +void +nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl) +{ +} + +void +nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ +} + +void +nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr, + uint32_t num_queues, nvme_cb_fn_t cb_fn, void *cb_arg) +{ +} + +void +nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr, + struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, + void *cb_arg) +{ +} + +void +nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr, + struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, + void *cb_arg) +{ +} + +void +nvme_ns_destruct(struct nvme_namespace *ns) +{ +} + +int +nvme_ns_construct(struct nvme_namespace *ns, uint16_t id, + struct nvme_controller *ctrlr) +{ + return 0; +} + + +struct nvme_request * +nvme_allocate_request(void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req = NULL; + nvme_alloc_request(&req); + + if (req != NULL) { + memset(req, 0, offsetof(struct nvme_request, children)); + + if (payload == NULL || payload_size == 0) { + req->u.payload = NULL; + req->payload_size = 0; + } else { + req->u.payload = payload; + req->payload_size = payload_size; + } + + req->cb_fn = cb_fn; + req->cb_arg = cb_arg; + req->timeout = true; + } + return req; +} + +void +test_nvme_ctrlr_fail() +{ + struct nvme_controller *ctrlr = NULL; + struct nvme_qpair qpair = {}; + uint64_t phys_addr = 0; + + ctrlr = nvme_malloc("nvme_controller", sizeof(struct nvme_controller), + 64, &phys_addr); + CU_ASSERT(ctrlr != NULL); + + ctrlr->num_io_queues = 0; + ctrlr->adminq = qpair; + nvme_ctrlr_fail(ctrlr); + + CU_ASSERT(ctrlr->is_failed == true); + nvme_free(ctrlr); +} + +int main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + if (CU_initialize_registry() != CUE_SUCCESS) { + return CU_get_error(); + } + + suite = CU_add_suite("nvme_ctrlr", NULL, NULL); + if (suite == NULL) { + CU_cleanup_registry(); + return CU_get_error(); + } + + if ( + CU_add_test(suite, "test nvme_ctrlr function nvme_ctrlr_fail", test_nvme_ctrlr_fail) == NULL + ) { + CU_cleanup_registry(); + return CU_get_error(); + } + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +} diff --git a/test/lib/nvme/unit/nvme_ctrlr_cmd_c/.gitignore b/test/lib/nvme/unit/nvme_ctrlr_cmd_c/.gitignore new file mode 100644 index 000000000..1568b4763 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ctrlr_cmd_c/.gitignore @@ -0,0 +1 @@ +nvme_ctrlr_cmd_ut diff --git a/test/lib/nvme/unit/nvme_ctrlr_cmd_c/Makefile b/test/lib/nvme/unit/nvme_ctrlr_cmd_c/Makefile new file mode 100644 index 000000000..195e00df5 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ctrlr_cmd_c/Makefile @@ -0,0 +1,37 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +TEST_FILE = nvme_ctrlr_cmd_ut.c + +include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk + diff --git a/test/lib/nvme/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut.c b/test/lib/nvme/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut.c new file mode 100644 index 000000000..05a9ff805 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut.c @@ -0,0 +1,310 @@ + +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "nvme/nvme_internal.h" + +#include "CUnit/Basic.h" + +#include "nvme/nvme_ctrlr_cmd.c" + +char outbuf[OUTBUF_SIZE]; + +struct nvme_command *cmd = NULL; + +uint64_t nvme_vtophys(void *buf) +{ + return (uintptr_t)buf; +} + +typedef void (*verify_request_fn_t)(struct nvme_request *req); +verify_request_fn_t verify_fn; + +void verify_firmware_log_page(struct nvme_request *req) +{ + cmd = &req->cmd; + CU_ASSERT(cmd->opc == NVME_OPC_GET_LOG_PAGE); + nvme_free_request(req); +} + +void verify_health_log_page(struct nvme_request *req) +{ + cmd = &req->cmd; + CU_ASSERT(cmd->opc == NVME_OPC_GET_LOG_PAGE); + nvme_free_request(req); +} + +void verify_error_log_page(struct nvme_request *req) +{ + cmd = &req->cmd; + CU_ASSERT(cmd->opc == NVME_OPC_GET_LOG_PAGE); + nvme_free_request(req); +} + +void verify_get_feature_cmd(struct nvme_request *req) +{ + cmd = &req->cmd; + CU_ASSERT(cmd->opc == NVME_OPC_GET_FEATURES); + nvme_free_request(req); +} + +void verify_abort_cmd(struct nvme_request *req) +{ + cmd = &req->cmd; + CU_ASSERT(cmd->opc == NVME_OPC_ABORT); + nvme_free_request(req); +} + +void verify_io_raw_cmd(struct nvme_request *req) +{ + struct nvme_command command = {0}; + uint64_t phys_addr = 0; + int rc = 100; + + + cmd = &req->cmd; + CU_ASSERT(cmd != NULL); + rc = memcmp(cmd, &command, sizeof(cmd)); + CU_ASSERT(rc == 0); + nvme_free_request(req); +} + +struct nvme_request * +nvme_allocate_request(void *payload, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req = NULL; + nvme_alloc_request(&req); + + if (req != NULL) { + memset(req, 0, offsetof(struct nvme_request, children)); + + if (payload == NULL || payload_size == 0) { + req->u.payload = NULL; + req->payload_size = 0; + } else { + req->u.payload = payload; + req->payload_size = payload_size; + } + + req->cb_fn = cb_fn; + req->cb_arg = cb_arg; + req->timeout = true; + } + return req; +} + +void +nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, + struct nvme_request *req) +{ + verify_fn(req); +} + +void +nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, struct nvme_request *req) +{ + verify_fn(req); +} + + +void +test_firmware_get_log_page() +{ + struct nvme_controller ctrlr = {}; + struct nvme_firmware_page *payload = NULL; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + uint64_t phys_addr = 0; + + payload = nvme_malloc("nvme_firmware_page", sizeof(struct nvme_firmware_page), + 64, &phys_addr); + CU_ASSERT(payload != NULL); + + verify_fn = verify_firmware_log_page; + + nvme_ctrlr_cmd_get_firmware_page(&ctrlr, + payload, cb_fn, cb_arg); + + nvme_free(payload); +} + +void +test_health_get_log_page() +{ + struct nvme_controller ctrlr = {}; + struct nvme_health_information_page *payload = NULL; + uint32_t nsid = 0; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + uint64_t phys_addr = 0; + + payload = nvme_malloc("nvme_health_information_page", sizeof(struct nvme_health_information_page), + 64, &phys_addr); + CU_ASSERT(payload != NULL); + + verify_fn = verify_health_log_page; + + nvme_ctrlr_cmd_get_health_information_page(&ctrlr, nsid, + payload, cb_fn, cb_arg); + + nvme_free(payload); +} + +void +test_error_get_log_page() +{ + struct nvme_controller *ctrlr = NULL; + struct nvme_controller_data *ctrldata = NULL; + struct nvme_error_information_entry *payload = NULL; + uint32_t num_entries = 1; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + uint64_t phys_addr = 0; + + payload = nvme_malloc("nvme_error_information_entry", sizeof(struct nvme_error_information_entry), + 64, &phys_addr); + CU_ASSERT(payload != NULL); + + ctrlr = nvme_malloc("nvme_controller", sizeof(struct nvme_controller), + 64, &phys_addr); + CU_ASSERT(ctrlr != NULL); + + ctrldata = nvme_malloc("nvme_controller_data", sizeof(struct nvme_controller_data), + 64, &phys_addr); + CU_ASSERT(ctrldata != NULL); + + ctrlr->cdata = *ctrldata; + ctrlr->cdata.elpe = 5; + + verify_fn = verify_error_log_page; + + nvme_ctrlr_cmd_get_error_page(ctrlr, payload, + num_entries, cb_fn, cb_arg); + num_entries = 50; + nvme_ctrlr_cmd_get_error_page(ctrlr, payload, + num_entries, cb_fn, cb_arg); + + + nvme_free(payload); + nvme_free(ctrlr); + nvme_free(ctrldata); +} + +void +test_get_feature_cmd() +{ + struct nvme_controller ctrlr = {}; + uint8_t feature = 1; + uint32_t cdw11 = 1; + void *payload = NULL; + uint32_t payload_size = 0; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + + verify_fn = verify_get_feature_cmd; + + nvme_ctrlr_cmd_get_feature(&ctrlr, feature, cdw11, payload, + payload_size, cb_fn, cb_arg); +} + +void +test_abort_cmd() +{ + struct nvme_controller ctrlr = {}; + uint16_t cid = 0; + uint16_t sqid = 0; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + + verify_fn = verify_abort_cmd; + + nvme_ctrlr_cmd_abort(&ctrlr, cid, sqid, cb_fn, cb_arg); +} + +void +test_io_raw_cmd() +{ + struct nvme_controller ctrlr = {}; + struct nvme_command *cmd = NULL; + void *buf = NULL; + uint32_t len = 1; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + uint64_t phys_addr = 0; + + cmd = nvme_malloc("nvme_command", sizeof(struct nvme_command), + 64, &phys_addr); + CU_ASSERT(cmd != NULL); + memset(cmd, 0, sizeof(cmd)); + + verify_fn = verify_io_raw_cmd; + + nvme_ctrlr_cmd_io_raw(&ctrlr, cmd, buf, len, cb_fn, cb_arg); + nvme_free(cmd); +} + +int main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + if (CU_initialize_registry() != CUE_SUCCESS) { + return CU_get_error(); + } + + suite = CU_add_suite("nvme_ctrlr_cmd", NULL, NULL); + if (suite == NULL) { + CU_cleanup_registry(); + return CU_get_error(); + } + + if ( + CU_add_test(suite, "test ctrlr cmd get_firmware_page", test_firmware_get_log_page) == NULL + || CU_add_test(suite, "test ctrlr cmd get_health_page", test_health_get_log_page) == NULL + || CU_add_test(suite, "test ctrlr cmd get_error_page", test_error_get_log_page) == NULL + || CU_add_test(suite, "test ctrlr cmd get_feature", test_get_feature_cmd) == NULL + || CU_add_test(suite, "test ctrlr cmd abort_cmd", test_abort_cmd) == NULL + || CU_add_test(suite, "test ctrlr cmd io_raw_cmd", test_io_raw_cmd) == NULL + ) { + CU_cleanup_registry(); + return CU_get_error(); + } + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +} diff --git a/test/lib/nvme/unit/nvme_impl.h b/test/lib/nvme/unit/nvme_impl.h new file mode 100644 index 000000000..7512e1145 --- /dev/null +++ b/test/lib/nvme/unit/nvme_impl.h @@ -0,0 +1,102 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NVME_IMPL_H__ +#define __NVME_IMPL_H__ + +#include +#include +#include + +#define nvme_malloc(tag, size, align, phys_addr) malloc(size) +#define nvme_free(buf) free(buf) +#define OUTBUF_SIZE 1024 +extern char outbuf[OUTBUF_SIZE]; +#define nvme_printf(ctrlr, fmt, args...) snprintf(outbuf, OUTBUF_SIZE, fmt, ##args) +#define nvme_get_num_ioq() 8 +#define nvme_get_ioq_idx() 0 +#define nvme_assert(check, str) \ +do \ + { \ + if (!(check)) \ + printf str; \ + } \ + while (0); +uint64_t nvme_vtophys(void *buf); +#define nvme_alloc_request(bufp) \ +do \ + { \ + *bufp = malloc(sizeof(struct nvme_request)); \ + } \ + while (0); + +#define nvme_free_request(buf) free(buf) +#define nvme_pcicfg_read32(handle, var, offset) +#define nvme_pcicfg_write32(handle, var, offset) + +static inline +int nvme_pcicfg_map_bar(void *pci_handle, int bar, int read_only, void **addr) +{ + *addr = NULL; + return 0; +} + +static inline int +nvme_pcicfg_unmap_bar(void *devhandle, uint32_t bar, void *addr) +{ + return 0; +} + +typedef pthread_mutex_t nvme_mutex_t; + +#define nvme_mutex_init(x) pthread_mutex_init((x), NULL) +#define nvme_mutex_destroy(x) pthread_mutex_destroy((x)) +#define nvme_mutex_lock pthread_mutex_lock +#define nvme_mutex_unlock pthread_mutex_unlock +#define NVME_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +static inline int +nvme_mutex_init_recursive(nvme_mutex_t *mtx) +{ + pthread_mutexattr_t attr; + + if (pthread_mutexattr_init(&attr)) { + return -1; + } + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) { + return -1; + } + return pthread_mutex_init(mtx, &attr); +} + +#endif /* __NVME_IMPL_H__ */ diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/.gitignore b/test/lib/nvme/unit/nvme_ns_cmd_c/.gitignore new file mode 100644 index 000000000..5583ec23e --- /dev/null +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/.gitignore @@ -0,0 +1 @@ +nvme_ns_cmd_ut diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/Makefile b/test/lib/nvme/unit/nvme_ns_cmd_c/Makefile new file mode 100644 index 000000000..3a3b8960a --- /dev/null +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/Makefile @@ -0,0 +1,38 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +TEST_FILE = nvme_ns_cmd_ut.c +OTHER_FILES = nvme.c + +include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk + diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c new file mode 100644 index 000000000..e40d4e352 --- /dev/null +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c @@ -0,0 +1,243 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme/nvme_internal.h" + +#include "CUnit/Basic.h" + +#include "nvme/nvme_ns_cmd.c" + +char outbuf[OUTBUF_SIZE]; + +struct nvme_request *g_request = NULL; + +uint64_t nvme_vtophys(void *buf) +{ + return (uintptr_t)buf; +} + +int +nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr) +{ + return 0; +} + +int +nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle) +{ + return 0; +} + +void +nvme_ctrlr_destruct(struct nvme_controller *ctrlr) +{ +} + +int +nvme_ctrlr_start(struct nvme_controller *ctrlr) +{ + return 0; +} + +uint32_t +nvme_ns_get_sector_size(struct nvme_namespace *ns) +{ + return ns->sector_size; +} + +uint32_t +nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns) +{ + return ns->ctrlr->max_xfer_size; +} + +void +nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, + struct nvme_request *req) +{ + g_request = req; +} + +void +prepare_for_test(struct nvme_namespace *ns, struct nvme_controller *ctrlr, + uint32_t sector_size, uint32_t max_xfer_size, + uint32_t stripe_size) +{ + ctrlr->max_xfer_size = max_xfer_size; + ns->ctrlr = ctrlr; + ns->sector_size = sector_size; + ns->stripe_size = stripe_size; + ns->sectors_per_max_io = nvme_ns_get_max_io_xfer_size(ns) / ns->sector_size; + ns->sectors_per_stripe = ns->stripe_size / ns->sector_size; + + g_request = NULL; +} + +void +split_test(void) +{ + struct nvme_namespace ns; + struct nvme_controller ctrlr; + void *payload; + uint64_t lba; + uint32_t lba_count; + int rc; + + prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0); + payload = 0x0; + lba = 0; + lba_count = 1; + + rc = nvme_ns_cmd_read(&ns, payload, lba, lba_count, NULL, NULL); + + CU_ASSERT(rc == 0); + CU_ASSERT(g_request != NULL); + CU_ASSERT(g_request->num_children == 0); + + nvme_free_request(g_request); +} + +void +split_test2(void) +{ + struct nvme_namespace ns; + struct nvme_controller ctrlr; + struct nvme_request *child; + void *payload; + uint64_t lba; + uint32_t lba_count; + int rc; + + prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0); + payload = malloc(256 * 1024); + lba = 0; + lba_count = (256 * 1024) / 512; + + rc = nvme_ns_cmd_read(&ns, payload, lba, lba_count, NULL, NULL); + + CU_ASSERT(rc == 0); + CU_ASSERT(g_request != NULL); + CU_ASSERT(g_request->num_children == 2); + + child = TAILQ_FIRST(&g_request->children); + TAILQ_REMOVE(&g_request->children, child, child_tailq); + CU_ASSERT(child->num_children == 0); + CU_ASSERT(child->payload_size == 128 * 1024); + + child = TAILQ_FIRST(&g_request->children); + TAILQ_REMOVE(&g_request->children, child, child_tailq); + CU_ASSERT(child->num_children == 0); + CU_ASSERT(child->payload_size == 128 * 1024); + + CU_ASSERT(TAILQ_EMPTY(&g_request->children)); + + free(payload); + nvme_free_request(g_request); +} + +void +test_nvme_ns_cmd_flush(void) +{ + struct nvme_namespace ns; + struct nvme_controller ctrlr; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + + prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0); + + nvme_ns_cmd_flush(&ns, cb_fn, cb_arg); + CU_ASSERT(g_request->cmd.opc == NVME_OPC_FLUSH); + CU_ASSERT(g_request->cmd.nsid == ns.id); + + nvme_free_request(g_request); +} + +void +test_nvme_ns_cmd_deallocate(void) +{ + struct nvme_namespace ns; + struct nvme_controller ctrlr; + nvme_cb_fn_t cb_fn = NULL; + void *cb_arg = NULL; + uint8_t num_ranges = 1; + void *payload = NULL; + int rc = 0; + + prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0); + payload = malloc(num_ranges * sizeof(struct nvme_dsm_range)); + + nvme_ns_cmd_deallocate(&ns, payload, num_ranges, cb_fn, cb_arg); + CU_ASSERT(g_request->cmd.opc == NVME_OPC_DATASET_MANAGEMENT); + CU_ASSERT(g_request->cmd.nsid == ns.id); + CU_ASSERT(g_request->cmd.cdw10 == num_ranges - 1); + CU_ASSERT(g_request->cmd.cdw11 == NVME_DSM_ATTR_DEALLOCATE); + free(payload); + nvme_free_request(g_request); + + payload = NULL; + num_ranges = 0; + rc = nvme_ns_cmd_deallocate(&ns, payload, num_ranges, cb_fn, cb_arg); + CU_ASSERT(rc != 0); +} + +int main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + if (CU_initialize_registry() != CUE_SUCCESS) { + return CU_get_error(); + } + + suite = CU_add_suite("nvme_ns_cmd", NULL, NULL); + if (suite == NULL) { + CU_cleanup_registry(); + return CU_get_error(); + } + + if ( + CU_add_test(suite, "split_test", split_test) == NULL + || CU_add_test(suite, "split_test2", split_test2) == NULL + || CU_add_test(suite, "nvme_ns_cmd_flush testing", test_nvme_ns_cmd_flush) == NULL + || CU_add_test(suite, "nvme_ns_cmd_deallocate testing", test_nvme_ns_cmd_deallocate) == NULL + ) { + CU_cleanup_registry(); + return CU_get_error(); + } + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +} diff --git a/test/lib/nvme/unit/nvme_qpair_c/.gitignore b/test/lib/nvme/unit/nvme_qpair_c/.gitignore new file mode 100644 index 000000000..1bb18e997 --- /dev/null +++ b/test/lib/nvme/unit/nvme_qpair_c/.gitignore @@ -0,0 +1 @@ +nvme_qpair_ut diff --git a/test/lib/nvme/unit/nvme_qpair_c/Makefile b/test/lib/nvme/unit/nvme_qpair_c/Makefile new file mode 100644 index 000000000..009d572b7 --- /dev/null +++ b/test/lib/nvme/unit/nvme_qpair_c/Makefile @@ -0,0 +1,37 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +TEST_FILE = nvme_qpair_ut.c + +include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk + diff --git a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c new file mode 100644 index 000000000..5ce8cd195 --- /dev/null +++ b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c @@ -0,0 +1,433 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "nvme/nvme_internal.h" + +#include "CUnit/Basic.h" + +#include "nvme/nvme_qpair.c" + +struct nvme_driver g_nvme_driver = { + .lock = NVME_MUTEX_INITIALIZER, + .max_io_queues = NVME_MAX_IO_QUEUES, +}; + +int32_t nvme_retry_count = 1; + +char outbuf[OUTBUF_SIZE]; + +bool fail_vtophys = false; + +uint64_t nvme_vtophys(void *buf) +{ + if (fail_vtophys) { + return (uint64_t) - 1; + } else { + return (uintptr_t)buf; + } +} + +void nvme_dump_completion(struct nvme_completion *cpl) +{ +} + +void prepare_for_test(void) +{ +} + +void +test1(void) +{ + struct nvme_qpair qpair = {}; + struct nvme_command cmd = {}; + + outbuf[0] = '\0'; + + /* + * qpair.id == 0 means it is an admin queue. Ensure + * that the opc is decoded as an admin opc and not an + * I/o opc. + */ + qpair.id = 0; + cmd.opc = NVME_OPC_IDENTIFY; + + nvme_qpair_print_command(&qpair, &cmd); + + CU_ASSERT(strstr(outbuf, "IDENTIFY") != NULL); +} + +void +test2(void) +{ + struct nvme_qpair qpair = {}; + struct nvme_command cmd = {}; + + outbuf[0] = '\0'; + + /* + * qpair.id != 0 means it is an I/O queue. Ensure + * that the opc is decoded as an I/O opc and not an + * admin opc. + */ + qpair.id = 1; + cmd.opc = NVME_OPC_DATASET_MANAGEMENT; + + nvme_qpair_print_command(&qpair, &cmd); + + CU_ASSERT(strstr(outbuf, "DATASET MANAGEMENT") != NULL); +} + +void +prepare_submit_request_test(struct nvme_qpair *qpair, + struct nvme_controller *ctrlr, + struct nvme_registers *regs) +{ + memset(ctrlr, 0, sizeof(*ctrlr)); + ctrlr->regs = regs; + nvme_qpair_construct(qpair, 1, 128, 32, ctrlr); + + CU_ASSERT(qpair->sq_tail == 0); + CU_ASSERT(qpair->cq_head == 0); + + fail_vtophys = false; +} + +void +cleanup_submit_request_test(struct nvme_qpair *qpair) +{ + nvme_qpair_destroy(qpair); +} + +void +expected_success_callback(void *arg, const struct nvme_completion *cpl) +{ + CU_ASSERT(!nvme_completion_is_error(cpl)); +} + +void +expected_failure_callback(void *arg, const struct nvme_completion *cpl) +{ + CU_ASSERT(nvme_completion_is_error(cpl)); +} + +void +test3(void) +{ + struct nvme_qpair qpair = {}; + struct nvme_request *req; + struct nvme_controller ctrlr = {}; + struct nvme_registers regs = {}; + + prepare_submit_request_test(&qpair, &ctrlr, ®s); + + nvme_alloc_request(&req); + req->payload_size = 4096; + req->cb_fn = expected_success_callback; + + CU_ASSERT(qpair.sq_tail == 0); + + nvme_qpair_submit_request(&qpair, req); + + CU_ASSERT(qpair.sq_tail == 1); + + cleanup_submit_request_test(&qpair); + nvme_free_request(req); +} + +void +test4(void) +{ + struct nvme_qpair qpair = {}; + struct nvme_request *req; + struct nvme_controller ctrlr = {}; + struct nvme_registers regs = {}; + + prepare_submit_request_test(&qpair, &ctrlr, ®s); + + nvme_alloc_request(&req); + req->payload_size = 4096; + req->cb_fn = expected_failure_callback; + + /* Force vtophys to return a failure. This should + * result in the nvme_qpair manually failing + * the request with error status to signify + * a bad payload buffer. + */ + fail_vtophys = true; + outbuf[0] = '\0'; + + CU_ASSERT(qpair.sq_tail == 0); + + nvme_qpair_submit_request(&qpair, req); + + CU_ASSERT(qpair.sq_tail == 0); + /* Assert that command/completion data was printed to log. */ + CU_ASSERT(strlen(outbuf) > 0); + + cleanup_submit_request_test(&qpair); +} + +void +test_ctrlr_failed(void) +{ + struct nvme_qpair qpair = {}; + struct nvme_request *req; + struct nvme_controller ctrlr = {}; + struct nvme_registers regs = {}; + + prepare_submit_request_test(&qpair, &ctrlr, ®s); + + nvme_alloc_request(&req); + req->payload_size = 4096; + req->cb_fn = expected_failure_callback; + + /* Disable the queue and set the controller to failed. + * Set the controller to resetting so that the qpair won't get re-enabled. + */ + qpair.is_enabled = false; + ctrlr.is_failed = true; + ctrlr.is_resetting = true; + + outbuf[0] = '\0'; + + CU_ASSERT(qpair.sq_tail == 0); + + nvme_qpair_submit_request(&qpair, req); + + CU_ASSERT(qpair.sq_tail == 0); + /* Assert that command/completion data was printed to log. */ + CU_ASSERT(strlen(outbuf) > 0); + + cleanup_submit_request_test(&qpair); +} + +void struct_packing(void) +{ + /* ctrlr is the first field in nvme_qpair after the fields + * that are used in the I/O path. Make sure the I/O path fields + * all fit into two cache lines. + */ + CU_ASSERT(offsetof(struct nvme_qpair, ctrlr) <= 128); +} + +void test_nvme_qpair_fail(void) +{ + struct nvme_qpair qpair = {0}; + struct nvme_request *req = NULL; + struct nvme_controller ctrlr = {}; + struct nvme_registers regs = {0}; + struct nvme_tracker *tr_temp; + uint64_t phys_addr = 0; + + prepare_submit_request_test(&qpair, &ctrlr, ®s); + + tr_temp = nvme_malloc("nvme_tracker", sizeof(struct nvme_tracker), + 64, &phys_addr); + tr_temp->req = NULL; + nvme_alloc_request(&tr_temp->req); + + LIST_INSERT_HEAD(&qpair.outstanding_tr, tr_temp, list); + nvme_qpair_fail(&qpair); + CU_ASSERT_TRUE(LIST_EMPTY(&qpair.outstanding_tr)); + + nvme_alloc_request(&req); + STAILQ_INSERT_HEAD(&qpair.queued_req, req, stailq); + nvme_qpair_fail(&qpair); + CU_ASSERT_TRUE(STAILQ_EMPTY(&qpair.queued_req)); + + cleanup_submit_request_test(&qpair); +} + +void test_nvme_qpair_process_completions(void) +{ + struct nvme_qpair qpair = {0}; + struct nvme_request *req = NULL; + struct nvme_controller ctrlr = {}; + struct nvme_registers regs = {0}; + + prepare_submit_request_test(&qpair, &ctrlr, ®s); + qpair.is_enabled = false; + qpair.ctrlr->is_resetting = true; + + nvme_qpair_process_completions(&qpair); + cleanup_submit_request_test(&qpair); +} + +void test_nvme_qpair_destroy(void) +{ + struct nvme_qpair qpair = {0}; + struct nvme_request *req = NULL; + struct nvme_controller ctrlr = {}; + struct nvme_registers regs = {0}; + struct nvme_tracker *tr_temp; + uint64_t phys_addr = 0; + + memset(&ctrlr, 0, sizeof(ctrlr)); + ctrlr.regs = ®s; + nvme_qpair_construct(&qpair, 1, 128, 32, &ctrlr); + qpair.cmd = nvme_malloc("nvme_command", sizeof(struct nvme_command), + 64, &phys_addr); + + qpair.cpl = nvme_malloc("nvme_completion", sizeof(struct nvme_completion), + 64, &phys_addr); + + qpair.act_tr = nvme_malloc("nvme_tracker", sizeof(struct nvme_tracker), + 64, &phys_addr); + + nvme_qpair_destroy(&qpair); +} + +void test_nvme_completion_is_retry(void) +{ + struct nvme_completion *cpl = NULL; + uint64_t phys_addr = 0; + bool ret_val = false; + + cpl = nvme_malloc("nvme_completion", sizeof(struct nvme_completion), + 64, &phys_addr); + + cpl->status.sct = NVME_SCT_GENERIC; + cpl->status.sc = NVME_SC_ABORTED_BY_REQUEST; + cpl->status.dnr = 0; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_TRUE(ret_val); + + cpl->status.sc = NVME_SC_INVALID_OPCODE; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_INVALID_FIELD; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_COMMAND_ID_CONFLICT; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_DATA_TRANSFER_ERROR; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_ABORTED_POWER_LOSS; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_INTERNAL_DEVICE_ERROR; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_ABORTED_FAILED_FUSED; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_ABORTED_MISSING_FUSED; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_INVALID_NAMESPACE_OR_FORMAT; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_COMMAND_SEQUENCE_ERROR; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_LBA_OUT_OF_RANGE; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = NVME_SC_CAPACITY_EXCEEDED; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sc = 0x70; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sct = NVME_SCT_COMMAND_SPECIFIC; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sct = NVME_SCT_MEDIA_ERROR; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sct = NVME_SCT_VENDOR_SPECIFIC; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + cpl->status.sct = 0x4; + ret_val = nvme_completion_is_retry(cpl); + CU_ASSERT_FALSE(ret_val); + + +} + +int main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + if (CU_initialize_registry() != CUE_SUCCESS) { + return CU_get_error(); + } + + suite = CU_add_suite("nvme_qpair", NULL, NULL); + if (suite == NULL) { + CU_cleanup_registry(); + return CU_get_error(); + } + + if ( + CU_add_test(suite, "test1", test1) == NULL + || CU_add_test(suite, "test2", test2) == NULL + || CU_add_test(suite, "test3", test3) == NULL + || CU_add_test(suite, "test4", test4) == NULL + || CU_add_test(suite, "ctrlr_failed", test_ctrlr_failed) == NULL + || CU_add_test(suite, "struct_packing", struct_packing) == NULL + || CU_add_test(suite, "nvme_qpair_fail", test_nvme_qpair_fail) == NULL + || CU_add_test(suite, "nvme_qpair_process_completions", test_nvme_qpair_process_completions) == NULL + || CU_add_test(suite, "nvme_qpair_destroy", test_nvme_qpair_destroy) == NULL + || CU_add_test(suite, "nvme_completion_is_retry", test_nvme_completion_is_retry) == NULL + ) { + CU_cleanup_registry(); + return CU_get_error(); + } + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +}