326 lines
10 KiB
C
326 lines
10 KiB
C
|
/*-
|
||
|
|
* BSD LICENSE
|
||
|
|
*
|
||
|
|
* Copyright (c) Intel Corporation.
|
||
|
|
* All rights reserved.
|
||
|
|
*
|
||
|
|
* Redistribution and use in source and binary forms, with or without
|
||
|
|
* modification, are permitted provided that the following conditions
|
||
|
|
* are met:
|
||
|
|
*
|
||
|
|
* * Redistributions of source code must retain the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer.
|
||
|
|
* * Redistributions in binary form must reproduce the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer in
|
||
|
|
* the documentation and/or other materials provided with the
|
||
|
|
* distribution.
|
||
|
|
* * Neither the name of Intel Corporation nor the names of its
|
||
|
|
* contributors may be used to endorse or promote products derived
|
||
|
|
* from this software without specific prior written permission.
|
||
|
|
*
|
||
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef SPDK_BDEV_RAID_INTERNAL_H
|
||
|
|
#define SPDK_BDEV_RAID_INTERNAL_H
|
||
|
|
|
||
|
|
#include <stdatomic.h>
|
||
|
|
#include "spdk/bdev_module.h"
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Raid state describes the state of the longhorn. This longhorn bdev can be either in
|
||
|
|
* configured list or configuring list
|
||
|
|
*/
|
||
|
|
enum longhorn_bdev_state {
|
||
|
|
/* longhorn bdev is ready and is seen by upper layers */
|
||
|
|
RAID_BDEV_STATE_ONLINE,
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn bdev is configuring, not all underlying bdevs are present.
|
||
|
|
* And can't be seen by upper layers.
|
||
|
|
*/
|
||
|
|
RAID_BDEV_STATE_CONFIGURING,
|
||
|
|
|
||
|
|
/*
|
||
|
|
* In offline state, longhorn bdev layer will complete all incoming commands without
|
||
|
|
* submitting to underlying base nvme bdevs
|
||
|
|
*/
|
||
|
|
RAID_BDEV_STATE_OFFLINE,
|
||
|
|
|
||
|
|
/* longhorn bdev max, new states should be added before this */
|
||
|
|
RAID_BDEV_MAX
|
||
|
|
};
|
||
|
|
|
||
|
|
enum longhorn_base_bdev_state {
|
||
|
|
LONGHORN_BASE_BDEV_RW,
|
||
|
|
LONGHORN_BASE_BDEV_WO,
|
||
|
|
LONGHORN_BASE_BDEV_ERR
|
||
|
|
};
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_base_bdev_info contains information for the base bdevs which are part of some
|
||
|
|
* longhorn. This structure contains the per base bdev information. Whatever is
|
||
|
|
* required per base device for longhorn bdev will be kept here
|
||
|
|
*/
|
||
|
|
struct longhorn_base_bdev_info {
|
||
|
|
/* pointer to base spdk bdev */
|
||
|
|
struct spdk_bdev *bdev;
|
||
|
|
|
||
|
|
/* pointer to base bdev descriptor opened by longhorn bdev */
|
||
|
|
struct spdk_bdev_desc *desc;
|
||
|
|
|
||
|
|
|
||
|
|
//struct spdk_io_channel *base_channel;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* When underlying base device calls the hot plug function on drive removal,
|
||
|
|
* this flag will be set and later after doing some processing, base device
|
||
|
|
* descriptor will be closed
|
||
|
|
*/
|
||
|
|
bool remove_scheduled;
|
||
|
|
|
||
|
|
/* thread where base device is opened */
|
||
|
|
struct spdk_thread *thread;
|
||
|
|
|
||
|
|
enum longhorn_base_bdev_state state;
|
||
|
|
|
||
|
|
bool is_local;
|
||
|
|
char *lvs;
|
||
|
|
char *remote_addr;
|
||
|
|
uint16_t nvmf_port;
|
||
|
|
uint16_t comm_port;
|
||
|
|
char *bdev_name;
|
||
|
|
|
||
|
|
TAILQ_ENTRY(longhorn_base_bdev_info) infos;
|
||
|
|
};
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_bdev_io is the context part of bdev_io. It contains the information
|
||
|
|
* related to bdev_io for a longhorn bdev
|
||
|
|
*/
|
||
|
|
struct longhorn_bdev_io {
|
||
|
|
/* The longhorn bdev associated with this IO */
|
||
|
|
struct longhorn_bdev *longhorn_bdev;
|
||
|
|
|
||
|
|
/* WaitQ entry, used only in waitq logic */
|
||
|
|
struct spdk_bdev_io_wait_entry waitq_entry;
|
||
|
|
|
||
|
|
/* Context of the original channel for this IO */
|
||
|
|
struct longhorn_bdev_io_channel *longhorn_ch;
|
||
|
|
|
||
|
|
/* Used for tracking progress on io requests sent to member disks. */
|
||
|
|
uint64_t base_bdev_io_remaining;
|
||
|
|
uint8_t base_bdev_io_submitted;
|
||
|
|
uint8_t base_bdev_io_status;
|
||
|
|
|
||
|
|
bool submitted;
|
||
|
|
|
||
|
|
};
|
||
|
|
|
||
|
|
TAILQ_HEAD(base_bdevs, longhorn_base_bdev_info);
|
||
|
|
|
||
|
|
struct longhorn_base_io_channel {
|
||
|
|
struct spdk_io_channel *base_channel;
|
||
|
|
|
||
|
|
struct longhorn_base_bdev_info *base_info;
|
||
|
|
|
||
|
|
TAILQ_ENTRY(longhorn_base_io_channel) channels;
|
||
|
|
};
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_bdev_io_channel is the context of spdk_io_channel for longhorn bdev device. It
|
||
|
|
* contains the relationship of longhorn bdev io channel with base bdev io channels.
|
||
|
|
*/
|
||
|
|
struct longhorn_bdev_io_channel {
|
||
|
|
struct longhorn_bdev *longhorn_bdev;
|
||
|
|
/* Array of IO channels of base bdevs */
|
||
|
|
//struct spdk_io_channel **base_channel;
|
||
|
|
|
||
|
|
/* Number of IO channels */
|
||
|
|
uint8_t num_channels;
|
||
|
|
|
||
|
|
struct spdk_thread *thread;
|
||
|
|
bool paused;
|
||
|
|
bool pause_complete;
|
||
|
|
bool deleted;
|
||
|
|
|
||
|
|
atomic_int io_ops;
|
||
|
|
|
||
|
|
TAILQ_HEAD(, longhorn_base_io_channel) base_channels;
|
||
|
|
|
||
|
|
TAILQ_ENTRY(longhorn_bdev_io_channel) channels;
|
||
|
|
|
||
|
|
struct longhorn_base_io_channel *last_read_io_ch;
|
||
|
|
};
|
||
|
|
|
||
|
|
TAILQ_HEAD(io_channels, longhorn_bdev_io_channel);
|
||
|
|
|
||
|
|
typedef void (*longhorn_pause_cb)(struct longhorn_bdev *bdev, void *arg);
|
||
|
|
struct longhorn_pause_cb_entry {
|
||
|
|
longhorn_pause_cb cb_fn;
|
||
|
|
void *cb_arg;
|
||
|
|
|
||
|
|
TAILQ_ENTRY(longhorn_pause_cb_entry) link;
|
||
|
|
};
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_bdev is the single entity structure which contains SPDK block device
|
||
|
|
* and the information related to any longhorn bdev either configured or
|
||
|
|
* in configuring list. io device is created on this.
|
||
|
|
*/
|
||
|
|
struct longhorn_bdev {
|
||
|
|
/* longhorn bdev device, this will get registered in bdev layer */
|
||
|
|
struct spdk_bdev bdev;
|
||
|
|
|
||
|
|
/* link of longhorn bdev to link it to configured, configuring or offline list */
|
||
|
|
TAILQ_ENTRY(longhorn_bdev) state_link;
|
||
|
|
|
||
|
|
/* link of longhorn bdev to link it to global longhorn bdev list */
|
||
|
|
TAILQ_ENTRY(longhorn_bdev) global_link;
|
||
|
|
|
||
|
|
TAILQ_HEAD(, longhorn_pause_cb_entry) pause_cbs;
|
||
|
|
|
||
|
|
/* pointer to config file entry */
|
||
|
|
struct longhorn_bdev_config *config;
|
||
|
|
|
||
|
|
|
||
|
|
/* array of base bdev info */
|
||
|
|
struct longhorn_base_bdev_info *base_bdev_info;
|
||
|
|
|
||
|
|
pthread_mutex_t base_bdevs_mutex;
|
||
|
|
struct base_bdevs base_bdevs_head;
|
||
|
|
struct io_channels io_channel_head;
|
||
|
|
|
||
|
|
uint32_t num_io_channels;
|
||
|
|
|
||
|
|
struct longhorn_base_bdev_info *last_read_info;
|
||
|
|
|
||
|
|
/* block length bit shift for optimized calculation */
|
||
|
|
uint32_t blocklen_shift;
|
||
|
|
|
||
|
|
/* state of longhorn bdev */
|
||
|
|
enum longhorn_bdev_state state;
|
||
|
|
|
||
|
|
/* number of base bdevs comprising longhorn bdev */
|
||
|
|
uint8_t num_base_bdevs;
|
||
|
|
|
||
|
|
/* number of base bdevs discovered */
|
||
|
|
uint8_t num_base_bdevs_discovered;
|
||
|
|
|
||
|
|
/* Set to true if destruct is called for this longhorn bdev */
|
||
|
|
bool destruct_called;
|
||
|
|
|
||
|
|
/* Set to true if destroy of this longhorn bdev is started. */
|
||
|
|
bool destroy_started;
|
||
|
|
|
||
|
|
atomic_int io_ops;
|
||
|
|
atomic_int channels_to_pause;
|
||
|
|
|
||
|
|
};
|
||
|
|
|
||
|
|
#define LONGHORN_FOR_EACH_BASE_BDEV(r, i) \
|
||
|
|
for (i = r->base_bdev_info; i < r->base_bdev_info + r->num_base_bdevs; i++)
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_base_bdev_config is the per base bdev data structure which contains
|
||
|
|
* information w.r.t to per base bdev during parsing config
|
||
|
|
*/
|
||
|
|
struct longhorn_base_bdev_config {
|
||
|
|
/* base bdev name from config file */
|
||
|
|
char *name;
|
||
|
|
};
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_bdev_config contains the longhorn bdev config related information after
|
||
|
|
* parsing the config file
|
||
|
|
*/
|
||
|
|
struct longhorn_bdev_config {
|
||
|
|
/* base bdev config per underlying bdev */
|
||
|
|
struct longhorn_base_bdev_config *base_bdev;
|
||
|
|
|
||
|
|
/* Points to already created longhorn bdev */
|
||
|
|
struct longhorn_bdev *longhorn_bdev;
|
||
|
|
|
||
|
|
char *name;
|
||
|
|
|
||
|
|
/* number of base bdevs */
|
||
|
|
uint8_t num_base_bdevs;
|
||
|
|
|
||
|
|
TAILQ_ENTRY(longhorn_bdev_config) link;
|
||
|
|
};
|
||
|
|
|
||
|
|
/*
|
||
|
|
* longhorn_config is the top level structure representing the longhorn bdev config as read
|
||
|
|
* from config file for all longhorns
|
||
|
|
*/
|
||
|
|
struct longhorn_config {
|
||
|
|
/* longhorn bdev context from config file */
|
||
|
|
TAILQ_HEAD(, longhorn_bdev_config) longhorn_bdev_config_head;
|
||
|
|
|
||
|
|
/* total longhorn bdev from config file */
|
||
|
|
uint8_t total_longhorn_bdev;
|
||
|
|
};
|
||
|
|
|
||
|
|
/* TAIL heads for various longhorn bdev lists */
|
||
|
|
TAILQ_HEAD(longhorn_configured_tailq, longhorn_bdev);
|
||
|
|
TAILQ_HEAD(longhorn_configuring_tailq, longhorn_bdev);
|
||
|
|
TAILQ_HEAD(longhorn_all_tailq, longhorn_bdev);
|
||
|
|
TAILQ_HEAD(longhorn_offline_tailq, longhorn_bdev);
|
||
|
|
|
||
|
|
extern struct longhorn_configured_tailq g_longhorn_bdev_configured_list;
|
||
|
|
extern struct longhorn_configuring_tailq g_longhorn_bdev_configuring_list;
|
||
|
|
extern struct longhorn_all_tailq g_longhorn_bdev_list;
|
||
|
|
extern struct longhorn_offline_tailq g_longhorn_bdev_offline_list;
|
||
|
|
extern struct longhorn_config g_longhorn_config;
|
||
|
|
|
||
|
|
typedef void (*longhorn_bdev_destruct_cb)(void *cb_ctx, int rc);
|
||
|
|
|
||
|
|
//int longhorn_bdev_create(struct longhorn_bdev_config *longhorn_cfg);
|
||
|
|
int longhorn_bdev_create(const char *name, uint8_t num_base_bdevs);
|
||
|
|
int longhorn_bdev_add_base_devices(struct longhorn_bdev_config *longhorn_cfg);
|
||
|
|
void longhorn_bdev_remove_base_devices(struct longhorn_bdev_config *longhorn_cfg,
|
||
|
|
longhorn_bdev_destruct_cb cb_fn, void *cb_ctx);
|
||
|
|
int longhorn_bdev_config_add(const char *longhorn_name, uint8_t num_base_bdevs,
|
||
|
|
struct longhorn_bdev_config **_longhorn_cfg);
|
||
|
|
int longhorn_bdev_config_add_base_bdev(struct longhorn_bdev_config *longhorn_cfg,
|
||
|
|
const char *base_bdev_name, uint8_t slot);
|
||
|
|
void longhorn_bdev_config_cleanup(struct longhorn_bdev_config *longhorn_cfg);
|
||
|
|
struct longhorn_bdev_config *longhorn_bdev_config_find_by_name(const char *longhorn_name);
|
||
|
|
|
||
|
|
|
||
|
|
bool
|
||
|
|
longhorn_bdev_io_complete_part(struct longhorn_bdev_io *longhorn_io, uint64_t completed,
|
||
|
|
enum spdk_bdev_io_status status);
|
||
|
|
void
|
||
|
|
longhorn_bdev_queue_io_wait(struct longhorn_bdev_io *longhorn_io, struct spdk_bdev *bdev,
|
||
|
|
struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn);
|
||
|
|
void
|
||
|
|
longhorn_bdev_io_complete(struct longhorn_bdev_io *longhorn_io, enum spdk_bdev_io_status status);
|
||
|
|
|
||
|
|
//int longhorn_bdev_add_base_device(const char *name, const char *bdev_name);
|
||
|
|
int longhorn_bdev_add_base_device(struct longhorn_bdev *longhorn_bdev, struct longhorn_base_bdev_info *base_info);
|
||
|
|
int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port);
|
||
|
|
int
|
||
|
|
longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port);
|
||
|
|
|
||
|
|
void bdev_longhorn_pause_io(void *cb_arg);
|
||
|
|
void bdev_longhorn_unpause_io(void *cb_arg);
|
||
|
|
struct longhorn_bdev *longhorn_bdev_find_by_name(const char *longhorn_name);
|
||
|
|
|
||
|
|
void longhorn_unpause(struct longhorn_bdev *longhorn_bdev);
|
||
|
|
void longhorn_volume_add_pause_cb(struct longhorn_bdev *longhorn_dev,
|
||
|
|
longhorn_pause_cb cb_fn,
|
||
|
|
void *cb_arg);
|
||
|
|
|
||
|
|
#endif /* SPDK_BDEV_RAID_INTERNAL_H */
|