306 lines
8.3 KiB
C
306 lines
8.3 KiB
C
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||
|
|
/*
|
||
|
|
* Shared Memory Communications over RDMA (SMC-R) and RoCE
|
||
|
|
*
|
||
|
|
* Connection Data Control (CDC)
|
||
|
|
*
|
||
|
|
* Copyright IBM Corp. 2016
|
||
|
|
*
|
||
|
|
* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef SMC_CDC_H
|
||
|
|
#define SMC_CDC_H
|
||
|
|
|
||
|
|
#include <linux/kernel.h> /* max_t */
|
||
|
|
#include <linux/atomic.h>
|
||
|
|
#include <linux/in.h>
|
||
|
|
#include <linux/compiler.h>
|
||
|
|
|
||
|
|
#include "smc.h"
|
||
|
|
#include "smc_core.h"
|
||
|
|
#include "smc_wr.h"
|
||
|
|
|
||
|
|
#define SMC_CDC_MSG_TYPE 0xFE
|
||
|
|
|
||
|
|
/* in network byte order */
|
||
|
|
union smc_cdc_cursor { /* SMC cursor */
|
||
|
|
struct {
|
||
|
|
__be16 reserved;
|
||
|
|
__be16 wrap;
|
||
|
|
__be32 count;
|
||
|
|
};
|
||
|
|
#ifdef KERNEL_HAS_ATOMIC64
|
||
|
|
atomic64_t acurs; /* for atomic processing */
|
||
|
|
#else
|
||
|
|
u64 acurs; /* for atomic processing */
|
||
|
|
#endif
|
||
|
|
} __aligned(8);
|
||
|
|
|
||
|
|
/* in network byte order */
|
||
|
|
struct smc_cdc_msg {
|
||
|
|
struct smc_wr_rx_hdr common; /* .type = 0xFE */
|
||
|
|
u8 len; /* 44 */
|
||
|
|
__be16 seqno;
|
||
|
|
__be32 token;
|
||
|
|
union smc_cdc_cursor prod;
|
||
|
|
union smc_cdc_cursor cons; /* piggy backed "ack" */
|
||
|
|
struct smc_cdc_producer_flags prod_flags;
|
||
|
|
struct smc_cdc_conn_state_flags conn_state_flags;
|
||
|
|
u8 reserved[18];
|
||
|
|
};
|
||
|
|
|
||
|
|
/* SMC-D cursor format */
|
||
|
|
union smcd_cdc_cursor {
|
||
|
|
struct {
|
||
|
|
u16 wrap;
|
||
|
|
u32 count;
|
||
|
|
struct smc_cdc_producer_flags prod_flags;
|
||
|
|
struct smc_cdc_conn_state_flags conn_state_flags;
|
||
|
|
} __packed;
|
||
|
|
#ifdef KERNEL_HAS_ATOMIC64
|
||
|
|
atomic64_t acurs; /* for atomic processing */
|
||
|
|
#else
|
||
|
|
u64 acurs; /* for atomic processing */
|
||
|
|
#endif
|
||
|
|
} __aligned(8);
|
||
|
|
|
||
|
|
/* CDC message for SMC-D */
|
||
|
|
struct smcd_cdc_msg {
|
||
|
|
struct smc_wr_rx_hdr common; /* Type = 0xFE */
|
||
|
|
u8 res1[7];
|
||
|
|
union smcd_cdc_cursor prod;
|
||
|
|
union smcd_cdc_cursor cons;
|
||
|
|
u8 res3[8];
|
||
|
|
} __aligned(8);
|
||
|
|
|
||
|
|
static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
|
||
|
|
conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline bool smc_cdc_rxed_any_close_or_senddone(
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
return smc_cdc_rxed_any_close(conn) ||
|
||
|
|
conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smc_curs_add(int size, union smc_host_cursor *curs,
|
||
|
|
int value)
|
||
|
|
{
|
||
|
|
curs->count += value;
|
||
|
|
if (curs->count >= size) {
|
||
|
|
curs->wrap++;
|
||
|
|
curs->count -= size;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Copy cursor src into tgt */
|
||
|
|
static inline void smc_curs_copy(union smc_host_cursor *tgt,
|
||
|
|
union smc_host_cursor *src,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
#ifndef KERNEL_HAS_ATOMIC64
|
||
|
|
unsigned long flags;
|
||
|
|
|
||
|
|
spin_lock_irqsave(&conn->acurs_lock, flags);
|
||
|
|
tgt->acurs = src->acurs;
|
||
|
|
spin_unlock_irqrestore(&conn->acurs_lock, flags);
|
||
|
|
#else
|
||
|
|
atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
|
||
|
|
union smc_cdc_cursor *src,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
#ifndef KERNEL_HAS_ATOMIC64
|
||
|
|
unsigned long flags;
|
||
|
|
|
||
|
|
spin_lock_irqsave(&conn->acurs_lock, flags);
|
||
|
|
tgt->acurs = src->acurs;
|
||
|
|
spin_unlock_irqrestore(&conn->acurs_lock, flags);
|
||
|
|
#else
|
||
|
|
atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
|
||
|
|
union smcd_cdc_cursor *src,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
#ifndef KERNEL_HAS_ATOMIC64
|
||
|
|
unsigned long flags;
|
||
|
|
|
||
|
|
spin_lock_irqsave(&conn->acurs_lock, flags);
|
||
|
|
tgt->acurs = src->acurs;
|
||
|
|
spin_unlock_irqrestore(&conn->acurs_lock, flags);
|
||
|
|
#else
|
||
|
|
atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
/* calculate cursor difference between old and new, where old <= new and
|
||
|
|
* difference cannot exceed size
|
||
|
|
*/
|
||
|
|
static inline int smc_curs_diff(unsigned int size,
|
||
|
|
union smc_host_cursor *old,
|
||
|
|
union smc_host_cursor *new)
|
||
|
|
{
|
||
|
|
if (old->wrap != new->wrap)
|
||
|
|
return max_t(int, 0,
|
||
|
|
((size - old->count) + new->count));
|
||
|
|
|
||
|
|
return max_t(int, 0, (new->count - old->count));
|
||
|
|
}
|
||
|
|
|
||
|
|
/* calculate cursor difference between old and new - returns negative
|
||
|
|
* value in case old > new
|
||
|
|
*/
|
||
|
|
static inline int smc_curs_comp(unsigned int size,
|
||
|
|
union smc_host_cursor *old,
|
||
|
|
union smc_host_cursor *new)
|
||
|
|
{
|
||
|
|
if (old->wrap > new->wrap ||
|
||
|
|
(old->wrap == new->wrap && old->count > new->count))
|
||
|
|
return -smc_curs_diff(size, new, old);
|
||
|
|
return smc_curs_diff(size, old, new);
|
||
|
|
}
|
||
|
|
|
||
|
|
/* calculate cursor difference between old and new, where old <= new and
|
||
|
|
* difference may exceed size
|
||
|
|
*/
|
||
|
|
static inline int smc_curs_diff_large(unsigned int size,
|
||
|
|
union smc_host_cursor *old,
|
||
|
|
union smc_host_cursor *new)
|
||
|
|
{
|
||
|
|
if (old->wrap < new->wrap)
|
||
|
|
return min_t(int,
|
||
|
|
(size - old->count) + new->count +
|
||
|
|
(new->wrap - old->wrap - 1) * size,
|
||
|
|
size);
|
||
|
|
|
||
|
|
if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
|
||
|
|
return min_t(int,
|
||
|
|
(size - old->count) + new->count +
|
||
|
|
(new->wrap + 0xffff - old->wrap) * size,
|
||
|
|
size);
|
||
|
|
|
||
|
|
return max_t(int, 0, (new->count - old->count));
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
|
||
|
|
union smc_host_cursor *local,
|
||
|
|
union smc_host_cursor *save,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
smc_curs_copy(save, local, conn);
|
||
|
|
peer->count = htonl(save->count);
|
||
|
|
peer->wrap = htons(save->wrap);
|
||
|
|
/* peer->reserved = htons(0); must be ensured by caller */
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
|
||
|
|
struct smc_connection *conn,
|
||
|
|
union smc_host_cursor *save)
|
||
|
|
{
|
||
|
|
struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
|
||
|
|
|
||
|
|
peer->common.type = local->common.type;
|
||
|
|
peer->len = local->len;
|
||
|
|
peer->seqno = htons(local->seqno);
|
||
|
|
peer->token = htonl(local->token);
|
||
|
|
smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
|
||
|
|
smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
|
||
|
|
peer->prod_flags = local->prod_flags;
|
||
|
|
peer->conn_state_flags = local->conn_state_flags;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
|
||
|
|
union smc_cdc_cursor *peer,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
union smc_host_cursor temp, old;
|
||
|
|
union smc_cdc_cursor net;
|
||
|
|
|
||
|
|
smc_curs_copy(&old, local, conn);
|
||
|
|
smc_curs_copy_net(&net, peer, conn);
|
||
|
|
temp.count = ntohl(net.count);
|
||
|
|
temp.wrap = ntohs(net.wrap);
|
||
|
|
if ((old.wrap > temp.wrap) && temp.wrap)
|
||
|
|
return;
|
||
|
|
if ((old.wrap == temp.wrap) &&
|
||
|
|
(old.count > temp.count))
|
||
|
|
return;
|
||
|
|
smc_curs_copy(local, &temp, conn);
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
|
||
|
|
struct smc_cdc_msg *peer,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
local->common.type = peer->common.type;
|
||
|
|
local->len = peer->len;
|
||
|
|
local->seqno = ntohs(peer->seqno);
|
||
|
|
local->token = ntohl(peer->token);
|
||
|
|
smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
|
||
|
|
smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
|
||
|
|
local->prod_flags = peer->prod_flags;
|
||
|
|
local->conn_state_flags = peer->conn_state_flags;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
|
||
|
|
struct smcd_cdc_msg *peer,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
union smc_host_cursor temp;
|
||
|
|
|
||
|
|
temp.wrap = peer->prod.wrap;
|
||
|
|
temp.count = peer->prod.count;
|
||
|
|
smc_curs_copy(&local->prod, &temp, conn);
|
||
|
|
|
||
|
|
temp.wrap = peer->cons.wrap;
|
||
|
|
temp.count = peer->cons.count;
|
||
|
|
smc_curs_copy(&local->cons, &temp, conn);
|
||
|
|
local->prod_flags = peer->cons.prod_flags;
|
||
|
|
local->conn_state_flags = peer->cons.conn_state_flags;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
|
||
|
|
struct smc_cdc_msg *peer,
|
||
|
|
struct smc_connection *conn)
|
||
|
|
{
|
||
|
|
if (conn->lgr->is_smcd)
|
||
|
|
smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
|
||
|
|
else
|
||
|
|
smcr_cdc_msg_to_host(local, peer, conn);
|
||
|
|
}
|
||
|
|
|
||
|
|
struct smc_cdc_tx_pend {
|
||
|
|
struct smc_connection *conn; /* socket connection */
|
||
|
|
union smc_host_cursor cursor; /* tx sndbuf cursor sent */
|
||
|
|
union smc_host_cursor p_cursor; /* rx RMBE cursor produced */
|
||
|
|
u16 ctrl_seq; /* conn. tx sequence # */
|
||
|
|
};
|
||
|
|
|
||
|
|
int smc_cdc_get_free_slot(struct smc_connection *conn,
|
||
|
|
struct smc_link *link,
|
||
|
|
struct smc_wr_buf **wr_buf,
|
||
|
|
struct smc_rdma_wr **wr_rdma_buf,
|
||
|
|
struct smc_cdc_tx_pend **pend);
|
||
|
|
void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
|
||
|
|
int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
|
||
|
|
struct smc_cdc_tx_pend *pend);
|
||
|
|
int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
|
||
|
|
int smcd_cdc_msg_send(struct smc_connection *conn);
|
||
|
|
int smcr_cdc_msg_send_validation(struct smc_connection *conn,
|
||
|
|
struct smc_cdc_tx_pend *pend,
|
||
|
|
struct smc_wr_buf *wr_buf);
|
||
|
|
int smc_cdc_init(void) __init;
|
||
|
|
void smcd_cdc_rx_init(struct smc_connection *conn);
|
||
|
|
|
||
|
|
#endif /* SMC_CDC_H */
|