Merge branch 'nfp-ipsec-offload'

Simon Horman says:

====================
nfp: IPsec offload support

Huanhuan Wang says:

this series adds support for IPsec offload to the NFP driver.

It covers three enhancements:

1. Patches 1/3:
   - Extend the capability word and control word to to support
     new features.

2. Patch 2/3:
   - Add framework to support IPsec offloading for NFP driver,
     but IPsec offload control plane interface xfrm callbacks which
     interact with upper layer are not implemented in this patch.

3. Patch 3/3:
   - IPsec control plane interface xfrm callbacks are implemented
     in this patch.

Changes since v3
* Remove structure fields that describe firmware but
  are not used for Kernel offload
* Add WARN_ON(!xa_empty()) before call to xa_destroy()
* Added helpers for hash methods

Changes since v2
* OFFLOAD_HANDLE_ERROR macro and the associated code removed
* Unnecessary logging removed
* Hook function xdo_dev_state_free in struct xfrmdev_ops removed
* Use Xarray to maintain SA entries

Changes since v1
* Explicitly return failure when XFRM_STATE_ESN is set
* Fix the issue that AEAD algorithm is not correctly offloaded
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

10 files changed, 735 insertions, 15 deletions

diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig
index 8844d1ac053a..e785c00b5845 100644
--- a/drivers/net/ethernet/netronome/Kconfig
+++ b/drivers/net/ethernet/netronome/Kconfig
@@ -54,6 +54,17 @@ config NFP_APP_ABM_NIC
 	  functionality.
 	  Code will be built into the nfp.ko driver.
 
+config NFP_NET_IPSEC
+	bool "NFP IPsec crypto offload support"
+	depends on NFP
+	depends on XFRM_OFFLOAD
+	default y
+	help
+	  Enable driver support IPsec crypto offload on NFP NIC.
+	  Say Y, if you are planning to make use of IPsec crypto
+	  offload. NOTE that IPsec crypto offload on NFP NIC
+	  requires specific FW to work.
+
 config NFP_DEBUG
 	bool "Debug support for Netronome(R) NFP4000/NFP6000 NIC drivers"
 	depends on NFP
diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile
index 9c0861d03634..8a250214e289 100644
--- a/drivers/net/ethernet/netronome/nfp/Makefile
+++ b/drivers/net/ethernet/netronome/nfp/Makefile
@@ -80,4 +80,6 @@ nfp-objs += \
 	    abm/main.o
 endif
 
+nfp-$(CONFIG_NFP_NET_IPSEC) += crypto/ipsec.o nfd3/ipsec.o
+
 nfp-$(CONFIG_NFP_DEBUG) += nfp_net_debugfs.o
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/crypto.h b/drivers/net/ethernet/netronome/nfp/crypto/crypto.h
index bffe58bb2f27..1df73d658938 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/crypto.h
+++ b/drivers/net/ethernet/netronome/nfp/crypto/crypto.h
@@ -39,4 +39,27 @@ nfp_net_tls_rx_resync_req(struct net_device *netdev,
 }
 #endif
 
+/* IPsec related structures and functions */
+struct nfp_ipsec_offload {
+	u32 seq_hi;
+	u32 seq_low;
+	u32 handle;
+};
+
+#ifndef CONFIG_NFP_NET_IPSEC
+static inline void nfp_net_ipsec_init(struct nfp_net *nn)
+{
+}
+
+static inline void nfp_net_ipsec_clean(struct nfp_net *nn)
+{
+}
+#else
+void nfp_net_ipsec_init(struct nfp_net *nn);
+void nfp_net_ipsec_clean(struct nfp_net *nn);
+bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,
+			   struct nfp_ipsec_offload *offload_info);
+int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb);
+#endif
+
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
new file mode 100644
index 000000000000..3728870d8e9c
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
@@ -0,0 +1,587 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright (C) 2018 Netronome Systems, Inc */
+/* Copyright (C) 2021 Corigine, Inc */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/netdevice.h>
+#include <asm/unaligned.h>
+#include <linux/ktime.h>
+#include <net/xfrm.h>
+
+#include "../nfp_net_ctrl.h"
+#include "../nfp_net.h"
+#include "crypto.h"
+
+#define NFP_NET_IPSEC_MAX_SA_CNT  (16 * 1024) /* Firmware support a maximum of 16K SA offload */
+
+/* IPsec config message cmd codes */
+enum nfp_ipsec_cfg_mssg_cmd_codes {
+	NFP_IPSEC_CFG_MSSG_ADD_SA,	 /* Add a new SA */
+	NFP_IPSEC_CFG_MSSG_INV_SA	 /* Invalidate an existing SA */
+};
+
+/* IPsec config message response codes */
+enum nfp_ipsec_cfg_mssg_rsp_codes {
+	NFP_IPSEC_CFG_MSSG_OK,
+	NFP_IPSEC_CFG_MSSG_FAILED,
+	NFP_IPSEC_CFG_MSSG_SA_VALID,
+	NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED,
+	NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED,
+	NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD
+};
+
+/* Protocol */
+enum nfp_ipsec_sa_prot {
+	NFP_IPSEC_PROTOCOL_AH = 0,
+	NFP_IPSEC_PROTOCOL_ESP = 1
+};
+
+/* Mode */
+enum nfp_ipsec_sa_mode {
+	NFP_IPSEC_PROTMODE_TRANSPORT = 0,
+	NFP_IPSEC_PROTMODE_TUNNEL = 1
+};
+
+/* Cipher types */
+enum nfp_ipsec_sa_cipher {
+	NFP_IPSEC_CIPHER_NULL,
+	NFP_IPSEC_CIPHER_3DES,
+	NFP_IPSEC_CIPHER_AES128,
+	NFP_IPSEC_CIPHER_AES192,
+	NFP_IPSEC_CIPHER_AES256,
+	NFP_IPSEC_CIPHER_AES128_NULL,
+	NFP_IPSEC_CIPHER_AES192_NULL,
+	NFP_IPSEC_CIPHER_AES256_NULL,
+	NFP_IPSEC_CIPHER_CHACHA20
+};
+
+/* Cipher modes */
+enum nfp_ipsec_sa_cipher_mode {
+	NFP_IPSEC_CIMODE_ECB,
+	NFP_IPSEC_CIMODE_CBC,
+	NFP_IPSEC_CIMODE_CFB,
+	NFP_IPSEC_CIMODE_OFB,
+	NFP_IPSEC_CIMODE_CTR
+};
+
+/* Hash types */
+enum nfp_ipsec_sa_hash_type {
+	NFP_IPSEC_HASH_NONE,
+	NFP_IPSEC_HASH_MD5_96,
+	NFP_IPSEC_HASH_SHA1_96,
+	NFP_IPSEC_HASH_SHA256_96,
+	NFP_IPSEC_HASH_SHA384_96,
+	NFP_IPSEC_HASH_SHA512_96,
+	NFP_IPSEC_HASH_MD5_128,
+	NFP_IPSEC_HASH_SHA1_80,
+	NFP_IPSEC_HASH_SHA256_128,
+	NFP_IPSEC_HASH_SHA384_192,
+	NFP_IPSEC_HASH_SHA512_256,
+	NFP_IPSEC_HASH_GF128_128,
+	NFP_IPSEC_HASH_POLY1305_128
+};
+
+/* IPSEC_CFG_MSSG_ADD_SA */
+struct nfp_ipsec_cfg_add_sa {
+	u32 ciph_key[8];		  /* Cipher Key */
+	union {
+		u32 auth_key[16];	  /* Authentication Key */
+		struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */
+			u32 salt;	  /* Initialized with SA */
+			u32 resv[15];
+		} aesgcm_fields;
+	};
+	struct sa_ctrl_word {
+		uint32_t hash   :4;	  /* From nfp_ipsec_sa_hash_type */
+		uint32_t cimode :4;	  /* From nfp_ipsec_sa_cipher_mode */
+		uint32_t cipher :4;	  /* From nfp_ipsec_sa_cipher */
+		uint32_t mode   :2;	  /* From nfp_ipsec_sa_mode */
+		uint32_t proto  :2;	  /* From nfp_ipsec_sa_prot */
+		uint32_t dir :1;	  /* SA direction */
+		uint32_t resv0 :12;
+		uint32_t encap_dsbl:1;	  /* Encap/Decap disable */
+		uint32_t resv1 :2;	  /* Must be set to 0 */
+	} ctrl_word;
+	u32 spi;			  /* SPI Value */
+	uint32_t pmtu_limit :16;          /* PMTU Limit */
+	uint32_t resv0 :5;
+	uint32_t ipv6       :1;		  /* Outbound IPv6 addr format */
+	uint32_t resv1	 :10;
+	u32 resv2[2];
+	u32 src_ip[4];			  /* Src IP addr */
+	u32 dst_ip[4];			  /* Dst IP addr */
+	u32 resv3[6];
+};
+
+/* IPSEC_CFG_MSSG */
+struct nfp_ipsec_cfg_mssg {
+	union {
+		struct{
+			uint32_t cmd:16;     /* One of nfp_ipsec_cfg_mssg_cmd_codes */
+			uint32_t rsp:16;     /* One of nfp_ipsec_cfg_mssg_rsp_codes */
+			uint32_t sa_idx:16;  /* SA table index */
+			uint32_t spare0:16;
+			struct nfp_ipsec_cfg_add_sa cfg_add_sa;
+		};
+		u32 raw[64];
+	};
+};
+
+static int nfp_ipsec_cfg_cmd_issue(struct nfp_net *nn, int type, int saidx,
+				   struct nfp_ipsec_cfg_mssg *msg)
+{
+	int i, msg_size, ret;
+
+	msg->cmd = type;
+	msg->sa_idx = saidx;
+	msg->rsp = 0;
+	msg_size = ARRAY_SIZE(msg->raw);
+
+	for (i = 0; i < msg_size; i++)
+		nn_writel(nn, NFP_NET_CFG_MBOX_VAL + 4 * i, msg->raw[i]);
+
+	ret = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_IPSEC);
+	if (ret < 0)
+		return ret;
+
+	/* For now we always read the whole message response back */
+	for (i = 0; i < msg_size; i++)
+		msg->raw[i] = nn_readl(nn, NFP_NET_CFG_MBOX_VAL + 4 * i);
+
+	switch (msg->rsp) {
+	case NFP_IPSEC_CFG_MSSG_OK:
+		return 0;
+	case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD:
+		return -EINVAL;
+	case NFP_IPSEC_CFG_MSSG_SA_VALID:
+		return -EEXIST;
+	case NFP_IPSEC_CFG_MSSG_FAILED:
+	case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED:
+	case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED:
+		return -EIO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen)
+{
+	bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC);
+
+	switch (keylen) {
+	case 128:
+		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL :
+						   NFP_IPSEC_CIPHER_AES128;
+		break;
+	case 192:
+		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL :
+						   NFP_IPSEC_CIPHER_AES192;
+		break;
+	case 256:
+		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL :
+						   NFP_IPSEC_CIPHER_AES256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
+{
+	switch (*trunc_len) {
+	case 96:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96;
+		break;
+	case 128:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128;
+		break;
+	default:
+		*trunc_len = 0;
+	}
+}
+
+static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
+{
+	switch (*trunc_len) {
+	case 96:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96;
+		break;
+	case 80:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80;
+		break;
+	default:
+		*trunc_len = 0;
+	}
+}
+
+static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
+{
+	switch (*trunc_len) {
+	case 96:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96;
+		break;
+	case 128:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128;
+		break;
+	default:
+		*trunc_len = 0;
+	}
+}
+
+static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
+{
+	switch (*trunc_len) {
+	case 96:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96;
+		break;
+	case 192:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192;
+		break;
+	default:
+		*trunc_len = 0;
+	}
+}
+
+static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
+{
+	switch (*trunc_len) {
+	case 96:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96;
+		break;
+	case 256:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256;
+		break;
+	default:
+		*trunc_len = 0;
+	}
+}
+
+static int nfp_net_xfrm_add_state(struct xfrm_state *x)
+{
+	struct net_device *netdev = x->xso.dev;
+	struct nfp_ipsec_cfg_mssg msg = {};
+	int i, key_len, trunc_len, err = 0;
+	struct nfp_ipsec_cfg_add_sa *cfg;
+	struct nfp_net *nn;
+	unsigned int saidx;
+
+	nn = netdev_priv(netdev);
+	cfg = &msg.cfg_add_sa;
+
+	/* General */
+	switch (x->props.mode) {
+	case XFRM_MODE_TUNNEL:
+		cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL;
+		break;
+	case XFRM_MODE_TRANSPORT:
+		cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT;
+		break;
+	default:
+		nn_err(nn, "Unsupported mode for xfrm offload\n");
+		return -EINVAL;
+	}
+
+	switch (x->id.proto) {
+	case IPPROTO_ESP:
+		cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP;
+		break;
+	case IPPROTO_AH:
+		cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH;
+		break;
+	default:
+		nn_err(nn, "Unsupported protocol for xfrm offload\n");
+		return -EINVAL;
+	}
+
+	if (x->props.flags & XFRM_STATE_ESN) {
+		nn_err(nn, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload\n");
+		return -EINVAL;
+	}
+
+	cfg->spi = ntohl(x->id.spi);
+
+	/* Hash/Authentication */
+	if (x->aalg)
+		trunc_len = x->aalg->alg_trunc_len;
+	else
+		trunc_len = 0;
+
+	switch (x->props.aalgo) {
+	case SADB_AALG_NONE:
+		if (x->aead) {
+			trunc_len = -1;
+		} else {
+			nn_err(nn, "Unsupported authentication algorithm\n");
+			return -EINVAL;
+		}
+		break;
+	case SADB_X_AALG_NULL:
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE;
+		trunc_len = -1;
+		break;
+	case SADB_AALG_MD5HMAC:
+		set_md5hmac(cfg, &trunc_len);
+		break;
+	case SADB_AALG_SHA1HMAC:
+		set_sha1hmac(cfg, &trunc_len);
+		break;
+	case SADB_X_AALG_SHA2_256HMAC:
+		set_sha2_256hmac(cfg, &trunc_len);
+		break;
+	case SADB_X_AALG_SHA2_384HMAC:
+		set_sha2_384hmac(cfg, &trunc_len);
+		break;
+	case SADB_X_AALG_SHA2_512HMAC:
+		set_sha2_512hmac(cfg, &trunc_len);
+		break;
+	default:
+		nn_err(nn, "Unsupported authentication algorithm\n");
+		return -EINVAL;
+	}
+
+	if (!trunc_len) {
+		nn_err(nn, "Unsupported authentication algorithm trunc length\n");
+		return -EINVAL;
+	}
+
+	if (x->aalg) {
+		key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE);
+		if (key_len > sizeof(cfg->auth_key)) {
+			nn_err(nn, "Insufficient space for offloaded auth key\n");
+			return -EINVAL;
+		}
+		for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++)
+			cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key +
+							      sizeof(cfg->auth_key[0]) * i);
+	}
+
+	/* Encryption */
+	switch (x->props.ealgo) {
+	case SADB_EALG_NONE:
+	case SADB_EALG_NULL:
+		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
+		cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL;
+		break;
+	case SADB_EALG_3DESCBC:
+		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
+		cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES;
+		break;
+	case SADB_X_EALG_AES_GCM_ICV16:
+	case SADB_X_EALG_NULL_AES_GMAC:
+		if (!x->aead) {
+			nn_err(nn, "Invalid AES key data\n");
+			return -EINVAL;
+		}
+
+		if (x->aead->alg_icv_len != 128) {
+			nn_err(nn, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16\n");
+			return -EINVAL;
+		}
+		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR;
+		cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128;
+
+		/* Aead->alg_key_len includes 32-bit salt */
+		if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) {
+			nn_err(nn, "Unsupported AES key length %d\n", x->aead->alg_key_len);
+			return -EINVAL;
+		}
+		break;
+	case SADB_X_EALG_AESCBC:
+		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
+		if (!x->ealg) {
+			nn_err(nn, "Invalid AES key data\n");
+			return -EINVAL;
+		}
+		if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) {
+			nn_err(nn, "Unsupported AES key length %d\n", x->ealg->alg_key_len);
+			return -EINVAL;
+		}
+		break;
+	default:
+		nn_err(nn, "Unsupported encryption algorithm for offload\n");
+		return -EINVAL;
+	}
+
+	if (x->aead) {
+		int salt_len = 4;
+
+		key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE);
+		key_len -= salt_len;
+
+		if (key_len > sizeof(cfg->ciph_key)) {
+			nn_err(nn, "aead: Insufficient space for offloaded key\n");
+			return -EINVAL;
+		}
+
+		for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++)
+			cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key +
+							      sizeof(cfg->ciph_key[0]) * i);
+
+		/* Load up the salt */
+		cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len);
+	}
+
+	if (x->ealg) {
+		key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE);
+
+		if (key_len > sizeof(cfg->ciph_key)) {
+			nn_err(nn, "ealg: Insufficient space for offloaded key\n");
+			return -EINVAL;
+		}
+		for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++)
+			cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key +
+							      sizeof(cfg->ciph_key[0]) * i);
+	}
+
+	/* IP related info */
+	switch (x->props.family) {
+	case AF_INET:
+		cfg->ipv6 = 0;
+		cfg->src_ip[0] = ntohl(x->props.saddr.a4);
+		cfg->dst_ip[0] = ntohl(x->id.daddr.a4);
+		break;
+	case AF_INET6:
+		cfg->ipv6 = 1;
+		for (i = 0; i < 4; i++) {
+			cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]);
+			cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]);
+		}
+		break;
+	default:
+		nn_err(nn, "Unsupported address family\n");
+		return -EINVAL;
+	}
+
+	/* Maximum nic IPsec code could handle. Other limits may apply. */
+	cfg->pmtu_limit = 0xffff;
+	cfg->ctrl_word.encap_dsbl = 1;
+
+	/* SA direction */
+	cfg->ctrl_word.dir = x->xso.dir;
+
+	/* Find unused SA data*/
+	err = xa_alloc(&nn->xa_ipsec, &saidx, x,
+		       XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL);
+	if (err < 0) {
+		nn_err(nn, "Unable to get sa_data number for IPsec\n");
+		return err;
+	}
+
+	/* Allocate saidx and commit the SA */
+	err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_ADD_SA, saidx, &msg);
+	if (err) {
+		xa_erase(&nn->xa_ipsec, saidx);
+		nn_err(nn, "Failed to issue IPsec command err ret=%d\n", err);
+		return err;
+	}
+
+	/* 0 is invalid offload_handle for kernel */
+	x->xso.offload_handle = saidx + 1;
+	return 0;
+}
+
+static void nfp_net_xfrm_del_state(struct xfrm_state *x)
+{
+	struct net_device *netdev = x->xso.dev;
+	struct nfp_ipsec_cfg_mssg msg;
+	struct nfp_net *nn;
+	int err;
+
+	nn = netdev_priv(netdev);
+	err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_INV_SA,
+				      x->xso.offload_handle - 1, &msg);
+	if (err)
+		nn_warn(nn, "Failed to invalidate SA in hardware\n");
+
+	xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1);
+}
+
+static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
+{
+	if (x->props.family == AF_INET)
+		/* Offload with IPv4 options is not supported yet */
+		return ip_hdr(skb)->ihl == 5;
+
+	/* Offload with IPv6 extension headers is not support yet */
+	return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr));
+}
+
+static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = {
+	.xdo_dev_state_add = nfp_net_xfrm_add_state,
+	.xdo_dev_state_delete = nfp_net_xfrm_del_state,
+	.xdo_dev_offload_ok = nfp_net_ipsec_offload_ok,
+};
+
+void nfp_net_ipsec_init(struct nfp_net *nn)
+{
+	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
+		return;
+
+	xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC);
+	nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops;
+}
+
+void nfp_net_ipsec_clean(struct nfp_net *nn)
+{
+	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
+		return;
+
+	WARN_ON(!xa_empty(&nn->xa_ipsec));
+	xa_destroy(&nn->xa_ipsec);
+}
+
+bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,
+			   struct nfp_ipsec_offload *offload_info)
+{
+	struct xfrm_offload *xo = xfrm_offload(skb);
+	struct xfrm_state *x;
+
+	x = xfrm_input_state(skb);
+	if (!x)
+		return false;
+
+	offload_info->seq_hi = xo->seq.hi;
+	offload_info->seq_low = xo->seq.low;
+	offload_info->handle = x->xso.offload_handle;
+
+	return true;
+}
+
+int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb)
+{
+	struct net_device *netdev = skb->dev;
+	struct xfrm_offload *xo;
+	struct xfrm_state *x;
+	struct sec_path *sp;
+	struct nfp_net *nn;
+	u32 saidx;
+
+	nn = netdev_priv(netdev);
+
+	saidx = meta->ipsec_saidx - 1;
+	if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT)
+		return -EINVAL;
+
+	sp = secpath_set(skb);
+	if (unlikely(!sp))
+		return -ENOMEM;
+
+	xa_lock(&nn->xa_ipsec);
+	x = xa_load(&nn->xa_ipsec, saidx);
+	xa_unlock(&nn->xa_ipsec);
+	if (!x)
+		return -EINVAL;
+
+	xfrm_state_hold(x);
+	sp->xvec[sp->len++] = x;
+	sp->olen++;
+	xo = xfrm_offload(skb);
+	xo->flags = CRYPTO_DONE;
+	xo->status = CRYPTO_SUCCESS;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/dp.c b/drivers/net/ethernet/netronome/nfp/nfd3/dp.c
index 448c1c1afaee..861082c5dbff 100644
--- a/drivers/net/ethernet/netronome/nfp/nfd3/dp.c
+++ b/drivers/net/ethernet/netronome/nfp/nfd3/dp.c
@@ -4,6 +4,7 @@
 #include <linux/bpf_trace.h>
 #include <linux/netdevice.h>
 #include <linux/bitfield.h>
+#include <net/xfrm.h>
 
 #include "../nfp_app.h"
 #include "../nfp_net.h"
@@ -167,28 +168,34 @@ nfp_nfd3_tx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
 	u64_stats_update_end(&r_vec->tx_sync);
 }
 
-static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb, u64 tls_handle)
+static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb,
+				 u64 tls_handle, bool *ipsec)
 {
 	struct metadata_dst *md_dst = skb_metadata_dst(skb);
+	struct nfp_ipsec_offload offload_info;
 	unsigned char *data;
 	bool vlan_insert;
 	u32 meta_id = 0;
 	int md_bytes;
 
-	if (unlikely(md_dst || tls_handle)) {
-		if (unlikely(md_dst && md_dst->type != METADATA_HW_PORT_MUX))
-			md_dst = NULL;
-	}
+#ifdef CONFIG_NFP_NET_IPSEC
+	if (xfrm_offload(skb))
+		*ipsec = nfp_net_ipsec_tx_prep(dp, skb, &offload_info);
+#endif
+
+	if (unlikely(md_dst && md_dst->type != METADATA_HW_PORT_MUX))
+		md_dst = NULL;
 
 	vlan_insert = skb_vlan_tag_present(skb) && (dp->ctrl & NFP_NET_CFG_CTRL_TXVLAN_V2);
 
-	if (!(md_dst || tls_handle || vlan_insert))
+	if (!(md_dst || tls_handle || vlan_insert || *ipsec))
 		return 0;
 
 	md_bytes = sizeof(meta_id) +
 		   !!md_dst * NFP_NET_META_PORTID_SIZE +
 		   !!tls_handle * NFP_NET_META_CONN_HANDLE_SIZE +
-		   vlan_insert * NFP_NET_META_VLAN_SIZE;
+		   vlan_insert * NFP_NET_META_VLAN_SIZE +
+		   *ipsec * NFP_NET_META_IPSEC_FIELD_SIZE; /* IPsec has 12 bytes of metadata */
 
 	if (unlikely(skb_cow_head(skb, md_bytes)))
 		return -ENOMEM;
@@ -218,6 +225,19 @@ static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb, u64
 		meta_id <<= NFP_NET_META_FIELD_SIZE;
 		meta_id |= NFP_NET_META_VLAN;
 	}
+	if (*ipsec) {
+		/* IPsec has three consecutive 4-bit IPsec metadata types,
+		 * so in total IPsec has three 4 bytes of metadata.
+		 */
+		data -= NFP_NET_META_IPSEC_SIZE;
+		put_unaligned_be32(offload_info.seq_hi, data);
+		data -= NFP_NET_META_IPSEC_SIZE;
+		put_unaligned_be32(offload_info.seq_low, data);
+		data -= NFP_NET_META_IPSEC_SIZE;
+		put_unaligned_be32(offload_info.handle - 1, data);
+		meta_id <<= NFP_NET_META_IPSEC_FIELD_SIZE;
+		meta_id |= NFP_NET_META_IPSEC << 8 | NFP_NET_META_IPSEC << 4 | NFP_NET_META_IPSEC;
+	}
 
 	data -= sizeof(meta_id);
 	put_unaligned_be32(meta_id, data);
@@ -246,6 +266,7 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev)
 	dma_addr_t dma_addr;
 	unsigned int fsize;
 	u64 tls_handle = 0;
+	bool ipsec = false;
 	u16 qidx;
 
 	dp = &nn->dp;
@@ -273,7 +294,7 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev)
 		return NETDEV_TX_OK;
 	}
 
-	md_bytes = nfp_nfd3_prep_tx_meta(dp, skb, tls_handle);
+	md_bytes = nfp_nfd3_prep_tx_meta(dp, skb, tls_handle, &ipsec);
 	if (unlikely(md_bytes < 0))
 		goto err_flush;
 
@@ -312,6 +333,8 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev)
 		txd->vlan = cpu_to_le16(skb_vlan_tag_get(skb));
 	}
 
+	if (ipsec)
+		nfp_nfd3_ipsec_tx(txd, skb);
 	/* Gather DMA */
 	if (nr_frags > 0) {
 		__le64 second_half;
@@ -764,6 +787,15 @@ nfp_nfd3_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta,
 				return false;
 			data += sizeof(struct nfp_net_tls_resync_req);
 			break;
+#ifdef CONFIG_NFP_NET_IPSEC
+		case NFP_NET_META_IPSEC:
+			/* Note: IPsec packet will have zero saidx, so need add 1
+			 * to indicate packet is IPsec packet within driver.
+			 */
+			meta->ipsec_saidx = get_unaligned_be32(data) + 1;
+			data += 4;
+			break;
+#endif
 		default:
 			return true;
 		}
@@ -876,12 +908,11 @@ static int nfp_nfd3_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
 	struct nfp_net_tx_ring *tx_ring;
 	struct bpf_prog *xdp_prog;
+	int idx, pkts_polled = 0;
 	bool xdp_tx_cmpl = false;
 	unsigned int true_bufsz;
 	struct sk_buff *skb;
-	int pkts_polled = 0;
 	struct xdp_buff xdp;
-	int idx;
 
 	xdp_prog = READ_ONCE(dp->xdp_prog);
 	true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz;
@@ -1081,6 +1112,13 @@ static int nfp_nfd3_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 			continue;
 		}
 
+#ifdef CONFIG_NFP_NET_IPSEC
+		if (meta.ipsec_saidx != 0 && unlikely(nfp_net_ipsec_rx(&meta, skb))) {
+			nfp_nfd3_rx_drop(dp, r_vec, rx_ring, NULL, skb);
+			continue;
+		}
+#endif
+
 		if (meta_len_xdp)
 			skb_metadata_set(skb, meta_len_xdp);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/ipsec.c b/drivers/net/ethernet/netronome/nfp/nfd3/ipsec.c
new file mode 100644
index 000000000000..e90f8c975903
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfd3/ipsec.c
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright (C) 2018 Netronome Systems, Inc */
+/* Copyright (C) 2021 Corigine, Inc */
+
+#include <net/xfrm.h>
+
+#include "../nfp_net.h"
+#include "nfd3.h"
+
+void nfp_nfd3_ipsec_tx(struct nfp_nfd3_tx_desc *txd, struct sk_buff *skb)
+{
+	struct xfrm_state *x = xfrm_input_state(skb);
+
+	if (x->xso.dev && (x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)) {
+		txd->flags |= NFD3_DESC_TX_CSUM | NFD3_DESC_TX_IP4_CSUM |
+			      NFD3_DESC_TX_TCP_CSUM | NFD3_DESC_TX_UDP_CSUM;
+	}
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/nfd3.h b/drivers/net/ethernet/netronome/nfp/nfd3/nfd3.h
index 7a0df9e6c3c4..9c1c10dcbaee 100644
--- a/drivers/net/ethernet/netronome/nfp/nfd3/nfd3.h
+++ b/drivers/net/ethernet/netronome/nfp/nfd3/nfd3.h
@@ -103,4 +103,12 @@ void nfp_nfd3_rx_ring_fill_freelist(struct nfp_net_dp *dp,
 void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf);
 int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget);
 
+#ifndef CONFIG_NFP_NET_IPSEC
+static inline void nfp_nfd3_ipsec_tx(struct nfp_nfd3_tx_desc *txd, struct sk_buff *skb)
+{
+}
+#else
+void nfp_nfd3_ipsec_tx(struct nfp_nfd3_tx_desc *txd, struct sk_buff *skb);
+#endif
+
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index a101ff30a1ae..6c83e47d8b3d 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -263,6 +263,10 @@ struct nfp_meta_parsed {
 		u8 tpid;
 		u16 tci;
 	} vlan;
+
+#ifdef CONFIG_NFP_NET_IPSEC
+	u32 ipsec_saidx;
+#endif
 };
 
 struct nfp_net_rx_hash {
@@ -541,6 +545,7 @@ struct nfp_net_dp {
  * @id:			vNIC id within the PF (0 for VFs)
  * @fw_ver:		Firmware version
  * @cap:                Capabilities advertised by the Firmware
+ * @cap_w1:             Extended capabilities word advertised by the Firmware
  * @max_mtu:            Maximum support MTU advertised by the Firmware
  * @rss_hfunc:		RSS selected hash function
  * @rss_cfg:            RSS configuration
@@ -583,6 +588,7 @@ struct nfp_net_dp {
  * @qcp_cfg:            Pointer to QCP queue used for configuration notification
  * @tx_bar:             Pointer to mapped TX queues
  * @rx_bar:             Pointer to mapped FL/RX queues
+ * @xa_ipsec:           IPsec xarray SA data
  * @tlv_caps:		Parsed TLV capabilities
  * @ktls_tx_conn_cnt:	Number of offloaded kTLS TX connections
  * @ktls_rx_conn_cnt:	Number of offloaded kTLS RX connections
@@ -617,6 +623,7 @@ struct nfp_net {
 	u32 id;
 
 	u32 cap;
+	u32 cap_w1;
 	u32 max_mtu;
 
 	u8 rss_hfunc;
@@ -670,6 +677,10 @@ struct nfp_net {
 	u8 __iomem *tx_bar;
 	u8 __iomem *rx_bar;
 
+#ifdef CONFIG_NFP_NET_IPSEC
+	struct xarray xa_ipsec;
+#endif
+
 	struct nfp_net_tlv_caps tlv_caps;
 
 	unsigned int ktls_tx_conn_cnt;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 5620faa63c7e..682a9198fb54 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -2371,6 +2371,12 @@ static void nfp_net_netdev_init(struct nfp_net *nn)
 	}
 	if (nn->cap & NFP_NET_CFG_CTRL_RSS_ANY)
 		netdev->hw_features |= NETIF_F_RXHASH;
+
+#ifdef CONFIG_NFP_NET_IPSEC
+	if (nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)
+		netdev->hw_features |= NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM;
+#endif
+
 	if (nn->cap & NFP_NET_CFG_CTRL_VXLAN) {
 		if (nn->cap & NFP_NET_CFG_CTRL_LSO) {
 			netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
@@ -2452,6 +2458,7 @@ static int nfp_net_read_caps(struct nfp_net *nn)
 {
 	/* Get some of the read-only fields from the BAR */
 	nn->cap = nn_readl(nn, NFP_NET_CFG_CAP);
+	nn->cap_w1 = nn_readq(nn, NFP_NET_CFG_CAP_WORD1);
 	nn->max_mtu = nn_readl(nn, NFP_NET_CFG_MAX_MTU);
 
 	/* ABI 4.x and ctrl vNIC always use chained metadata, in other cases
@@ -2563,6 +2570,8 @@ int nfp_net_init(struct nfp_net *nn)
 		err = nfp_net_tls_init(nn);
 		if (err)
 			goto err_clean_mbox;
+
+		nfp_net_ipsec_init(nn);
 	}
 
 	nfp_net_vecs_init(nn);
@@ -2586,6 +2595,7 @@ void nfp_net_clean(struct nfp_net *nn)
 		return;
 
 	unregister_netdev(nn->dp.netdev);
+	nfp_net_ipsec_clean(nn);
 	nfp_ccm_mbox_clean(nn);
 	nfp_net_reconfig_wait_posted(nn);
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
index 6714d5e8fdab..cc11b3dc1252 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
@@ -48,6 +48,7 @@
 #define NFP_NET_META_CSUM		6 /* checksum complete type */
 #define NFP_NET_META_CONN_HANDLE	7
 #define NFP_NET_META_RESYNC_INFO	8 /* RX resync info request */
+#define NFP_NET_META_IPSEC		9 /* IPsec SA index for tx and rx */
 
 #define NFP_META_PORT_ID_CTRL		~0U
 
@@ -55,6 +56,8 @@
 #define NFP_NET_META_VLAN_SIZE			4
 #define NFP_NET_META_PORTID_SIZE		4
 #define NFP_NET_META_CONN_HANDLE_SIZE		8
+#define NFP_NET_META_IPSEC_SIZE			4
+#define NFP_NET_META_IPSEC_FIELD_SIZE		12
 /* Hash type pre-pended when a RSS hash was computed */
 #define NFP_NET_RSS_NONE		0
 #define NFP_NET_RSS_IPV4		1
@@ -257,10 +260,19 @@
 #define   NFP_NET_CFG_BPF_CFG_MASK	7ULL
 #define   NFP_NET_CFG_BPF_ADDR_MASK	(~NFP_NET_CFG_BPF_CFG_MASK)
 
-/* 40B reserved for future use (0x0098 - 0x00c0)
+/* 3 words reserved for extended ctrl words (0x0098 - 0x00a4)
+ * 3 words reserved for extended cap words (0x00a4 - 0x00b0)
+ * Currently only one word is used, can be extended in future.
  */
-#define NFP_NET_CFG_RESERVED		0x0098
-#define NFP_NET_CFG_RESERVED_SZ		0x0028
+#define NFP_NET_CFG_CTRL_WORD1		0x0098
+#define   NFP_NET_CFG_CTRL_PKT_TYPE	  (0x1 << 0) /* Pkttype offload */
+#define   NFP_NET_CFG_CTRL_IPSEC	  (0x1 << 1) /* IPsec offload */
+
+#define NFP_NET_CFG_CAP_WORD1		0x00a4
+
+/* 16B reserved for future use (0x00b0 - 0x00c0) */
+#define NFP_NET_CFG_RESERVED		0x00b0
+#define NFP_NET_CFG_RESERVED_SZ		0x0010
 
 /* RSS configuration (0x0100 - 0x01ac):
  * Used only when NFP_NET_CFG_CTRL_RSS is enabled
@@ -390,14 +402,14 @@
  */
 #define NFP_NET_CFG_MBOX_BASE		0x1800
 #define NFP_NET_CFG_MBOX_VAL_MAX_SZ	0x1F8
-
+#define NFP_NET_CFG_MBOX_VAL		0x1808
 #define NFP_NET_CFG_MBOX_SIMPLE_CMD	0x0
 #define NFP_NET_CFG_MBOX_SIMPLE_RET	0x4
 #define NFP_NET_CFG_MBOX_SIMPLE_VAL	0x8
 
 #define NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_ADD 1
 #define NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_KILL 2
-
+#define NFP_NET_CFG_MBOX_CMD_IPSEC 3
 #define NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET	5
 #define NFP_NET_CFG_MBOX_CMD_TLV_CMSG			6