diff mbox

[net-next,11/13] RDS: IB: add Fastreg MR (FRMR) detection support

Message ID 1455967804-27644-12-git-send-email-santosh.shilimkar@oracle.com
State Changes Requested, archived
Delegated to: David Miller
Headers show

Commit Message

Santosh Shilimkar Feb. 20, 2016, 11:30 a.m. UTC 
Discovere Fast Memmory Registration support using IB device
IB_DEVICE_MEM_MGT_EXTENSIONS. Certain HCA might support just FRMR
or FMR or both FMR and FRWR. In case both mr type are supported,
default FMR is used. Using module parameter 'prefer_frmr',
user can choose its preferred MR method for RDS. Ofcourse the
module parameter has no effect if the HCA support only FRMR
or only FRMR.

Default MR is still kept as FMR against what everyone else
is following. Default will be changed to FRMR once the
RDS performance with FRMR is comparable with FMR. The
work is in progress for the same.

Signed-off-by: Santosh Shilimkar <ssantosh@kernel.org>
Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>
---
 net/rds/ib.c    | 14 ++++++++++++++
 net/rds/ib.h    |  4 ++++
 net/rds/ib_mr.h |  1 +
 3 files changed, 19 insertions(+)

Comments

David Miller Feb. 22, 2016, 3:36 a.m. UTC | #1 
From: Santosh Shilimkar <santosh.shilimkar@oracle.com>
Date: Sat, 20 Feb 2016 03:30:02 -0800

> @@ -54,6 +55,8 @@ module_param(rds_ib_mr_8k_pool_size, int, 0444);
>  MODULE_PARM_DESC(rds_ib_mr_8k_pool_size, " Max number of 8K mr per HCA");
>  module_param(rds_ib_retry_count, int, 0444);
>  MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error");
> +module_param(prefer_frmr, bool, 0444);
> +MODULE_PARM_DESC(prefer_frmr, "Preferred MR method if both FMR and FRMR supported");

Sorry, you're going to have to create a real run time method to configure
this parameter.

I'm strongly against module parameters.

Please don't go into details about why this might be difficult to do,
I'm totally not interested.  Doing things properly is sometimes not
easy, that's life.
Bart Van Assche Feb. 22, 2016, 3:38 p.m. UTC | #2 
On 02/21/16 19:36, David Miller wrote:
> From: Santosh Shilimkar <santosh.shilimkar@oracle.com>
> Date: Sat, 20 Feb 2016 03:30:02 -0800
>
>> @@ -54,6 +55,8 @@ module_param(rds_ib_mr_8k_pool_size, int, 0444);
>>   MODULE_PARM_DESC(rds_ib_mr_8k_pool_size, " Max number of 8K mr per HCA");
>>   module_param(rds_ib_retry_count, int, 0444);
>>   MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error");
>> +module_param(prefer_frmr, bool, 0444);
>> +MODULE_PARM_DESC(prefer_frmr, "Preferred MR method if both FMR and FRMR supported");
>
> Sorry, you're going to have to create a real run time method to configure
> this parameter.
>
> I'm strongly against module parameters.
>
> Please don't go into details about why this might be difficult to do,
> I'm totally not interested.  Doing things properly is sometimes not
> easy, that's life.

Hello Santosh,

What is the purpose of the prefer_frmr kernel module parameter ? Is this 
a parameter that is useful to RDS users or is its only purpose to allow 
developers of the RDS module to test both the FMR and FRMR code paths on 
hardware that supports both MR methods ?

Bart.
Santosh Shilimkar Feb. 22, 2016, 4:38 p.m. UTC | #3 
On 2/22/2016 7:38 AM, Bart Van Assche wrote:
> On 02/21/16 19:36, David Miller wrote:
>> From: Santosh Shilimkar <santosh.shilimkar@oracle.com>
>> Date: Sat, 20 Feb 2016 03:30:02 -0800
>>
>>> @@ -54,6 +55,8 @@ module_param(rds_ib_mr_8k_pool_size, int, 0444);
>>>   MODULE_PARM_DESC(rds_ib_mr_8k_pool_size, " Max number of 8K mr per
>>> HCA");
>>>   module_param(rds_ib_retry_count, int, 0444);
>>>   MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before
>>> reporting an error");
>>> +module_param(prefer_frmr, bool, 0444);
>>> +MODULE_PARM_DESC(prefer_frmr, "Preferred MR method if both FMR and
>>> FRMR supported");
>>
>> Sorry, you're going to have to create a real run time method to configure
>> this parameter.
>>
>> I'm strongly against module parameters.
>>
>> Please don't go into details about why this might be difficult to do,
>> I'm totally not interested.  Doing things properly is sometimes not
>> easy, that's life.
>
Sure Dave. Will drop the parameter. The runtime detection is already
in place. When an HCA hardware supports both FMR and FRMR features,
parameter can be used as over-ride over a default selection.

> Hello Santosh,
>
> What is the purpose of the prefer_frmr kernel module parameter ? Is this
> a parameter that is useful to RDS users or is its only purpose to allow
> developers of the RDS module to test both the FMR and FRMR code paths on
> hardware that supports both MR methods ?
>
Right. Since FRMR in early phase still for RDS, it was useful on HCA's
which supports both registration methods. Its not a deal breaker so
am going to drop the parameter as mentioned above.

Regards,
Santosh
diff mbox

Patch

diff --git a/net/rds/ib.c b/net/rds/ib.c
index bb32cb9..68c94b0 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -47,6 +47,7 @@ 
 unsigned int rds_ib_mr_1m_pool_size = RDS_MR_1M_POOL_SIZE;
 unsigned int rds_ib_mr_8k_pool_size = RDS_MR_8K_POOL_SIZE;
 unsigned int rds_ib_retry_count = RDS_IB_DEFAULT_RETRY_COUNT;
+bool prefer_frmr;
 
 module_param(rds_ib_mr_1m_pool_size, int, 0444);
 MODULE_PARM_DESC(rds_ib_mr_1m_pool_size, " Max number of 1M mr per HCA");
@@ -54,6 +55,8 @@  module_param(rds_ib_mr_8k_pool_size, int, 0444);
 MODULE_PARM_DESC(rds_ib_mr_8k_pool_size, " Max number of 8K mr per HCA");
 module_param(rds_ib_retry_count, int, 0444);
 MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error");
+module_param(prefer_frmr, bool, 0444);
+MODULE_PARM_DESC(prefer_frmr, "Preferred MR method if both FMR and FRMR supported");
 
 /*
  * we have a clumsy combination of RCU and a rwsem protecting this list
@@ -140,6 +143,13 @@  static void rds_ib_add_one(struct ib_device *device)
 	rds_ibdev->max_wrs = device->attrs.max_qp_wr;
 	rds_ibdev->max_sge = min(device->attrs.max_sge, RDS_IB_MAX_SGE);
 
+	rds_ibdev->has_fr = (device->attrs.device_cap_flags &
+				  IB_DEVICE_MEM_MGT_EXTENSIONS);
+	rds_ibdev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
+			    device->map_phys_fmr && device->unmap_fmr);
+	rds_ibdev->use_fastreg = (rds_ibdev->has_fr &&
+				 (!rds_ibdev->has_fmr || prefer_frmr));
+
 	rds_ibdev->fmr_max_remaps = device->attrs.max_map_per_fmr?: 32;
 	rds_ibdev->max_1m_mrs = device->attrs.max_mr ?
 		min_t(unsigned int, (device->attrs.max_mr / 2),
@@ -178,6 +188,10 @@  static void rds_ib_add_one(struct ib_device *device)
 		 rds_ibdev->fmr_max_remaps, rds_ibdev->max_1m_mrs,
 		 rds_ibdev->max_8k_mrs);
 
+	pr_info("RDS/IB: %s: %s supported and preferred\n",
+		device->name,
+		rds_ibdev->use_fastreg ? "FRMR" : "FMR");
+
 	INIT_LIST_HEAD(&rds_ibdev->ipaddr_list);
 	INIT_LIST_HEAD(&rds_ibdev->conn_list);
 
diff --git a/net/rds/ib.h b/net/rds/ib.h
index 62fe7d5..c5eddc2 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -200,6 +200,10 @@  struct rds_ib_device {
 	struct list_head	conn_list;
 	struct ib_device	*dev;
 	struct ib_pd		*pd;
+	bool                    has_fmr;
+	bool                    has_fr;
+	bool                    use_fastreg;
+
 	unsigned int		max_mrs;
 	struct rds_ib_mr_pool	*mr_1m_pool;
 	struct rds_ib_mr_pool   *mr_8k_pool;
diff --git a/net/rds/ib_mr.h b/net/rds/ib_mr.h
index add7725..2f9b9c3 100644
--- a/net/rds/ib_mr.h
+++ b/net/rds/ib_mr.h
@@ -93,6 +93,7 @@  struct rds_ib_mr_pool {
 extern struct workqueue_struct *rds_ib_mr_wq;
 extern unsigned int rds_ib_mr_1m_pool_size;
 extern unsigned int rds_ib_mr_8k_pool_size;
+extern bool prefer_frmr;
 
 struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_dev,
 					     int npages);