1 /*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 * $Id: fmr_pool.c 2730 2005-06-28 16:43:03Z sean.hefty $
34 */
35
36 #include <linux/errno.h>
37 #include <linux/spinlock.h>
38 #include <linux/slab.h>
39 #include <linux/jhash.h>
40 #include <linux/kthread.h>
41
42 #include <rdma/ib_fmr_pool.h>
43
44 #include "core_priv.h"
45
46 #define PFX "fmr_pool: "
47
48 enum {
49 IB_FMR_MAX_REMAPS = 32,
50
51 IB_FMR_HASH_BITS = 8,
52 IB_FMR_HASH_SIZE = 1 << IB_FMR_HASH_BITS,
53 IB_FMR_HASH_MASK = IB_FMR_HASH_SIZE - 1
54 };
55
56 /*
57 * If an FMR is not in use, then the list member will point to either
58 * its pool's free_list (if the FMR can be mapped again; that is,
59 * remap_count < pool->max_remaps) or its pool's dirty_list (if the
60 * FMR needs to be unmapped before being remapped). In either of
61 * these cases it is a bug if the ref_count is not 0. In other words,
62 * if ref_count is > 0, then the list member must not be linked into
63 * either free_list or dirty_list.
64 *
65 * The cache_node member is used to link the FMR into a cache bucket
66 * (if caching is enabled). This is independent of the reference
67 * count of the FMR. When a valid FMR is released, its ref_count is
68 * decremented, and if ref_count reaches 0, the FMR is placed in
69 * either free_list or dirty_list as appropriate. However, it is not
70 * removed from the cache and may be "revived" if a call to
71 * ib_fmr_register_physical() occurs before the FMR is remapped. In
72 * this case we just increment the ref_count and remove the FMR from
73 * free_list/dirty_list.
74 *
75 * Before we remap an FMR from free_list, we remove it from the cache
76 * (to prevent another user from obtaining a stale FMR). When an FMR
77 * is released, we add it to the tail of the free list, so that our
78 * cache eviction policy is "least recently used."
79 *
80 * All manipulation of ref_count, list and cache_node is protected by
81 * pool_lock to maintain consistency.
82 */
83
84 struct ib_fmr_pool {
85 spinlock_t pool_lock;
86
87 int pool_size;
88 int max_pages;
89 int max_remaps;
90 int dirty_watermark;
91 int dirty_len;
92 struct list_head free_list;
93 struct list_head dirty_list;
94 struct hlist_head *cache_bucket;
95
96 void (*flush_function)(struct ib_fmr_pool *pool,
97 void * arg);
98 void *flush_arg;
99
100 struct task_struct *thread;
101
102 atomic_t req_ser;
103 atomic_t flush_ser;
104
105 wait_queue_head_t force_wait;
106 };
107
108 static inline u32 ib_fmr_hash(u64 first_page)
109 {
110 return jhash_2words((u32) first_page, (u32) (first_page >> 32), 0) &
111 (IB_FMR_HASH_SIZE - 1);
112 }
113
114 /* Caller must hold pool_lock */
115 static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,
116 u64 *page_list,
117 int page_list_len,
118 u64 io_virtual_address)
119 {
120 struct hlist_head *bucket;
121 struct ib_pool_fmr *fmr;
122 struct hlist_node *pos;
123
124 if (!pool->cache_bucket)
125 return NULL;
126
127 bucket = pool->cache_bucket + ib_fmr_hash(*page_list);
128
129 hlist_for_each_entry(fmr, pos, bucket, cache_node)
130 if (io_virtual_address == fmr->io_virtual_address &&
131 page_list_len == fmr->page_list_len &&
132 !memcmp(page_list, fmr->page_list,
133 page_list_len * sizeof *page_list))
134 return fmr;
135
136 return NULL;
137 }
138
139 static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
140 {
141 int ret;
142 struct ib_pool_fmr *fmr;
143 LIST_HEAD(unmap_list);
144 LIST_HEAD(fmr_list);
145
146 spin_lock_irq(&pool->pool_lock);
147
148 list_for_each_entry(fmr, &pool->dirty_list, list) {
149 hlist_del_init(&fmr->cache_node);
150 fmr->remap_count = 0;
151 list_add_tail(&fmr->fmr->list, &fmr_list);
152
153 #ifdef DEBUG
154 if (fmr->ref_count !=0) {
155 printk(KERN_WARNING PFX "Unmapping FMR 0x%08x with ref count %d\n",
156 fmr, fmr->ref_count);
157 }
158 #endif
159 }
160
161 list_splice(&pool->dirty_list, &unmap_list);
162 INIT_LIST_HEAD(&pool->dirty_list);
163 pool->dirty_len = 0;
164
165 spin_unlock_irq(&pool->pool_lock);
166
167 if (list_empty(&unmap_list)) {
168 return;
169 }
170
171 ret = ib_unmap_fmr(&fmr_list);
172 if (ret)
173 printk(KERN_WARNING PFX "ib_unmap_fmr returned %d\n", ret);
174
175 spin_lock_irq(&pool->pool_lock);
176 list_splice(&unmap_list, &pool->free_list);
177 spin_unlock_irq(&pool->pool_lock);
178 }
179
180 static int ib_fmr_cleanup_thread(void *pool_ptr)
181 {
182 struct ib_fmr_pool *pool = pool_ptr;
183
184 do {
185 if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
186 ib_fmr_batch_release(pool);
187
188 atomic_inc(&pool->flush_ser);
189 wake_up_interruptible(&pool->force_wait);
190
191 if (pool->flush_function)
192 pool->flush_function(pool, pool->flush_arg);
193 }
194
195 set_current_state(TASK_INTERRUPTIBLE);
196 if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
197 !kthread_should_stop())
198 schedule();
199 __set_current_state(TASK_RUNNING);
200 } while (!kthread_should_stop());
201
202 return 0;
203 }
204
205 /**
206 * ib_create_fmr_pool - Create an FMR pool
207 * @pd:Protection domain for FMRs
208 * @params:FMR pool parameters
209 *
210 * Create a pool of FMRs. Return value is pointer to new pool or
211 * error code if creation failed.
212 */
213 struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
214 struct ib_fmr_pool_param *params)
215 {
216 struct ib_device *device;
217 struct ib_fmr_pool *pool;
218 struct ib_device_attr *attr;
219 int i;
220 int ret;
221 int max_remaps;
222
223 if (!params)
224 return ERR_PTR(-EINVAL);
225
226 device = pd->device;
227 if (!device->alloc_fmr || !device->dealloc_fmr ||
228 !device->map_phys_fmr || !device->unmap_fmr) {
229 printk(KERN_INFO PFX "Device %s does not support FMRs\n",
230 device->name);
231 return ERR_PTR(-ENOSYS);
232 }
233
234 attr = kmalloc(sizeof *attr, GFP_KERNEL);
235 if (!attr) {
236 printk(KERN_WARNING PFX "couldn't allocate device attr struct\n");
237 return ERR_PTR(-ENOMEM);
238 }
239
240 ret = ib_query_device(device, attr);
241 if (ret) {
242 printk(KERN_WARNING PFX "couldn't query device: %d\n", ret);
243 kfree(attr);
244 return ERR_PTR(ret);
245 }
246
247 if (!attr->max_map_per_fmr)
248 max_remaps = IB_FMR_MAX_REMAPS;
249 else
250 max_remaps = attr->max_map_per_fmr;
251
252 kfree(attr);
253
254 pool = kmalloc(sizeof *pool, GFP_KERNEL);
255 if (!pool) {
256 printk(KERN_WARNING PFX "couldn't allocate pool struct\n");
257 return ERR_PTR(-ENOMEM);
258 }
259
260 pool->cache_bucket = NULL;
261
262 pool->flush_function = params->flush_function;
263 pool->flush_arg = params->flush_arg;
264
265 INIT_LIST_HEAD(&pool->free_list);
266 INIT_LIST_HEAD(&pool->dirty_list);
267
268 if (params->cache) {
269 pool->cache_bucket =
270 kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,
271 GFP_KERNEL);
272 if (!pool->cache_bucket) {
273 printk(KERN_WARNING PFX "Failed to allocate cache in pool\n");
274 ret = -ENOMEM;
275 goto out_free_pool;
276 }
277
278 for (i = 0; i < IB_FMR_HASH_SIZE; ++i)
279 INIT_HLIST_HEAD(pool->cache_bucket + i);
280 }
281
282 pool->pool_size = 0;
283 pool->max_pages = params->max_pages_per_fmr;
284 pool->max_remaps = max_remaps;
285 pool->dirty_watermark = params->dirty_watermark;
286 pool->dirty_len = 0;
287 spin_lock_init(&pool->pool_lock);
288 atomic_set(&pool->req_ser, 0);
289 atomic_set(&pool->flush_ser, 0);
290 init_waitqueue_head(&pool->force_wait);
291
292 pool->thread = kthread_run(ib_fmr_cleanup_thread,
293 pool,
294 "ib_fmr(%s)",
295 device->name);
296 if (IS_ERR(pool->thread)) {
297 printk(KERN_WARNING PFX "couldn't start cleanup thread\n");
298 ret = PTR_ERR(pool->thread);
299 goto out_free_pool;
300 }
301
302 {
303 struct ib_pool_fmr *fmr;
304 struct ib_fmr_attr fmr_attr = {
305 .max_pages = params->max_pages_per_fmr,
306 .max_maps = pool->max_remaps,
307 .page_shift = params->page_shift
308 };
309 int bytes_per_fmr = sizeof *fmr;
310
311 if (pool->cache_bucket)
312 bytes_per_fmr += params->max_pages_per_fmr * sizeof (u64);
313
314 for (i = 0; i < params->pool_size; ++i) {
315 fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
316 if (!fmr) {
317 printk(KERN_WARNING PFX "failed to allocate fmr "
318 "struct for FMR %d\n", i);
319 goto out_fail;
320 }
321
322 fmr->pool = pool;
323 fmr->remap_count = 0;
324 fmr->ref_count = 0;
325 INIT_HLIST_NODE(&fmr->cache_node);
326
327 fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
328 if (IS_ERR(fmr->fmr)) {
329 printk(KERN_WARNING PFX "fmr_create failed "
330 "for FMR %d\n", i);
331 kfree(fmr);
332 goto out_fail;
333 }
334
335 list_add_tail(&fmr->list, &pool->free_list);
336 ++pool->pool_size;
337 }
338 }
339
340 return pool;
341
342 out_free_pool:
343 kfree(pool->cache_bucket);
344 kfree(pool);
345
346 return ERR_PTR(ret);
347
348 out_fail:
349 ib_destroy_fmr_pool(pool);
350
351 return ERR_PTR(-ENOMEM);
352 }
353 EXPORT_SYMBOL(ib_create_fmr_pool);
354
355 /**
356 * ib_destroy_fmr_pool - Free FMR pool
357 * @pool:FMR pool to free
358 *
359 * Destroy an FMR pool and free all associated resources.
360 */
361 void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)
362 {
363 struct ib_pool_fmr *fmr;
364 struct ib_pool_fmr *tmp;
365 LIST_HEAD(fmr_list);
366 int i;
367
368 kthread_stop(pool->thread);
369 ib_fmr_batch_release(pool);
370
371 i = 0;
372 list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
373 if (fmr->remap_count) {
374 INIT_LIST_HEAD(&fmr_list);
375 list_add_tail(&fmr->fmr->list, &fmr_list);
376 ib_unmap_fmr(&fmr_list);
377 }
378 ib_dealloc_fmr(fmr->fmr);
379 list_del(&fmr->list);
380 kfree(fmr);
381 ++i;
382 }
383
384 if (i < pool->pool_size)
385 printk(KERN_WARNING PFX "pool still has %d regions registered\n",
386 pool->pool_size - i);
387
388 kfree(pool->cache_bucket);
389 kfree(pool);
390 }
391 EXPORT_SYMBOL(ib_destroy_fmr_pool);
392
393 /**
394 * ib_flush_fmr_pool - Invalidate all unmapped FMRs
395 * @pool:FMR pool to flush
396 *
397 * Ensure that all unmapped FMRs are fully invalidated.
398 */
399 int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
400 {
401 int serial;
402 struct ib_pool_fmr *fmr, *next;
403
404 /*
405 * The free_list holds FMRs that may have been used
406 * but have not been remapped enough times to be dirty.
407 * Put them on the dirty list now so that the cleanup
408 * thread will reap them too.
409 */
410 spin_lock_irq(&pool->pool_lock);
411 list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
412 if (fmr->remap_count > 0)
413 list_move(&fmr->list, &pool->dirty_list);
414 }
415 spin_unlock_irq(&pool->pool_lock);
416
417 serial = atomic_inc_return(&pool->req_ser);
418 wake_up_process(pool->thread);
419
420 if (wait_event_interruptible(pool->force_wait,
421 atomic_read(&pool->flush_ser) - serial >= 0))
422 return -EINTR;
423
424 return 0;
425 }
426 EXPORT_SYMBOL(ib_flush_fmr_pool);
427
428 /**
429 * ib_fmr_pool_map_phys -
430 * @pool:FMR pool to allocate FMR from
431 * @page_list:List of pages to map
432 * @list_len:Number of pages in @page_list
433 * @io_virtual_address:I/O virtual address for new FMR
434 *
435 * Map an FMR from an FMR pool.
436 */
437 struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
438 u64 *page_list,
439 int list_len,
440 u64 io_virtual_address)
441 {
442 struct ib_fmr_pool *pool = pool_handle;
443 struct ib_pool_fmr *fmr;
444 unsigned long flags;
445 int result;
446
447 if (list_len < 1 || list_len > pool->max_pages)
448 return ERR_PTR(-EINVAL);
449
450 spin_lock_irqsave(&pool->pool_lock, flags);
451 fmr = ib_fmr_cache_lookup(pool,
452 page_list,
453 list_len,
454 io_virtual_address);
455 if (fmr) {
456 /* found in cache */
457 ++fmr->ref_count;
458 if (fmr->ref_count == 1) {
459 list_del(&fmr->list);
460 }
461
462 spin_unlock_irqrestore(&pool->pool_lock, flags);
463
464 return fmr;
465 }
466
467 if (list_empty(&pool->free_list)) {
468 spin_unlock_irqrestore(&pool->pool_lock, flags);
469 return ERR_PTR(-EAGAIN);
470 }
471
472 fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);
473 list_del(&fmr->list);
474 hlist_del_init(&fmr->cache_node);
475 spin_unlock_irqrestore(&pool->pool_lock, flags);
476
477 result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
478 io_virtual_address);
479
480 if (result) {
481 spin_lock_irqsave(&pool->pool_lock, flags);
482 list_add(&fmr->list, &pool->free_list);
483 spin_unlock_irqrestore(&pool->pool_lock, flags);
484
485 printk(KERN_WARNING PFX "fmr_map returns %d\n", result);
486
487 return ERR_PTR(result);
488 }
489
490 ++fmr->remap_count;
491 fmr->ref_count = 1;
492
493 if (pool->cache_bucket) {
494 fmr->io_virtual_address = io_virtual_address;
495 fmr->page_list_len = list_len;
496 memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));
497
498 spin_lock_irqsave(&pool->pool_lock, flags);
499 hlist_add_head(&fmr->cache_node,
500 pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));
501 spin_unlock_irqrestore(&pool->pool_lock, flags);
502 }
503
504 return fmr;
505 }
506 EXPORT_SYMBOL(ib_fmr_pool_map_phys);
507
508 /**
509 * ib_fmr_pool_unmap - Unmap FMR
510 * @fmr:FMR to unmap
511 *
512 * Unmap an FMR. The FMR mapping may remain valid until the FMR is
513 * reused (or until ib_flush_fmr_pool() is called).
514 */
515 int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
516 {
517 struct ib_fmr_pool *pool;
518 unsigned long flags;
519
520 pool = fmr->pool;
521
522 spin_lock_irqsave(&pool->pool_lock, flags);
523
524 --fmr->ref_count;
525 if (!fmr->ref_count) {
526 if (fmr->remap_count < pool->max_remaps) {
527 list_add_tail(&fmr->list, &pool->free_list);
528 } else {
529 list_add_tail(&fmr->list, &pool->dirty_list);
530 if (++pool->dirty_len >= pool->dirty_watermark) {
531 atomic_inc(&pool->req_ser);
532 wake_up_process(pool->thread);
533 }
534 }
535 }
536
537 #ifdef DEBUG
538 if (fmr->ref_count < 0)
539 printk(KERN_WARNING PFX "FMR %p has ref count %d < 0\n",
540 fmr, fmr->ref_count);
541 #endif
542
543 spin_unlock_irqrestore(&pool->pool_lock, flags);
544
545 return 0;
546 }
547 EXPORT_SYMBOL(ib_fmr_pool_unmap);
548
|
This page was automatically generated by the
LXR engine.
|