| Linux kernel & device driver programming |
| [ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] |
1 /* 1 /*
2 * Copyright (c) 2006, 2007 Cisco Systems, Inc 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technolog <<
4 * 3 *
5 * This software is available to you under a c 4 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed un 5 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, ava 6 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this sourc 7 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below: 8 * OpenIB.org BSD license below:
10 * 9 *
11 * Redistribution and use in source and bi 10 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted pro 11 * without modification, are permitted provided that the following
13 * conditions are met: 12 * conditions are met:
14 * 13 *
15 * - Redistributions of source code must 14 * - Redistributions of source code must retain the above
16 * copyright notice, this list of condi 15 * copyright notice, this list of conditions and the following
17 * disclaimer. 16 * disclaimer.
18 * 17 *
19 * - Redistributions in binary form must 18 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of condi 19 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/ 20 * disclaimer in the documentation and/or other materials
22 * provided with the distribution. 21 * provided with the distribution.
23 * 22 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT W 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMIT 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR P 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTH 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER L 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARIS 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE. 30 * SOFTWARE.
32 */ 31 */
33 32
34 #include <linux/errno.h> 33 #include <linux/errno.h>
35 #include <linux/slab.h> 34 #include <linux/slab.h>
36 #include <linux/mm.h> <<
37 #include <linux/bitmap.h> 35 #include <linux/bitmap.h>
38 #include <linux/dma-mapping.h> 36 #include <linux/dma-mapping.h>
39 #include <linux/vmalloc.h> 37 #include <linux/vmalloc.h>
40 38
41 #include "mlx4.h" 39 #include "mlx4.h"
42 40
43 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitm 41 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
44 { 42 {
45 u32 obj; 43 u32 obj;
46 44
47 spin_lock(&bitmap->lock); 45 spin_lock(&bitmap->lock);
48 46
49 obj = find_next_zero_bit(bitmap->table 47 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
50 if (obj >= bitmap->max) { 48 if (obj >= bitmap->max) {
51 bitmap->top = (bitmap->top + b !! 49 bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask;
52 & bitmap->mask <<
53 obj = find_first_zero_bit(bitm 50 obj = find_first_zero_bit(bitmap->table, bitmap->max);
54 } 51 }
55 52
56 if (obj < bitmap->max) { 53 if (obj < bitmap->max) {
57 set_bit(obj, bitmap->table); 54 set_bit(obj, bitmap->table);
58 bitmap->last = (obj + 1); !! 55 bitmap->last = (obj + 1) & (bitmap->max - 1);
59 if (bitmap->last == bitmap->ma <<
60 bitmap->last = 0; <<
61 obj |= bitmap->top; 56 obj |= bitmap->top;
62 } else 57 } else
63 obj = -1; 58 obj = -1;
64 59
65 spin_unlock(&bitmap->lock); 60 spin_unlock(&bitmap->lock);
66 61
67 return obj; 62 return obj;
68 } 63 }
69 64
70 void mlx4_bitmap_free(struct mlx4_bitmap *bitm 65 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj)
71 { 66 {
72 mlx4_bitmap_free_range(bitmap, obj, 1) !! 67 obj &= bitmap->max - 1;
73 } <<
74 <<
75 static unsigned long find_aligned_range(unsign <<
76 u32 st <<
77 int le <<
78 { <<
79 unsigned long end, i; <<
80 <<
81 again: <<
82 start = ALIGN(start, align); <<
83 <<
84 while ((start < nbits) && test_bit(sta <<
85 start += align; <<
86 <<
87 if (start >= nbits) <<
88 return -1; <<
89 <<
90 end = start+len; <<
91 if (end > nbits) <<
92 return -1; <<
93 <<
94 for (i = start + 1; i < end; i++) { <<
95 if (test_bit(i, bitmap)) { <<
96 start = i + 1; <<
97 goto again; <<
98 } <<
99 } <<
100 <<
101 return start; <<
102 } <<
103 <<
104 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap <<
105 { <<
106 u32 obj, i; <<
107 <<
108 if (likely(cnt == 1 && align == 1)) <<
109 return mlx4_bitmap_alloc(bitma <<
110 <<
111 spin_lock(&bitmap->lock); <<
112 <<
113 obj = find_aligned_range(bitmap->table <<
114 bitmap->max, <<
115 if (obj >= bitmap->max) { <<
116 bitmap->top = (bitmap->top + b <<
117 & bitmap->mask <<
118 obj = find_aligned_range(bitma <<
119 cnt, <<
120 } <<
121 <<
122 if (obj < bitmap->max) { <<
123 for (i = 0; i < cnt; i++) <<
124 set_bit(obj + i, bitma <<
125 if (obj == bitmap->last) { <<
126 bitmap->last = (obj + <<
127 if (bitmap->last >= bi <<
128 bitmap->last = <<
129 } <<
130 obj |= bitmap->top; <<
131 } else <<
132 obj = -1; <<
133 <<
134 spin_unlock(&bitmap->lock); <<
135 <<
136 return obj; <<
137 } <<
138 <<
139 void mlx4_bitmap_free_range(struct mlx4_bitmap <<
140 { <<
141 u32 i; <<
142 <<
143 obj &= bitmap->max + bitmap->reserved_ <<
144 68
145 spin_lock(&bitmap->lock); 69 spin_lock(&bitmap->lock);
146 for (i = 0; i < cnt; i++) !! 70 clear_bit(obj, bitmap->table);
147 clear_bit(obj + i, bitmap->tab <<
148 bitmap->last = min(bitmap->last, obj); 71 bitmap->last = min(bitmap->last, obj);
149 bitmap->top = (bitmap->top + bitmap->m !! 72 bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask;
150 & bitmap->mask; <<
151 spin_unlock(&bitmap->lock); 73 spin_unlock(&bitmap->lock);
152 } 74 }
153 75
154 int mlx4_bitmap_init(struct mlx4_bitmap *bitma !! 76 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, u32 reserved)
155 u32 reserved_bot, u32 res <<
156 { 77 {
157 int i; 78 int i;
158 79
159 /* num must be a power of 2 */ 80 /* num must be a power of 2 */
160 if (num != roundup_pow_of_two(num)) 81 if (num != roundup_pow_of_two(num))
161 return -EINVAL; 82 return -EINVAL;
162 83
163 bitmap->last = 0; 84 bitmap->last = 0;
164 bitmap->top = 0; 85 bitmap->top = 0;
165 bitmap->max = num - reserved_top; !! 86 bitmap->max = num;
166 bitmap->mask = mask; 87 bitmap->mask = mask;
167 bitmap->reserved_top = reserved_top; <<
168 spin_lock_init(&bitmap->lock); 88 spin_lock_init(&bitmap->lock);
169 bitmap->table = kzalloc(BITS_TO_LONGS( !! 89 bitmap->table = kzalloc(BITS_TO_LONGS(num) * sizeof (long), GFP_KERNEL);
170 sizeof (long), <<
171 if (!bitmap->table) 90 if (!bitmap->table)
172 return -ENOMEM; 91 return -ENOMEM;
173 92
174 for (i = 0; i < reserved_bot; ++i) !! 93 for (i = 0; i < reserved; ++i)
175 set_bit(i, bitmap->table); 94 set_bit(i, bitmap->table);
176 95
177 return 0; 96 return 0;
178 } 97 }
179 98
180 void mlx4_bitmap_cleanup(struct mlx4_bitmap *b 99 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
181 { 100 {
182 kfree(bitmap->table); 101 kfree(bitmap->table);
183 } 102 }
184 103
185 /* 104 /*
186 * Handling for queue buffers -- we allocate a 105 * Handling for queue buffers -- we allocate a bunch of memory and
187 * register it in a memory region at HCA virtu 106 * register it in a memory region at HCA virtual address 0. If the
188 * requested size is > max_direct, we split th 107 * requested size is > max_direct, we split the allocation into
189 * multiple pages, so we don't require too muc 108 * multiple pages, so we don't require too much contiguous memory.
190 */ 109 */
191 110
192 int mlx4_buf_alloc(struct mlx4_dev *dev, int s 111 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
193 struct mlx4_buf *buf) 112 struct mlx4_buf *buf)
194 { 113 {
195 dma_addr_t t; 114 dma_addr_t t;
196 115
197 if (size <= max_direct) { 116 if (size <= max_direct) {
198 buf->nbufs = 1; 117 buf->nbufs = 1;
199 buf->npages = 1; 118 buf->npages = 1;
200 buf->page_shift = get_order( 119 buf->page_shift = get_order(size) + PAGE_SHIFT;
201 buf->direct.buf = dma_alloc_ 120 buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev,
202 121 size, &t, GFP_KERNEL);
203 if (!buf->direct.buf) 122 if (!buf->direct.buf)
204 return -ENOMEM; 123 return -ENOMEM;
205 124
206 buf->direct.map = t; 125 buf->direct.map = t;
207 126
208 while (t & ((1 << buf->page_sh 127 while (t & ((1 << buf->page_shift) - 1)) {
209 --buf->page_shift; 128 --buf->page_shift;
210 buf->npages *= 2; 129 buf->npages *= 2;
211 } 130 }
212 131
213 memset(buf->direct.buf, 0, siz 132 memset(buf->direct.buf, 0, size);
214 } else { 133 } else {
215 int i; 134 int i;
216 135
217 buf->nbufs = (size + PAG 136 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
218 buf->npages = buf->nbufs; 137 buf->npages = buf->nbufs;
219 buf->page_shift = PAGE_SHIFT; 138 buf->page_shift = PAGE_SHIFT;
220 buf->page_list = kzalloc(buf 139 buf->page_list = kzalloc(buf->nbufs * sizeof *buf->page_list,
221 GFP 140 GFP_KERNEL);
222 if (!buf->page_list) 141 if (!buf->page_list)
223 return -ENOMEM; 142 return -ENOMEM;
224 143
225 for (i = 0; i < buf->nbufs; ++ 144 for (i = 0; i < buf->nbufs; ++i) {
226 buf->page_list[i].buf 145 buf->page_list[i].buf =
227 dma_alloc_cohe 146 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
228 147 &t, GFP_KERNEL);
229 if (!buf->page_list[i] 148 if (!buf->page_list[i].buf)
230 goto err_free; 149 goto err_free;
231 150
232 buf->page_list[i].map 151 buf->page_list[i].map = t;
233 152
234 memset(buf->page_list[ 153 memset(buf->page_list[i].buf, 0, PAGE_SIZE);
235 } 154 }
236 155
237 if (BITS_PER_LONG == 64) { 156 if (BITS_PER_LONG == 64) {
238 struct page **pages; 157 struct page **pages;
239 pages = kmalloc(sizeof 158 pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL);
240 if (!pages) 159 if (!pages)
241 goto err_free; 160 goto err_free;
242 for (i = 0; i < buf->n 161 for (i = 0; i < buf->nbufs; ++i)
243 pages[i] = vir 162 pages[i] = virt_to_page(buf->page_list[i].buf);
244 buf->direct.buf = vmap 163 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
245 kfree(pages); 164 kfree(pages);
246 if (!buf->direct.buf) 165 if (!buf->direct.buf)
247 goto err_free; 166 goto err_free;
248 } 167 }
249 } 168 }
250 169
251 return 0; 170 return 0;
252 171
253 err_free: 172 err_free:
254 mlx4_buf_free(dev, size, buf); 173 mlx4_buf_free(dev, size, buf);
255 174
256 return -ENOMEM; 175 return -ENOMEM;
257 } 176 }
258 EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 177 EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
259 178
260 void mlx4_buf_free(struct mlx4_dev *dev, int s 179 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
261 { 180 {
262 int i; 181 int i;
263 182
264 if (buf->nbufs == 1) 183 if (buf->nbufs == 1)
265 dma_free_coherent(&dev->pdev-> 184 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,
266 buf->direct. 185 buf->direct.map);
267 else { 186 else {
268 if (BITS_PER_LONG == 64) 187 if (BITS_PER_LONG == 64)
269 vunmap(buf->direct.buf 188 vunmap(buf->direct.buf);
270 189
271 for (i = 0; i < buf->nbufs; ++ 190 for (i = 0; i < buf->nbufs; ++i)
272 if (buf->page_list[i]. 191 if (buf->page_list[i].buf)
273 dma_free_coher 192 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
274 193 buf->page_list[i].buf,
275 194 buf->page_list[i].map);
276 kfree(buf->page_list); 195 kfree(buf->page_list);
277 } 196 }
278 } 197 }
279 EXPORT_SYMBOL_GPL(mlx4_buf_free); 198 EXPORT_SYMBOL_GPL(mlx4_buf_free);
280 <<
281 static struct mlx4_db_pgdir *mlx4_alloc_db_pgd <<
282 { <<
283 struct mlx4_db_pgdir *pgdir; <<
284 <<
285 pgdir = kzalloc(sizeof *pgdir, GFP_KER <<
286 if (!pgdir) <<
287 return NULL; <<
288 <<
289 bitmap_fill(pgdir->order1, MLX4_DB_PER <<
290 pgdir->bits[0] = pgdir->order0; <<
291 pgdir->bits[1] = pgdir->order1; <<
292 pgdir->db_page = dma_alloc_coherent(dm <<
293 &p <<
294 if (!pgdir->db_page) { <<
295 kfree(pgdir); <<
296 return NULL; <<
297 } <<
298 <<
299 return pgdir; <<
300 } <<
301 <<
302 static int mlx4_alloc_db_from_pgdir(struct mlx <<
303 struct mlx <<
304 { <<
305 int o; <<
306 int i; <<
307 <<
308 for (o = order; o <= 1; ++o) { <<
309 i = find_first_bit(pgdir->bits <<
310 if (i < MLX4_DB_PER_PAGE >> o) <<
311 goto found; <<
312 } <<
313 <<
314 return -ENOMEM; <<
315 <<
316 found: <<
317 clear_bit(i, pgdir->bits[o]); <<
318 <<
319 i <<= o; <<
320 <<
321 if (o > order) <<
322 set_bit(i ^ 1, pgdir->bits[ord <<
323 <<
324 db->u.pgdir = pgdir; <<
325 db->index = i; <<
326 db->db = pgdir->db_page + db->ind <<
327 db->dma = pgdir->db_dma + db->ind <<
328 db->order = order; <<
329 <<
330 return 0; <<
331 } <<
332 <<
333 int mlx4_db_alloc(struct mlx4_dev *dev, struct <<
334 { <<
335 struct mlx4_priv *priv = mlx4_priv(dev <<
336 struct mlx4_db_pgdir *pgdir; <<
337 int ret = 0; <<
338 <<
339 mutex_lock(&priv->pgdir_mutex); <<
340 <<
341 list_for_each_entry(pgdir, &priv->pgdi <<
342 if (!mlx4_alloc_db_from_pgdir( <<
343 goto out; <<
344 <<
345 pgdir = mlx4_alloc_db_pgdir(&(dev->pde <<
346 if (!pgdir) { <<
347 ret = -ENOMEM; <<
348 goto out; <<
349 } <<
350 <<
351 list_add(&pgdir->list, &priv->pgdir_li <<
352 <<
353 /* This should never fail -- we just a <<
354 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir <<
355 <<
356 out: <<
357 mutex_unlock(&priv->pgdir_mutex); <<
358 <<
359 return ret; <<
360 } <<
361 EXPORT_SYMBOL_GPL(mlx4_db_alloc); <<
362 <<
363 void mlx4_db_free(struct mlx4_dev *dev, struct <<
364 { <<
365 struct mlx4_priv *priv = mlx4_priv(dev <<
366 int o; <<
367 int i; <<
368 <<
369 mutex_lock(&priv->pgdir_mutex); <<
370 <<
371 o = db->order; <<
372 i = db->index; <<
373 <<
374 if (db->order == 0 && test_bit(i ^ 1, <<
375 clear_bit(i ^ 1, db->u.pgdir-> <<
376 ++o; <<
377 } <<
378 i >>= o; <<
379 set_bit(i, db->u.pgdir->bits[o]); <<
380 <<
381 if (bitmap_full(db->u.pgdir->order1, M <<
382 dma_free_coherent(&(dev->pdev- <<
383 db->u.pgdir- <<
384 list_del(&db->u.pgdir->list); <<
385 kfree(db->u.pgdir); <<
386 } <<
387 <<
388 mutex_unlock(&priv->pgdir_mutex); <<
389 } <<
390 EXPORT_SYMBOL_GPL(mlx4_db_free); <<
391 <<
392 int mlx4_alloc_hwq_res(struct mlx4_dev *dev, s <<
393 int size, int max_direc <<
394 { <<
395 int err; <<
396 <<
397 err = mlx4_db_alloc(dev, &wqres->db, 1 <<
398 if (err) <<
399 return err; <<
400 <<
401 *wqres->db.db = 0; <<
402 <<
403 err = mlx4_buf_alloc(dev, size, max_di <<
404 if (err) <<
405 goto err_db; <<
406 <<
407 err = mlx4_mtt_init(dev, wqres->buf.np <<
408 &wqres->mtt); <<
409 if (err) <<
410 goto err_buf; <<
411 <<
412 err = mlx4_buf_write_mtt(dev, &wqres-> <<
413 if (err) <<
414 goto err_mtt; <<
415 <<
416 return 0; <<
417 <<
418 err_mtt: <<
419 mlx4_mtt_cleanup(dev, &wqres->mtt); <<
420 err_buf: <<
421 mlx4_buf_free(dev, size, &wqres->buf); <<
422 err_db: <<
423 mlx4_db_free(dev, &wqres->db); <<
424 <<
425 return err; <<
426 } <<
427 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); <<
428 <<
429 void mlx4_free_hwq_res(struct mlx4_dev *dev, s <<
430 int size) <<
431 { <<
432 mlx4_mtt_cleanup(dev, &wqres->mtt); <<
433 mlx4_buf_free(dev, size, &wqres->buf); <<
434 mlx4_db_free(dev, &wqres->db); <<
435 } <<
436 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); <<
437 199
| This page was automatically generated by the LXR engine. |