1 /*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28 #include "drmP.h"
29 #include "drm.h"
30 #include "i915_drm.h"
31 #include "i915_drv.h"
32 #include <linux/swap.h>
33 #include <linux/pci.h>
34
35 #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
36
37 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
38 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
39 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
40 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
41 int write);
42 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
43 uint64_t offset,
44 uint64_t size);
45 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
46 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
47 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
48 unsigned alignment);
49 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
50 static int i915_gem_evict_something(struct drm_device *dev);
51 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
52 struct drm_i915_gem_pwrite *args,
53 struct drm_file *file_priv);
54
55 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
56 unsigned long end)
57 {
58 drm_i915_private_t *dev_priv = dev->dev_private;
59
60 if (start >= end ||
61 (start & (PAGE_SIZE - 1)) != 0 ||
62 (end & (PAGE_SIZE - 1)) != 0) {
63 return -EINVAL;
64 }
65
66 drm_mm_init(&dev_priv->mm.gtt_space, start,
67 end - start);
68
69 dev->gtt_total = (uint32_t) (end - start);
70
71 return 0;
72 }
73
74 int
75 i915_gem_init_ioctl(struct drm_device *dev, void *data,
76 struct drm_file *file_priv)
77 {
78 struct drm_i915_gem_init *args = data;
79 int ret;
80
81 mutex_lock(&dev->struct_mutex);
82 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
83 mutex_unlock(&dev->struct_mutex);
84
85 return ret;
86 }
87
88 int
89 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
90 struct drm_file *file_priv)
91 {
92 struct drm_i915_gem_get_aperture *args = data;
93
94 if (!(dev->driver->driver_features & DRIVER_GEM))
95 return -ENODEV;
96
97 args->aper_size = dev->gtt_total;
98 args->aper_available_size = (args->aper_size -
99 atomic_read(&dev->pin_memory));
100
101 return 0;
102 }
103
104
105 /**
106 * Creates a new mm object and returns a handle to it.
107 */
108 int
109 i915_gem_create_ioctl(struct drm_device *dev, void *data,
110 struct drm_file *file_priv)
111 {
112 struct drm_i915_gem_create *args = data;
113 struct drm_gem_object *obj;
114 int handle, ret;
115
116 args->size = roundup(args->size, PAGE_SIZE);
117
118 /* Allocate the new object */
119 obj = drm_gem_object_alloc(dev, args->size);
120 if (obj == NULL)
121 return -ENOMEM;
122
123 ret = drm_gem_handle_create(file_priv, obj, &handle);
124 mutex_lock(&dev->struct_mutex);
125 drm_gem_object_handle_unreference(obj);
126 mutex_unlock(&dev->struct_mutex);
127
128 if (ret)
129 return ret;
130
131 args->handle = handle;
132
133 return 0;
134 }
135
136 static inline int
137 fast_shmem_read(struct page **pages,
138 loff_t page_base, int page_offset,
139 char __user *data,
140 int length)
141 {
142 char __iomem *vaddr;
143 int unwritten;
144
145 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
146 if (vaddr == NULL)
147 return -ENOMEM;
148 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
149 kunmap_atomic(vaddr, KM_USER0);
150
151 if (unwritten)
152 return -EFAULT;
153
154 return 0;
155 }
156
157 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
158 {
159 drm_i915_private_t *dev_priv = obj->dev->dev_private;
160 struct drm_i915_gem_object *obj_priv = obj->driver_private;
161
162 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
163 obj_priv->tiling_mode != I915_TILING_NONE;
164 }
165
166 static inline int
167 slow_shmem_copy(struct page *dst_page,
168 int dst_offset,
169 struct page *src_page,
170 int src_offset,
171 int length)
172 {
173 char *dst_vaddr, *src_vaddr;
174
175 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
176 if (dst_vaddr == NULL)
177 return -ENOMEM;
178
179 src_vaddr = kmap_atomic(src_page, KM_USER1);
180 if (src_vaddr == NULL) {
181 kunmap_atomic(dst_vaddr, KM_USER0);
182 return -ENOMEM;
183 }
184
185 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
186
187 kunmap_atomic(src_vaddr, KM_USER1);
188 kunmap_atomic(dst_vaddr, KM_USER0);
189
190 return 0;
191 }
192
193 static inline int
194 slow_shmem_bit17_copy(struct page *gpu_page,
195 int gpu_offset,
196 struct page *cpu_page,
197 int cpu_offset,
198 int length,
199 int is_read)
200 {
201 char *gpu_vaddr, *cpu_vaddr;
202
203 /* Use the unswizzled path if this page isn't affected. */
204 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
205 if (is_read)
206 return slow_shmem_copy(cpu_page, cpu_offset,
207 gpu_page, gpu_offset, length);
208 else
209 return slow_shmem_copy(gpu_page, gpu_offset,
210 cpu_page, cpu_offset, length);
211 }
212
213 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
214 if (gpu_vaddr == NULL)
215 return -ENOMEM;
216
217 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
218 if (cpu_vaddr == NULL) {
219 kunmap_atomic(gpu_vaddr, KM_USER0);
220 return -ENOMEM;
221 }
222
223 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
224 * XORing with the other bits (A9 for Y, A9 and A10 for X)
225 */
226 while (length > 0) {
227 int cacheline_end = ALIGN(gpu_offset + 1, 64);
228 int this_length = min(cacheline_end - gpu_offset, length);
229 int swizzled_gpu_offset = gpu_offset ^ 64;
230
231 if (is_read) {
232 memcpy(cpu_vaddr + cpu_offset,
233 gpu_vaddr + swizzled_gpu_offset,
234 this_length);
235 } else {
236 memcpy(gpu_vaddr + swizzled_gpu_offset,
237 cpu_vaddr + cpu_offset,
238 this_length);
239 }
240 cpu_offset += this_length;
241 gpu_offset += this_length;
242 length -= this_length;
243 }
244
245 kunmap_atomic(cpu_vaddr, KM_USER1);
246 kunmap_atomic(gpu_vaddr, KM_USER0);
247
248 return 0;
249 }
250
251 /**
252 * This is the fast shmem pread path, which attempts to copy_from_user directly
253 * from the backing pages of the object to the user's address space. On a
254 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
255 */
256 static int
257 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
258 struct drm_i915_gem_pread *args,
259 struct drm_file *file_priv)
260 {
261 struct drm_i915_gem_object *obj_priv = obj->driver_private;
262 ssize_t remain;
263 loff_t offset, page_base;
264 char __user *user_data;
265 int page_offset, page_length;
266 int ret;
267
268 user_data = (char __user *) (uintptr_t) args->data_ptr;
269 remain = args->size;
270
271 mutex_lock(&dev->struct_mutex);
272
273 ret = i915_gem_object_get_pages(obj);
274 if (ret != 0)
275 goto fail_unlock;
276
277 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
278 args->size);
279 if (ret != 0)
280 goto fail_put_pages;
281
282 obj_priv = obj->driver_private;
283 offset = args->offset;
284
285 while (remain > 0) {
286 /* Operation in this page
287 *
288 * page_base = page offset within aperture
289 * page_offset = offset within page
290 * page_length = bytes to copy for this page
291 */
292 page_base = (offset & ~(PAGE_SIZE-1));
293 page_offset = offset & (PAGE_SIZE-1);
294 page_length = remain;
295 if ((page_offset + remain) > PAGE_SIZE)
296 page_length = PAGE_SIZE - page_offset;
297
298 ret = fast_shmem_read(obj_priv->pages,
299 page_base, page_offset,
300 user_data, page_length);
301 if (ret)
302 goto fail_put_pages;
303
304 remain -= page_length;
305 user_data += page_length;
306 offset += page_length;
307 }
308
309 fail_put_pages:
310 i915_gem_object_put_pages(obj);
311 fail_unlock:
312 mutex_unlock(&dev->struct_mutex);
313
314 return ret;
315 }
316
317 /**
318 * This is the fallback shmem pread path, which allocates temporary storage
319 * in kernel space to copy_to_user into outside of the struct_mutex, so we
320 * can copy out of the object's backing pages while holding the struct mutex
321 * and not take page faults.
322 */
323 static int
324 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
325 struct drm_i915_gem_pread *args,
326 struct drm_file *file_priv)
327 {
328 struct drm_i915_gem_object *obj_priv = obj->driver_private;
329 struct mm_struct *mm = current->mm;
330 struct page **user_pages;
331 ssize_t remain;
332 loff_t offset, pinned_pages, i;
333 loff_t first_data_page, last_data_page, num_pages;
334 int shmem_page_index, shmem_page_offset;
335 int data_page_index, data_page_offset;
336 int page_length;
337 int ret;
338 uint64_t data_ptr = args->data_ptr;
339 int do_bit17_swizzling;
340
341 remain = args->size;
342
343 /* Pin the user pages containing the data. We can't fault while
344 * holding the struct mutex, yet we want to hold it while
345 * dereferencing the user data.
346 */
347 first_data_page = data_ptr / PAGE_SIZE;
348 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
349 num_pages = last_data_page - first_data_page + 1;
350
351 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
352 if (user_pages == NULL)
353 return -ENOMEM;
354
355 down_read(&mm->mmap_sem);
356 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
357 num_pages, 1, 0, user_pages, NULL);
358 up_read(&mm->mmap_sem);
359 if (pinned_pages < num_pages) {
360 ret = -EFAULT;
361 goto fail_put_user_pages;
362 }
363
364 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
365
366 mutex_lock(&dev->struct_mutex);
367
368 ret = i915_gem_object_get_pages(obj);
369 if (ret != 0)
370 goto fail_unlock;
371
372 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
373 args->size);
374 if (ret != 0)
375 goto fail_put_pages;
376
377 obj_priv = obj->driver_private;
378 offset = args->offset;
379
380 while (remain > 0) {
381 /* Operation in this page
382 *
383 * shmem_page_index = page number within shmem file
384 * shmem_page_offset = offset within page in shmem file
385 * data_page_index = page number in get_user_pages return
386 * data_page_offset = offset with data_page_index page.
387 * page_length = bytes to copy for this page
388 */
389 shmem_page_index = offset / PAGE_SIZE;
390 shmem_page_offset = offset & ~PAGE_MASK;
391 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
392 data_page_offset = data_ptr & ~PAGE_MASK;
393
394 page_length = remain;
395 if ((shmem_page_offset + page_length) > PAGE_SIZE)
396 page_length = PAGE_SIZE - shmem_page_offset;
397 if ((data_page_offset + page_length) > PAGE_SIZE)
398 page_length = PAGE_SIZE - data_page_offset;
399
400 if (do_bit17_swizzling) {
401 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
402 shmem_page_offset,
403 user_pages[data_page_index],
404 data_page_offset,
405 page_length,
406 1);
407 } else {
408 ret = slow_shmem_copy(user_pages[data_page_index],
409 data_page_offset,
410 obj_priv->pages[shmem_page_index],
411 shmem_page_offset,
412 page_length);
413 }
414 if (ret)
415 goto fail_put_pages;
416
417 remain -= page_length;
418 data_ptr += page_length;
419 offset += page_length;
420 }
421
422 fail_put_pages:
423 i915_gem_object_put_pages(obj);
424 fail_unlock:
425 mutex_unlock(&dev->struct_mutex);
426 fail_put_user_pages:
427 for (i = 0; i < pinned_pages; i++) {
428 SetPageDirty(user_pages[i]);
429 page_cache_release(user_pages[i]);
430 }
431 drm_free_large(user_pages);
432
433 return ret;
434 }
435
436 /**
437 * Reads data from the object referenced by handle.
438 *
439 * On error, the contents of *data are undefined.
440 */
441 int
442 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
443 struct drm_file *file_priv)
444 {
445 struct drm_i915_gem_pread *args = data;
446 struct drm_gem_object *obj;
447 struct drm_i915_gem_object *obj_priv;
448 int ret;
449
450 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
451 if (obj == NULL)
452 return -EBADF;
453 obj_priv = obj->driver_private;
454
455 /* Bounds check source.
456 *
457 * XXX: This could use review for overflow issues...
458 */
459 if (args->offset > obj->size || args->size > obj->size ||
460 args->offset + args->size > obj->size) {
461 drm_gem_object_unreference(obj);
462 return -EINVAL;
463 }
464
465 if (i915_gem_object_needs_bit17_swizzle(obj)) {
466 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
467 } else {
468 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
469 if (ret != 0)
470 ret = i915_gem_shmem_pread_slow(dev, obj, args,
471 file_priv);
472 }
473
474 drm_gem_object_unreference(obj);
475
476 return ret;
477 }
478
479 /* This is the fast write path which cannot handle
480 * page faults in the source data
481 */
482
483 static inline int
484 fast_user_write(struct io_mapping *mapping,
485 loff_t page_base, int page_offset,
486 char __user *user_data,
487 int length)
488 {
489 char *vaddr_atomic;
490 unsigned long unwritten;
491
492 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
493 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
494 user_data, length);
495 io_mapping_unmap_atomic(vaddr_atomic);
496 if (unwritten)
497 return -EFAULT;
498 return 0;
499 }
500
501 /* Here's the write path which can sleep for
502 * page faults
503 */
504
505 static inline int
506 slow_kernel_write(struct io_mapping *mapping,
507 loff_t gtt_base, int gtt_offset,
508 struct page *user_page, int user_offset,
509 int length)
510 {
511 char *src_vaddr, *dst_vaddr;
512 unsigned long unwritten;
513
514 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
515 src_vaddr = kmap_atomic(user_page, KM_USER1);
516 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
517 src_vaddr + user_offset,
518 length);
519 kunmap_atomic(src_vaddr, KM_USER1);
520 io_mapping_unmap_atomic(dst_vaddr);
521 if (unwritten)
522 return -EFAULT;
523 return 0;
524 }
525
526 static inline int
527 fast_shmem_write(struct page **pages,
528 loff_t page_base, int page_offset,
529 char __user *data,
530 int length)
531 {
532 char __iomem *vaddr;
533 unsigned long unwritten;
534
535 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
536 if (vaddr == NULL)
537 return -ENOMEM;
538 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
539 kunmap_atomic(vaddr, KM_USER0);
540
541 if (unwritten)
542 return -EFAULT;
543 return 0;
544 }
545
546 /**
547 * This is the fast pwrite path, where we copy the data directly from the
548 * user into the GTT, uncached.
549 */
550 static int
551 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
552 struct drm_i915_gem_pwrite *args,
553 struct drm_file *file_priv)
554 {
555 struct drm_i915_gem_object *obj_priv = obj->driver_private;
556 drm_i915_private_t *dev_priv = dev->dev_private;
557 ssize_t remain;
558 loff_t offset, page_base;
559 char __user *user_data;
560 int page_offset, page_length;
561 int ret;
562
563 user_data = (char __user *) (uintptr_t) args->data_ptr;
564 remain = args->size;
565 if (!access_ok(VERIFY_READ, user_data, remain))
566 return -EFAULT;
567
568
569 mutex_lock(&dev->struct_mutex);
570 ret = i915_gem_object_pin(obj, 0);
571 if (ret) {
572 mutex_unlock(&dev->struct_mutex);
573 return ret;
574 }
575 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
576 if (ret)
577 goto fail;
578
579 obj_priv = obj->driver_private;
580 offset = obj_priv->gtt_offset + args->offset;
581
582 while (remain > 0) {
583 /* Operation in this page
584 *
585 * page_base = page offset within aperture
586 * page_offset = offset within page
587 * page_length = bytes to copy for this page
588 */
589 page_base = (offset & ~(PAGE_SIZE-1));
590 page_offset = offset & (PAGE_SIZE-1);
591 page_length = remain;
592 if ((page_offset + remain) > PAGE_SIZE)
593 page_length = PAGE_SIZE - page_offset;
594
595 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
596 page_offset, user_data, page_length);
597
598 /* If we get a fault while copying data, then (presumably) our
599 * source page isn't available. Return the error and we'll
600 * retry in the slow path.
601 */
602 if (ret)
603 goto fail;
604
605 remain -= page_length;
606 user_data += page_length;
607 offset += page_length;
608 }
609
610 fail:
611 i915_gem_object_unpin(obj);
612 mutex_unlock(&dev->struct_mutex);
613
614 return ret;
615 }
616
617 /**
618 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
619 * the memory and maps it using kmap_atomic for copying.
620 *
621 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
622 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
623 */
624 static int
625 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
626 struct drm_i915_gem_pwrite *args,
627 struct drm_file *file_priv)
628 {
629 struct drm_i915_gem_object *obj_priv = obj->driver_private;
630 drm_i915_private_t *dev_priv = dev->dev_private;
631 ssize_t remain;
632 loff_t gtt_page_base, offset;
633 loff_t first_data_page, last_data_page, num_pages;
634 loff_t pinned_pages, i;
635 struct page **user_pages;
636 struct mm_struct *mm = current->mm;
637 int gtt_page_offset, data_page_offset, data_page_index, page_length;
638 int ret;
639 uint64_t data_ptr = args->data_ptr;
640
641 remain = args->size;
642
643 /* Pin the user pages containing the data. We can't fault while
644 * holding the struct mutex, and all of the pwrite implementations
645 * want to hold it while dereferencing the user data.
646 */
647 first_data_page = data_ptr / PAGE_SIZE;
648 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
649 num_pages = last_data_page - first_data_page + 1;
650
651 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
652 if (user_pages == NULL)
653 return -ENOMEM;
654
655 down_read(&mm->mmap_sem);
656 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
657 num_pages, 0, 0, user_pages, NULL);
658 up_read(&mm->mmap_sem);
659 if (pinned_pages < num_pages) {
660 ret = -EFAULT;
661 goto out_unpin_pages;
662 }
663
664 mutex_lock(&dev->struct_mutex);
665 ret = i915_gem_object_pin(obj, 0);
666 if (ret)
667 goto out_unlock;
668
669 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
670 if (ret)
671 goto out_unpin_object;
672
673 obj_priv = obj->driver_private;
674 offset = obj_priv->gtt_offset + args->offset;
675
676 while (remain > 0) {
677 /* Operation in this page
678 *
679 * gtt_page_base = page offset within aperture
680 * gtt_page_offset = offset within page in aperture
681 * data_page_index = page number in get_user_pages return
682 * data_page_offset = offset with data_page_index page.
683 * page_length = bytes to copy for this page
684 */
685 gtt_page_base = offset & PAGE_MASK;
686 gtt_page_offset = offset & ~PAGE_MASK;
687 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
688 data_page_offset = data_ptr & ~PAGE_MASK;
689
690 page_length = remain;
691 if ((gtt_page_offset + page_length) > PAGE_SIZE)
692 page_length = PAGE_SIZE - gtt_page_offset;
693 if ((data_page_offset + page_length) > PAGE_SIZE)
694 page_length = PAGE_SIZE - data_page_offset;
695
696 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
697 gtt_page_base, gtt_page_offset,
698 user_pages[data_page_index],
699 data_page_offset,
700 page_length);
701
702 /* If we get a fault while copying data, then (presumably) our
703 * source page isn't available. Return the error and we'll
704 * retry in the slow path.
705 */
706 if (ret)
707 goto out_unpin_object;
708
709 remain -= page_length;
710 offset += page_length;
711 data_ptr += page_length;
712 }
713
714 out_unpin_object:
715 i915_gem_object_unpin(obj);
716 out_unlock:
717 mutex_unlock(&dev->struct_mutex);
718 out_unpin_pages:
719 for (i = 0; i < pinned_pages; i++)
720 page_cache_release(user_pages[i]);
721 drm_free_large(user_pages);
722
723 return ret;
724 }
725
726 /**
727 * This is the fast shmem pwrite path, which attempts to directly
728 * copy_from_user into the kmapped pages backing the object.
729 */
730 static int
731 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
732 struct drm_i915_gem_pwrite *args,
733 struct drm_file *file_priv)
734 {
735 struct drm_i915_gem_object *obj_priv = obj->driver_private;
736 ssize_t remain;
737 loff_t offset, page_base;
738 char __user *user_data;
739 int page_offset, page_length;
740 int ret;
741
742 user_data = (char __user *) (uintptr_t) args->data_ptr;
743 remain = args->size;
744
745 mutex_lock(&dev->struct_mutex);
746
747 ret = i915_gem_object_get_pages(obj);
748 if (ret != 0)
749 goto fail_unlock;
750
751 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
752 if (ret != 0)
753 goto fail_put_pages;
754
755 obj_priv = obj->driver_private;
756 offset = args->offset;
757 obj_priv->dirty = 1;
758
759 while (remain > 0) {
760 /* Operation in this page
761 *
762 * page_base = page offset within aperture
763 * page_offset = offset within page
764 * page_length = bytes to copy for this page
765 */
766 page_base = (offset & ~(PAGE_SIZE-1));
767 page_offset = offset & (PAGE_SIZE-1);
768 page_length = remain;
769 if ((page_offset + remain) > PAGE_SIZE)
770 page_length = PAGE_SIZE - page_offset;
771
772 ret = fast_shmem_write(obj_priv->pages,
773 page_base, page_offset,
774 user_data, page_length);
775 if (ret)
776 goto fail_put_pages;
777
778 remain -= page_length;
779 user_data += page_length;
780 offset += page_length;
781 }
782
783 fail_put_pages:
784 i915_gem_object_put_pages(obj);
785 fail_unlock:
786 mutex_unlock(&dev->struct_mutex);
787
788 return ret;
789 }
790
791 /**
792 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
793 * the memory and maps it using kmap_atomic for copying.
794 *
795 * This avoids taking mmap_sem for faulting on the user's address while the
796 * struct_mutex is held.
797 */
798 static int
799 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
800 struct drm_i915_gem_pwrite *args,
801 struct drm_file *file_priv)
802 {
803 struct drm_i915_gem_object *obj_priv = obj->driver_private;
804 struct mm_struct *mm = current->mm;
805 struct page **user_pages;
806 ssize_t remain;
807 loff_t offset, pinned_pages, i;
808 loff_t first_data_page, last_data_page, num_pages;
809 int shmem_page_index, shmem_page_offset;
810 int data_page_index, data_page_offset;
811 int page_length;
812 int ret;
813 uint64_t data_ptr = args->data_ptr;
814 int do_bit17_swizzling;
815
816 remain = args->size;
817
818 /* Pin the user pages containing the data. We can't fault while
819 * holding the struct mutex, and all of the pwrite implementations
820 * want to hold it while dereferencing the user data.
821 */
822 first_data_page = data_ptr / PAGE_SIZE;
823 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
824 num_pages = last_data_page - first_data_page + 1;
825
826 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
827 if (user_pages == NULL)
828 return -ENOMEM;
829
830 down_read(&mm->mmap_sem);
831 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
832 num_pages, 0, 0, user_pages, NULL);
833 up_read(&mm->mmap_sem);
834 if (pinned_pages < num_pages) {
835 ret = -EFAULT;
836 goto fail_put_user_pages;
837 }
838
839 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
840
841 mutex_lock(&dev->struct_mutex);
842
843 ret = i915_gem_object_get_pages(obj);
844 if (ret != 0)
845 goto fail_unlock;
846
847 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
848 if (ret != 0)
849 goto fail_put_pages;
850
851 obj_priv = obj->driver_private;
852 offset = args->offset;
853 obj_priv->dirty = 1;
854
855 while (remain > 0) {
856 /* Operation in this page
857 *
858 * shmem_page_index = page number within shmem file
859 * shmem_page_offset = offset within page in shmem file
860 * data_page_index = page number in get_user_pages return
861 * data_page_offset = offset with data_page_index page.
862 * page_length = bytes to copy for this page
863 */
864 shmem_page_index = offset / PAGE_SIZE;
865 shmem_page_offset = offset & ~PAGE_MASK;
866 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
867 data_page_offset = data_ptr & ~PAGE_MASK;
868
869 page_length = remain;
870 if ((shmem_page_offset + page_length) > PAGE_SIZE)
871 page_length = PAGE_SIZE - shmem_page_offset;
872 if ((data_page_offset + page_length) > PAGE_SIZE)
873 page_length = PAGE_SIZE - data_page_offset;
874
875 if (do_bit17_swizzling) {
876 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
877 shmem_page_offset,
878 user_pages[data_page_index],
879 data_page_offset,
880 page_length,
881 0);
882 } else {
883 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
884 shmem_page_offset,
885 user_pages[data_page_index],
886 data_page_offset,
887 page_length);
888 }
889 if (ret)
890 goto fail_put_pages;
891
892 remain -= page_length;
893 data_ptr += page_length;
894 offset += page_length;
895 }
896
897 fail_put_pages:
898 i915_gem_object_put_pages(obj);
899 fail_unlock:
900 mutex_unlock(&dev->struct_mutex);
901 fail_put_user_pages:
902 for (i = 0; i < pinned_pages; i++)
903 page_cache_release(user_pages[i]);
904 drm_free_large(user_pages);
905
906 return ret;
907 }
908
909 /**
910 * Writes data to the object referenced by handle.
911 *
912 * On error, the contents of the buffer that were to be modified are undefined.
913 */
914 int
915 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
916 struct drm_file *file_priv)
917 {
918 struct drm_i915_gem_pwrite *args = data;
919 struct drm_gem_object *obj;
920 struct drm_i915_gem_object *obj_priv;
921 int ret = 0;
922
923 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
924 if (obj == NULL)
925 return -EBADF;
926 obj_priv = obj->driver_private;
927
928 /* Bounds check destination.
929 *
930 * XXX: This could use review for overflow issues...
931 */
932 if (args->offset > obj->size || args->size > obj->size ||
933 args->offset + args->size > obj->size) {
934 drm_gem_object_unreference(obj);
935 return -EINVAL;
936 }
937
938 /* We can only do the GTT pwrite on untiled buffers, as otherwise
939 * it would end up going through the fenced access, and we'll get
940 * different detiling behavior between reading and writing.
941 * pread/pwrite currently are reading and writing from the CPU
942 * perspective, requiring manual detiling by the client.
943 */
944 if (obj_priv->phys_obj)
945 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
946 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
947 dev->gtt_total != 0) {
948 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
949 if (ret == -EFAULT) {
950 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
951 file_priv);
952 }
953 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
954 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
955 } else {
956 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
957 if (ret == -EFAULT) {
958 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
959 file_priv);
960 }
961 }
962
963 #if WATCH_PWRITE
964 if (ret)
965 DRM_INFO("pwrite failed %d\n", ret);
966 #endif
967
968 drm_gem_object_unreference(obj);
969
970 return ret;
971 }
972
973 /**
974 * Called when user space prepares to use an object with the CPU, either
975 * through the mmap ioctl's mapping or a GTT mapping.
976 */
977 int
978 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
979 struct drm_file *file_priv)
980 {
981 struct drm_i915_private *dev_priv = dev->dev_private;
982 struct drm_i915_gem_set_domain *args = data;
983 struct drm_gem_object *obj;
984 uint32_t read_domains = args->read_domains;
985 uint32_t write_domain = args->write_domain;
986 int ret;
987
988 if (!(dev->driver->driver_features & DRIVER_GEM))
989 return -ENODEV;
990
991 /* Only handle setting domains to types used by the CPU. */
992 if (write_domain & I915_GEM_GPU_DOMAINS)
993 return -EINVAL;
994
995 if (read_domains & I915_GEM_GPU_DOMAINS)
996 return -EINVAL;
997
998 /* Having something in the write domain implies it's in the read
999 * domain, and only that read domain. Enforce that in the request.
1000 */
1001 if (write_domain != 0 && read_domains != write_domain)
1002 return -EINVAL;
1003
1004 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1005 if (obj == NULL)
1006 return -EBADF;
1007
1008 mutex_lock(&dev->struct_mutex);
1009 #if WATCH_BUF
1010 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1011 obj, obj->size, read_domains, write_domain);
1012 #endif
1013 if (read_domains & I915_GEM_DOMAIN_GTT) {
1014 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1015
1016 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1017
1018 /* Update the LRU on the fence for the CPU access that's
1019 * about to occur.
1020 */
1021 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1022 list_move_tail(&obj_priv->fence_list,
1023 &dev_priv->mm.fence_list);
1024 }
1025
1026 /* Silently promote "you're not bound, there was nothing to do"
1027 * to success, since the client was just asking us to
1028 * make sure everything was done.
1029 */
1030 if (ret == -EINVAL)
1031 ret = 0;
1032 } else {
1033 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1034 }
1035
1036 drm_gem_object_unreference(obj);
1037 mutex_unlock(&dev->struct_mutex);
1038 return ret;
1039 }
1040
1041 /**
1042 * Called when user space has done writes to this buffer
1043 */
1044 int
1045 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1046 struct drm_file *file_priv)
1047 {
1048 struct drm_i915_gem_sw_finish *args = data;
1049 struct drm_gem_object *obj;
1050 struct drm_i915_gem_object *obj_priv;
1051 int ret = 0;
1052
1053 if (!(dev->driver->driver_features & DRIVER_GEM))
1054 return -ENODEV;
1055
1056 mutex_lock(&dev->struct_mutex);
1057 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1058 if (obj == NULL) {
1059 mutex_unlock(&dev->struct_mutex);
1060 return -EBADF;
1061 }
1062
1063 #if WATCH_BUF
1064 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1065 __func__, args->handle, obj, obj->size);
1066 #endif
1067 obj_priv = obj->driver_private;
1068
1069 /* Pinned buffers may be scanout, so flush the cache */
1070 if (obj_priv->pin_count)
1071 i915_gem_object_flush_cpu_write_domain(obj);
1072
1073 drm_gem_object_unreference(obj);
1074 mutex_unlock(&dev->struct_mutex);
1075 return ret;
1076 }
1077
1078 /**
1079 * Maps the contents of an object, returning the address it is mapped
1080 * into.
1081 *
1082 * While the mapping holds a reference on the contents of the object, it doesn't
1083 * imply a ref on the object itself.
1084 */
1085 int
1086 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1087 struct drm_file *file_priv)
1088 {
1089 struct drm_i915_gem_mmap *args = data;
1090 struct drm_gem_object *obj;
1091 loff_t offset;
1092 unsigned long addr;
1093
1094 if (!(dev->driver->driver_features & DRIVER_GEM))
1095 return -ENODEV;
1096
1097 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1098 if (obj == NULL)
1099 return -EBADF;
1100
1101 offset = args->offset;
1102
1103 down_write(¤t->mm->mmap_sem);
1104 addr = do_mmap(obj->filp, 0, args->size,
1105 PROT_READ | PROT_WRITE, MAP_SHARED,
1106 args->offset);
1107 up_write(¤t->mm->mmap_sem);
1108 mutex_lock(&dev->struct_mutex);
1109 drm_gem_object_unreference(obj);
1110 mutex_unlock(&dev->struct_mutex);
1111 if (IS_ERR((void *)addr))
1112 return addr;
1113
1114 args->addr_ptr = (uint64_t) addr;
1115
1116 return 0;
1117 }
1118
1119 /**
1120 * i915_gem_fault - fault a page into the GTT
1121 * vma: VMA in question
1122 * vmf: fault info
1123 *
1124 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1125 * from userspace. The fault handler takes care of binding the object to
1126 * the GTT (if needed), allocating and programming a fence register (again,
1127 * only if needed based on whether the old reg is still valid or the object
1128 * is tiled) and inserting a new PTE into the faulting process.
1129 *
1130 * Note that the faulting process may involve evicting existing objects
1131 * from the GTT and/or fence registers to make room. So performance may
1132 * suffer if the GTT working set is large or there are few fence registers
1133 * left.
1134 */
1135 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1136 {
1137 struct drm_gem_object *obj = vma->vm_private_data;
1138 struct drm_device *dev = obj->dev;
1139 struct drm_i915_private *dev_priv = dev->dev_private;
1140 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1141 pgoff_t page_offset;
1142 unsigned long pfn;
1143 int ret = 0;
1144 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1145
1146 /* We don't use vmf->pgoff since that has the fake offset */
1147 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1148 PAGE_SHIFT;
1149
1150 /* Now bind it into the GTT if needed */
1151 mutex_lock(&dev->struct_mutex);
1152 if (!obj_priv->gtt_space) {
1153 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1154 if (ret)
1155 goto unlock;
1156
1157 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1158
1159 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1160 if (ret)
1161 goto unlock;
1162 }
1163
1164 /* Need a new fence register? */
1165 if (obj_priv->tiling_mode != I915_TILING_NONE) {
1166 ret = i915_gem_object_get_fence_reg(obj);
1167 if (ret)
1168 goto unlock;
1169 }
1170
1171 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1172 page_offset;
1173
1174 /* Finally, remap it using the new GTT offset */
1175 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1176 unlock:
1177 mutex_unlock(&dev->struct_mutex);
1178
1179 switch (ret) {
1180 case 0:
1181 case -ERESTARTSYS:
1182 return VM_FAULT_NOPAGE;
1183 case -ENOMEM:
1184 case -EAGAIN:
1185 return VM_FAULT_OOM;
1186 default:
1187 return VM_FAULT_SIGBUS;
1188 }
1189 }
1190
1191 /**
1192 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1193 * @obj: obj in question
1194 *
1195 * GEM memory mapping works by handing back to userspace a fake mmap offset
1196 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1197 * up the object based on the offset and sets up the various memory mapping
1198 * structures.
1199 *
1200 * This routine allocates and attaches a fake offset for @obj.
1201 */
1202 static int
1203 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1204 {
1205 struct drm_device *dev = obj->dev;
1206 struct drm_gem_mm *mm = dev->mm_private;
1207 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1208 struct drm_map_list *list;
1209 struct drm_local_map *map;
1210 int ret = 0;
1211
1212 /* Set the object up for mmap'ing */
1213 list = &obj->map_list;
1214 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1215 if (!list->map)
1216 return -ENOMEM;
1217
1218 map = list->map;
1219 map->type = _DRM_GEM;
1220 map->size = obj->size;
1221 map->handle = obj;
1222
1223 /* Get a DRM GEM mmap offset allocated... */
1224 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1225 obj->size / PAGE_SIZE, 0, 0);
1226 if (!list->file_offset_node) {
1227 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1228 ret = -ENOMEM;
1229 goto out_free_list;
1230 }
1231
1232 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1233 obj->size / PAGE_SIZE, 0);
1234 if (!list->file_offset_node) {
1235 ret = -ENOMEM;
1236 goto out_free_list;
1237 }
1238
1239 list->hash.key = list->file_offset_node->start;
1240 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1241 DRM_ERROR("failed to add to map hash\n");
1242 goto out_free_mm;
1243 }
1244
1245 /* By now we should be all set, any drm_mmap request on the offset
1246 * below will get to our mmap & fault handler */
1247 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1248
1249 return 0;
1250
1251 out_free_mm:
1252 drm_mm_put_block(list->file_offset_node);
1253 out_free_list:
1254 kfree(list->map);
1255
1256 return ret;
1257 }
1258
1259 /**
1260 * i915_gem_release_mmap - remove physical page mappings
1261 * @obj: obj in question
1262 *
1263 * Preserve the reservation of the mmaping with the DRM core code, but
1264 * relinquish ownership of the pages back to the system.
1265 *
1266 * It is vital that we remove the page mapping if we have mapped a tiled
1267 * object through the GTT and then lose the fence register due to
1268 * resource pressure. Similarly if the object has been moved out of the
1269 * aperture, than pages mapped into userspace must be revoked. Removing the
1270 * mapping will then trigger a page fault on the next user access, allowing
1271 * fixup by i915_gem_fault().
1272 */
1273 void
1274 i915_gem_release_mmap(struct drm_gem_object *obj)
1275 {
1276 struct drm_device *dev = obj->dev;
1277 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1278
1279 if (dev->dev_mapping)
1280 unmap_mapping_range(dev->dev_mapping,
1281 obj_priv->mmap_offset, obj->size, 1);
1282 }
1283
1284 static void
1285 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1286 {
1287 struct drm_device *dev = obj->dev;
1288 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1289 struct drm_gem_mm *mm = dev->mm_private;
1290 struct drm_map_list *list;
1291
1292 list = &obj->map_list;
1293 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1294
1295 if (list->file_offset_node) {
1296 drm_mm_put_block(list->file_offset_node);
1297 list->file_offset_node = NULL;
1298 }
1299
1300 if (list->map) {
1301 kfree(list->map);
1302 list->map = NULL;
1303 }
1304
1305 obj_priv->mmap_offset = 0;
1306 }
1307
1308 /**
1309 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1310 * @obj: object to check
1311 *
1312 * Return the required GTT alignment for an object, taking into account
1313 * potential fence register mapping if needed.
1314 */
1315 static uint32_t
1316 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1317 {
1318 struct drm_device *dev = obj->dev;
1319 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1320 int start, i;
1321
1322 /*
1323 * Minimum alignment is 4k (GTT page size), but might be greater
1324 * if a fence register is needed for the object.
1325 */
1326 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1327 return 4096;
1328
1329 /*
1330 * Previous chips need to be aligned to the size of the smallest
1331 * fence register that can contain the object.
1332 */
1333 if (IS_I9XX(dev))
1334 start = 1024*1024;
1335 else
1336 start = 512*1024;
1337
1338 for (i = start; i < obj->size; i <<= 1)
1339 ;
1340
1341 return i;
1342 }
1343
1344 /**
1345 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1346 * @dev: DRM device
1347 * @data: GTT mapping ioctl data
1348 * @file_priv: GEM object info
1349 *
1350 * Simply returns the fake offset to userspace so it can mmap it.
1351 * The mmap call will end up in drm_gem_mmap(), which will set things
1352 * up so we can get faults in the handler above.
1353 *
1354 * The fault handler will take care of binding the object into the GTT
1355 * (since it may have been evicted to make room for something), allocating
1356 * a fence register, and mapping the appropriate aperture address into
1357 * userspace.
1358 */
1359 int
1360 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1361 struct drm_file *file_priv)
1362 {
1363 struct drm_i915_gem_mmap_gtt *args = data;
1364 struct drm_i915_private *dev_priv = dev->dev_private;
1365 struct drm_gem_object *obj;
1366 struct drm_i915_gem_object *obj_priv;
1367 int ret;
1368
1369 if (!(dev->driver->driver_features & DRIVER_GEM))
1370 return -ENODEV;
1371
1372 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1373 if (obj == NULL)
1374 return -EBADF;
1375
1376 mutex_lock(&dev->struct_mutex);
1377
1378 obj_priv = obj->driver_private;
1379
1380 if (!obj_priv->mmap_offset) {
1381 ret = i915_gem_create_mmap_offset(obj);
1382 if (ret) {
1383 drm_gem_object_unreference(obj);
1384 mutex_unlock(&dev->struct_mutex);
1385 return ret;
1386 }
1387 }
1388
1389 args->offset = obj_priv->mmap_offset;
1390
1391 obj_priv->gtt_alignment = i915_gem_get_gtt_alignment(obj);
1392
1393 /* Make sure the alignment is correct for fence regs etc */
1394 if (obj_priv->agp_mem &&
1395 (obj_priv->gtt_offset & (obj_priv->gtt_alignment - 1))) {
1396 drm_gem_object_unreference(obj);
1397 mutex_unlock(&dev->struct_mutex);
1398 return -EINVAL;
1399 }
1400
1401 /*
1402 * Pull it into the GTT so that we have a page list (makes the
1403 * initial fault faster and any subsequent flushing possible).
1404 */
1405 if (!obj_priv->agp_mem) {
1406 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1407 if (ret) {
1408 drm_gem_object_unreference(obj);
1409 mutex_unlock(&dev->struct_mutex);
1410 return ret;
1411 }
1412 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1413 }
1414
1415 drm_gem_object_unreference(obj);
1416 mutex_unlock(&dev->struct_mutex);
1417
1418 return 0;
1419 }
1420
1421 void
1422 i915_gem_object_put_pages(struct drm_gem_object *obj)
1423 {
1424 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1425 int page_count = obj->size / PAGE_SIZE;
1426 int i;
1427
1428 BUG_ON(obj_priv->pages_refcount == 0);
1429
1430 if (--obj_priv->pages_refcount != 0)
1431 return;
1432
1433 if (obj_priv->tiling_mode != I915_TILING_NONE)
1434 i915_gem_object_save_bit_17_swizzle(obj);
1435
1436 for (i = 0; i < page_count; i++)
1437 if (obj_priv->pages[i] != NULL) {
1438 if (obj_priv->dirty)
1439 set_page_dirty(obj_priv->pages[i]);
1440 mark_page_accessed(obj_priv->pages[i]);
1441 page_cache_release(obj_priv->pages[i]);
1442 }
1443 obj_priv->dirty = 0;
1444
1445 drm_free_large(obj_priv->pages);
1446 obj_priv->pages = NULL;
1447 }
1448
1449 static void
1450 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
1451 {
1452 struct drm_device *dev = obj->dev;
1453 drm_i915_private_t *dev_priv = dev->dev_private;
1454 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1455
1456 /* Add a reference if we're newly entering the active list. */
1457 if (!obj_priv->active) {
1458 drm_gem_object_reference(obj);
1459 obj_priv->active = 1;
1460 }
1461 /* Move from whatever list we were on to the tail of execution. */
1462 spin_lock(&dev_priv->mm.active_list_lock);
1463 list_move_tail(&obj_priv->list,
1464 &dev_priv->mm.active_list);
1465 spin_unlock(&dev_priv->mm.active_list_lock);
1466 obj_priv->last_rendering_seqno = seqno;
1467 }
1468
1469 static void
1470 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1471 {
1472 struct drm_device *dev = obj->dev;
1473 drm_i915_private_t *dev_priv = dev->dev_private;
1474 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1475
1476 BUG_ON(!obj_priv->active);
1477 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1478 obj_priv->last_rendering_seqno = 0;
1479 }
1480
1481 static void
1482 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1483 {
1484 struct drm_device *dev = obj->dev;
1485 drm_i915_private_t *dev_priv = dev->dev_private;
1486 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1487
1488 i915_verify_inactive(dev, __FILE__, __LINE__);
1489 if (obj_priv->pin_count != 0)
1490 list_del_init(&obj_priv->list);
1491 else
1492 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1493
1494 obj_priv->last_rendering_seqno = 0;
1495 if (obj_priv->active) {
1496 obj_priv->active = 0;
1497 drm_gem_object_unreference(obj);
1498 }
1499 i915_verify_inactive(dev, __FILE__, __LINE__);
1500 }
1501
1502 /**
1503 * Creates a new sequence number, emitting a write of it to the status page
1504 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1505 *
1506 * Must be called with struct_lock held.
1507 *
1508 * Returned sequence numbers are nonzero on success.
1509 */
1510 static uint32_t
1511 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1512 uint32_t flush_domains)
1513 {
1514 drm_i915_private_t *dev_priv = dev->dev_private;
1515 struct drm_i915_file_private *i915_file_priv = NULL;
1516 struct drm_i915_gem_request *request;
1517 uint32_t seqno;
1518 int was_empty;
1519 RING_LOCALS;
1520
1521 if (file_priv != NULL)
1522 i915_file_priv = file_priv->driver_priv;
1523
1524 request = kzalloc(sizeof(*request), GFP_KERNEL);
1525 if (request == NULL)
1526 return 0;
1527
1528 /* Grab the seqno we're going to make this request be, and bump the
1529 * next (skipping 0 so it can be the reserved no-seqno value).
1530 */
1531 seqno = dev_priv->mm.next_gem_seqno;
1532 dev_priv->mm.next_gem_seqno++;
1533 if (dev_priv->mm.next_gem_seqno == 0)
1534 dev_priv->mm.next_gem_seqno++;
1535
1536 BEGIN_LP_RING(4);
1537 OUT_RING(MI_STORE_DWORD_INDEX);
1538 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1539 OUT_RING(seqno);
1540
1541 OUT_RING(MI_USER_INTERRUPT);
1542 ADVANCE_LP_RING();
1543
1544 DRM_DEBUG("%d\n", seqno);
1545
1546 request->seqno = seqno;
1547 request->emitted_jiffies = jiffies;
1548 was_empty = list_empty(&dev_priv->mm.request_list);
1549 list_add_tail(&request->list, &dev_priv->mm.request_list);
1550 if (i915_file_priv) {
1551 list_add_tail(&request->client_list,
1552 &i915_file_priv->mm.request_list);
1553 } else {
1554 INIT_LIST_HEAD(&request->client_list);
1555 }
1556
1557 /* Associate any objects on the flushing list matching the write
1558 * domain we're flushing with our flush.
1559 */
1560 if (flush_domains != 0) {
1561 struct drm_i915_gem_object *obj_priv, *next;
1562
1563 list_for_each_entry_safe(obj_priv, next,
1564 &dev_priv->mm.flushing_list, list) {
1565 struct drm_gem_object *obj = obj_priv->obj;
1566
1567 if ((obj->write_domain & flush_domains) ==
1568 obj->write_domain) {
1569 obj->write_domain = 0;
1570 i915_gem_object_move_to_active(obj, seqno);
1571 }
1572 }
1573
1574 }
1575
1576 if (was_empty && !dev_priv->mm.suspended)
1577 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1578 return seqno;
1579 }
1580
1581 /**
1582 * Command execution barrier
1583 *
1584 * Ensures that all commands in the ring are finished
1585 * before signalling the CPU
1586 */
1587 static uint32_t
1588 i915_retire_commands(struct drm_device *dev)
1589 {
1590 drm_i915_private_t *dev_priv = dev->dev_private;
1591 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1592 uint32_t flush_domains = 0;
1593 RING_LOCALS;
1594
1595 /* The sampler always gets flushed on i965 (sigh) */
1596 if (IS_I965G(dev))
1597 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1598 BEGIN_LP_RING(2);
1599 OUT_RING(cmd);
1600 OUT_RING(0); /* noop */
1601 ADVANCE_LP_RING();
1602 return flush_domains;
1603 }
1604
1605 /**
1606 * Moves buffers associated only with the given active seqno from the active
1607 * to inactive list, potentially freeing them.
1608 */
1609 static void
1610 i915_gem_retire_request(struct drm_device *dev,
1611 struct drm_i915_gem_request *request)
1612 {
1613 drm_i915_private_t *dev_priv = dev->dev_private;
1614
1615 /* Move any buffers on the active list that are no longer referenced
1616 * by the ringbuffer to the flushing/inactive lists as appropriate.
1617 */
1618 spin_lock(&dev_priv->mm.active_list_lock);
1619 while (!list_empty(&dev_priv->mm.active_list)) {
1620 struct drm_gem_object *obj;
1621 struct drm_i915_gem_object *obj_priv;
1622
1623 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1624 struct drm_i915_gem_object,
1625 list);
1626 obj = obj_priv->obj;
1627
1628 /* If the seqno being retired doesn't match the oldest in the
1629 * list, then the oldest in the list must still be newer than
1630 * this seqno.
1631 */
1632 if (obj_priv->last_rendering_seqno != request->seqno)
1633 goto out;
1634
1635 #if WATCH_LRU
1636 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1637 __func__, request->seqno, obj);
1638 #endif
1639
1640 if (obj->write_domain != 0)
1641 i915_gem_object_move_to_flushing(obj);
1642 else {
1643 /* Take a reference on the object so it won't be
1644 * freed while the spinlock is held. The list
1645 * protection for this spinlock is safe when breaking
1646 * the lock like this since the next thing we do
1647 * is just get the head of the list again.
1648 */
1649 drm_gem_object_reference(obj);
1650 i915_gem_object_move_to_inactive(obj);
1651 spin_unlock(&dev_priv->mm.active_list_lock);
1652 drm_gem_object_unreference(obj);
1653 spin_lock(&dev_priv->mm.active_list_lock);
1654 }
1655 }
1656 out:
1657 spin_unlock(&dev_priv->mm.active_list_lock);
1658 }
1659
1660 /**
1661 * Returns true if seq1 is later than seq2.
1662 */
1663 static int
1664 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1665 {
1666 return (int32_t)(seq1 - seq2) >= 0;
1667 }
1668
1669 uint32_t
1670 i915_get_gem_seqno(struct drm_device *dev)
1671 {
1672 drm_i915_private_t *dev_priv = dev->dev_private;
1673
1674 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1675 }
1676
1677 /**
1678 * This function clears the request list as sequence numbers are passed.
1679 */
1680 void
1681 i915_gem_retire_requests(struct drm_device *dev)
1682 {
1683 drm_i915_private_t *dev_priv = dev->dev_private;
1684 uint32_t seqno;
1685
1686 if (!dev_priv->hw_status_page)
1687 return;
1688
1689 seqno = i915_get_gem_seqno(dev);
1690
1691 while (!list_empty(&dev_priv->mm.request_list)) {
1692 struct drm_i915_gem_request *request;
1693 uint32_t retiring_seqno;
1694
1695 request = list_first_entry(&dev_priv->mm.request_list,
1696 struct drm_i915_gem_request,
1697 list);
1698 retiring_seqno = request->seqno;
1699
1700 if (i915_seqno_passed(seqno, retiring_seqno) ||
1701 dev_priv->mm.wedged) {
1702 i915_gem_retire_request(dev, request);
1703
1704 list_del(&request->list);
1705 list_del(&request->client_list);
1706 kfree(request);
1707 } else
1708 break;
1709 }
1710 }
1711
1712 void
1713 i915_gem_retire_work_handler(struct work_struct *work)
1714 {
1715 drm_i915_private_t *dev_priv;
1716 struct drm_device *dev;
1717
1718 dev_priv = container_of(work, drm_i915_private_t,
1719 mm.retire_work.work);
1720 dev = dev_priv->dev;
1721
1722 mutex_lock(&dev->struct_mutex);
1723 i915_gem_retire_requests(dev);
1724 if (!dev_priv->mm.suspended &&
1725 !list_empty(&dev_priv->mm.request_list))
1726 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1727 mutex_unlock(&dev->struct_mutex);
1728 }
1729
1730 /**
1731 * Waits for a sequence number to be signaled, and cleans up the
1732 * request and object lists appropriately for that event.
1733 */
1734 static int
1735 i915_wait_request(struct drm_device *dev, uint32_t seqno)
1736 {
1737 drm_i915_private_t *dev_priv = dev->dev_private;
1738 u32 ier;
1739 int ret = 0;
1740
1741 BUG_ON(seqno == 0);
1742
1743 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
1744 if (IS_IGDNG(dev))
1745 ier = I915_READ(DEIER) | I915_READ(GTIER);
1746 else
1747 ier = I915_READ(IER);
1748 if (!ier) {
1749 DRM_ERROR("something (likely vbetool) disabled "
1750 "interrupts, re-enabling\n");
1751 i915_driver_irq_preinstall(dev);
1752 i915_driver_irq_postinstall(dev);
1753 }
1754
1755 dev_priv->mm.waiting_gem_seqno = seqno;
1756 i915_user_irq_get(dev);
1757 ret = wait_event_interruptible(dev_priv->irq_queue,
1758 i915_seqno_passed(i915_get_gem_seqno(dev),
1759 seqno) ||
1760 dev_priv->mm.wedged);
1761 i915_user_irq_put(dev);
1762 dev_priv->mm.waiting_gem_seqno = 0;
1763 }
1764 if (dev_priv->mm.wedged)
1765 ret = -EIO;
1766
1767 if (ret && ret != -ERESTARTSYS)
1768 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1769 __func__, ret, seqno, i915_get_gem_seqno(dev));
1770
1771 /* Directly dispatch request retiring. While we have the work queue
1772 * to handle this, the waiter on a request often wants an associated
1773 * buffer to have made it to the inactive list, and we would need
1774 * a separate wait queue to handle that.
1775 */
1776 if (ret == 0)
1777 i915_gem_retire_requests(dev);
1778
1779 return ret;
1780 }
1781
1782 static void
1783 i915_gem_flush(struct drm_device *dev,
1784 uint32_t invalidate_domains,
1785 uint32_t flush_domains)
1786 {
1787 drm_i915_private_t *dev_priv = dev->dev_private;
1788 uint32_t cmd;
1789 RING_LOCALS;
1790
1791 #if WATCH_EXEC
1792 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1793 invalidate_domains, flush_domains);
1794 #endif
1795
1796 if (flush_domains & I915_GEM_DOMAIN_CPU)
1797 drm_agp_chipset_flush(dev);
1798
1799 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
1800 /*
1801 * read/write caches:
1802 *
1803 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1804 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1805 * also flushed at 2d versus 3d pipeline switches.
1806 *
1807 * read-only caches:
1808 *
1809 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1810 * MI_READ_FLUSH is set, and is always flushed on 965.
1811 *
1812 * I915_GEM_DOMAIN_COMMAND may not exist?
1813 *
1814 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1815 * invalidated when MI_EXE_FLUSH is set.
1816 *
1817 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1818 * invalidated with every MI_FLUSH.
1819 *
1820 * TLBs:
1821 *
1822 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1823 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1824 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1825 * are flushed at any MI_FLUSH.
1826 */
1827
1828 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1829 if ((invalidate_domains|flush_domains) &
1830 I915_GEM_DOMAIN_RENDER)
1831 cmd &= ~MI_NO_WRITE_FLUSH;
1832 if (!IS_I965G(dev)) {
1833 /*
1834 * On the 965, the sampler cache always gets flushed
1835 * and this bit is reserved.
1836 */
1837 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1838 cmd |= MI_READ_FLUSH;
1839 }
1840 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1841 cmd |= MI_EXE_FLUSH;
1842
1843 #if WATCH_EXEC
1844 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1845 #endif
1846 BEGIN_LP_RING(2);
1847 OUT_RING(cmd);
1848 OUT_RING(0); /* noop */
1849 ADVANCE_LP_RING();
1850 }
1851 }
1852
1853 /**
1854 * Ensures that all rendering to the object has completed and the object is
1855 * safe to unbind from the GTT or access from the CPU.
1856 */
1857 static int
1858 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1859 {
1860 struct drm_device *dev = obj->dev;
1861 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1862 int ret;
1863
1864 /* This function only exists to support waiting for existing rendering,
1865 * not for emitting required flushes.
1866 */
1867 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1868
1869 /* If there is rendering queued on the buffer being evicted, wait for
1870 * it.
1871 */
1872 if (obj_priv->active) {
1873 #if WATCH_BUF
1874 DRM_INFO("%s: object %p wait for seqno %08x\n",
1875 __func__, obj, obj_priv->last_rendering_seqno);
1876 #endif
1877 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1878 if (ret != 0)
1879 return ret;
1880 }
1881
1882 return 0;
1883 }
1884
1885 /**
1886 * Unbinds an object from the GTT aperture.
1887 */
1888 int
1889 i915_gem_object_unbind(struct drm_gem_object *obj)
1890 {
1891 struct drm_device *dev = obj->dev;
1892 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1893 int ret = 0;
1894
1895 #if WATCH_BUF
1896 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1897 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1898 #endif
1899 if (obj_priv->gtt_space == NULL)
1900 return 0;
1901
1902 if (obj_priv->pin_count != 0) {
1903 DRM_ERROR("Attempting to unbind pinned buffer\n");
1904 return -EINVAL;
1905 }
1906
1907 /* Move the object to the CPU domain to ensure that
1908 * any possible CPU writes while it's not in the GTT
1909 * are flushed when we go to remap it. This will
1910 * also ensure that all pending GPU writes are finished
1911 * before we unbind.
1912 */
1913 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1914 if (ret) {
1915 if (ret != -ERESTARTSYS)
1916 DRM_ERROR("set_domain failed: %d\n", ret);
1917 return ret;
1918 }
1919
1920 if (obj_priv->agp_mem != NULL) {
1921 drm_unbind_agp(obj_priv->agp_mem);
1922 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1923 obj_priv->agp_mem = NULL;
1924 }
1925
1926 BUG_ON(obj_priv->active);
1927
1928 /* blow away mappings if mapped through GTT */
1929 i915_gem_release_mmap(obj);
1930
1931 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1932 i915_gem_clear_fence_reg(obj);
1933
1934 i915_gem_object_put_pages(obj);
1935
1936 if (obj_priv->gtt_space) {
1937 atomic_dec(&dev->gtt_count);
1938 atomic_sub(obj->size, &dev->gtt_memory);
1939
1940 drm_mm_put_block(obj_priv->gtt_space);
1941 obj_priv->gtt_space = NULL;
1942 }
1943
1944 /* Remove ourselves from the LRU list if present. */
1945 if (!list_empty(&obj_priv->list))
1946 list_del_init(&obj_priv->list);
1947
1948 return 0;
1949 }
1950
1951 static int
1952 i915_gem_evict_something(struct drm_device *dev)
1953 {
1954 drm_i915_private_t *dev_priv = dev->dev_private;
1955 struct drm_gem_object *obj;
1956 struct drm_i915_gem_object *obj_priv;
1957 int ret = 0;
1958
1959 for (;;) {
1960 /* If there's an inactive buffer available now, grab it
1961 * and be done.
1962 */
1963 if (!list_empty(&dev_priv->mm.inactive_list)) {
1964 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
1965 struct drm_i915_gem_object,
1966 list);
1967 obj = obj_priv->obj;
1968 BUG_ON(obj_priv->pin_count != 0);
1969 #if WATCH_LRU
1970 DRM_INFO("%s: evicting %p\n", __func__, obj);
1971 #endif
1972 BUG_ON(obj_priv->active);
1973
1974 /* Wait on the rendering and unbind the buffer. */
1975 ret = i915_gem_object_unbind(obj);
1976 break;
1977 }
1978
1979 /* If we didn't get anything, but the ring is still processing
1980 * things, wait for one of those things to finish and hopefully
1981 * leave us a buffer to evict.
1982 */
1983 if (!list_empty(&dev_priv->mm.request_list)) {
1984 struct drm_i915_gem_request *request;
1985
1986 request = list_first_entry(&dev_priv->mm.request_list,
1987 struct drm_i915_gem_request,
1988 list);
1989
1990 ret = i915_wait_request(dev, request->seqno);
1991 if (ret)
1992 break;
1993
1994 /* if waiting caused an object to become inactive,
1995 * then loop around and wait for it. Otherwise, we
1996 * assume that waiting freed and unbound something,
1997 * so there should now be some space in the GTT
1998 */
1999 if (!list_empty(&dev_priv->mm.inactive_list))
2000 continue;
2001 break;
2002 }
2003
2004 /* If we didn't have anything on the request list but there
2005 * are buffers awaiting a flush, emit one and try again.
2006 * When we wait on it, those buffers waiting for that flush
2007 * will get moved to inactive.
2008 */
2009 if (!list_empty(&dev_priv->mm.flushing_list)) {
2010 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
2011 struct drm_i915_gem_object,
2012 list);
2013 obj = obj_priv->obj;
2014
2015 i915_gem_flush(dev,
2016 obj->write_domain,
2017 obj->write_domain);
2018 i915_add_request(dev, NULL, obj->write_domain);
2019
2020 obj = NULL;
2021 continue;
2022 }
2023
2024 DRM_ERROR("inactive empty %d request empty %d "
2025 "flushing empty %d\n",
2026 list_empty(&dev_priv->mm.inactive_list),
2027 list_empty(&dev_priv->mm.request_list),
2028 list_empty(&dev_priv->mm.flushing_list));
2029 /* If we didn't do any of the above, there's nothing to be done
2030 * and we just can't fit it in.
2031 */
2032 return -ENOSPC;
2033 }
2034 return ret;
2035 }
2036
2037 static int
2038 i915_gem_evict_everything(struct drm_device *dev)
2039 {
2040 int ret;
2041
2042 for (;;) {
2043 ret = i915_gem_evict_something(dev);
2044 if (ret != 0)
2045 break;
2046 }
2047 if (ret == -ENOSPC)
2048 return 0;
2049 return ret;
2050 }
2051
2052 int
2053 i915_gem_object_get_pages(struct drm_gem_object *obj)
2054 {
2055 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2056 int page_count, i;
2057 struct address_space *mapping;
2058 struct inode *inode;
2059 struct page *page;
2060 int ret;
2061
2062 if (obj_priv->pages_refcount++ != 0)
2063 return 0;
2064
2065 /* Get the list of pages out of our struct file. They'll be pinned
2066 * at this point until we release them.
2067 */
2068 page_count = obj->size / PAGE_SIZE;
2069 BUG_ON(obj_priv->pages != NULL);
2070 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2071 if (obj_priv->pages == NULL) {
2072 DRM_ERROR("Faled to allocate page list\n");
2073 obj_priv->pages_refcount--;
2074 return -ENOMEM;
2075 }
2076
2077 inode = obj->filp->f_path.dentry->d_inode;
2078 mapping = inode->i_mapping;
2079 for (i = 0; i < page_count; i++) {
2080 page = read_mapping_page(mapping, i, NULL);
2081 if (IS_ERR(page)) {
2082 ret = PTR_ERR(page);
2083 DRM_ERROR("read_mapping_page failed: %d\n", ret);
2084 i915_gem_object_put_pages(obj);
2085 return ret;
2086 }
2087 obj_priv->pages[i] = page;
2088 }
2089
2090 if (obj_priv->tiling_mode != I915_TILING_NONE)
2091 i915_gem_object_do_bit_17_swizzle(obj);
2092
2093 return 0;
2094 }
2095
2096 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2097 {
2098 struct drm_gem_object *obj = reg->obj;
2099 struct drm_device *dev = obj->dev;
2100 drm_i915_private_t *dev_priv = dev->dev_private;
2101 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2102 int regnum = obj_priv->fence_reg;
2103 uint64_t val;
2104
2105 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2106 0xfffff000) << 32;
2107 val |= obj_priv->gtt_offset & 0xfffff000;
2108 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2109 if (obj_priv->tiling_mode == I915_TILING_Y)
2110 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2111 val |= I965_FENCE_REG_VALID;
2112
2113 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2114 }
2115
2116 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2117 {
2118 struct drm_gem_object *obj = reg->obj;
2119 struct drm_device *dev = obj->dev;
2120 drm_i915_private_t *dev_priv = dev->dev_private;
2121 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2122 int regnum = obj_priv->fence_reg;
2123 int tile_width;
2124 uint32_t fence_reg, val;
2125 uint32_t pitch_val;
2126
2127 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2128 (obj_priv->gtt_offset & (obj->size - 1))) {
2129 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2130 __func__, obj_priv->gtt_offset, obj->size);
2131 return;
2132 }
2133
2134 if (obj_priv->tiling_mode == I915_TILING_Y &&
2135 HAS_128_BYTE_Y_TILING(dev))
2136 tile_width = 128;
2137 else
2138 tile_width = 512;
2139
2140 /* Note: pitch better be a power of two tile widths */
2141 pitch_val = obj_priv->stride / tile_width;
2142 pitch_val = ffs(pitch_val) - 1;
2143
2144 val = obj_priv->gtt_offset;
2145 if (obj_priv->tiling_mode == I915_TILING_Y)
2146 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2147 val |= I915_FENCE_SIZE_BITS(obj->size);
2148 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2149 val |= I830_FENCE_REG_VALID;
2150
2151 if (regnum < 8)
2152 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2153 else
2154 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2155 I915_WRITE(fence_reg, val);
2156 }
2157
2158 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2159 {
2160 struct drm_gem_object *obj = reg->obj;
2161 struct drm_device *dev = obj->dev;
2162 drm_i915_private_t *dev_priv = dev->dev_private;
2163 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2164 int regnum = obj_priv->fence_reg;
2165 uint32_t val;
2166 uint32_t pitch_val;
2167 uint32_t fence_size_bits;
2168
2169 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2170 (obj_priv->gtt_offset & (obj->size - 1))) {
2171 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2172 __func__, obj_priv->gtt_offset);
2173 return;
2174 }
2175
2176 pitch_val = obj_priv->stride / 128;
2177 pitch_val = ffs(pitch_val) - 1;
2178 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2179
2180 val = obj_priv->gtt_offset;
2181 if (obj_priv->tiling_mode == I915_TILING_Y)
2182 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2183 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2184 WARN_ON(fence_size_bits & ~0x00000f00);
2185 val |= fence_size_bits;
2186 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2187 val |= I830_FENCE_REG_VALID;
2188
2189 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2190 }
2191
2192 /**
2193 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2194 * @obj: object to map through a fence reg
2195 *
2196 * When mapping objects through the GTT, userspace wants to be able to write
2197 * to them without having to worry about swizzling if the object is tiled.
2198 *
2199 * This function walks the fence regs looking for a free one for @obj,
2200 * stealing one if it can't find any.
2201 *
2202 * It then sets up the reg based on the object's properties: address, pitch
2203 * and tiling format.
2204 */
2205 int
2206 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2207 {
2208 struct drm_device *dev = obj->dev;
2209 struct drm_i915_private *dev_priv = dev->dev_private;
2210 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2211 struct drm_i915_fence_reg *reg = NULL;
2212 struct drm_i915_gem_object *old_obj_priv = NULL;
2213 int i, ret, avail;
2214
2215 /* Just update our place in the LRU if our fence is getting used. */
2216 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2217 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2218 return 0;
2219 }
2220
2221 switch (obj_priv->tiling_mode) {
2222 case I915_TILING_NONE:
2223 WARN(1, "allocating a fence for non-tiled object?\n");
2224 break;
2225 case I915_TILING_X:
2226 if (!obj_priv->stride)
2227 return -EINVAL;
2228 WARN((obj_priv->stride & (512 - 1)),
2229 "object 0x%08x is X tiled but has non-512B pitch\n",
2230 obj_priv->gtt_offset);
2231 break;
2232 case I915_TILING_Y:
2233 if (!obj_priv->stride)
2234 return -EINVAL;
2235 WARN((obj_priv->stride & (128 - 1)),
2236 "object 0x%08x is Y tiled but has non-128B pitch\n",
2237 obj_priv->gtt_offset);
2238 break;
2239 }
2240
2241 /* First try to find a free reg */
2242 avail = 0;
2243 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2244 reg = &dev_priv->fence_regs[i];
2245 if (!reg->obj)
2246 break;
2247
2248 old_obj_priv = reg->obj->driver_private;
2249 if (!old_obj_priv->pin_count)
2250 avail++;
2251 }
2252
2253 /* None available, try to steal one or wait for a user to finish */
2254 if (i == dev_priv->num_fence_regs) {
2255 struct drm_gem_object *old_obj = NULL;
2256
2257 if (avail == 0)
2258 return -ENOSPC;
2259
2260 list_for_each_entry(old_obj_priv, &dev_priv->mm.fence_list,
2261 fence_list) {
2262 old_obj = old_obj_priv->obj;
2263
2264 if (old_obj_priv->pin_count)
2265 continue;
2266
2267 /* Take a reference, as otherwise the wait_rendering
2268 * below may cause the object to get freed out from
2269 * under us.
2270 */
2271 drm_gem_object_reference(old_obj);
2272
2273 /* i915 uses fences for GPU access to tiled buffers */
2274 if (IS_I965G(dev) || !old_obj_priv->active)
2275 break;
2276
2277 /* This brings the object to the head of the LRU if it
2278 * had been written to. The only way this should
2279 * result in us waiting longer than the expected
2280 * optimal amount of time is if there was a
2281 * fence-using buffer later that was read-only.
2282 */
2283 i915_gem_object_flush_gpu_write_domain(old_obj);
2284 ret = i915_gem_object_wait_rendering(old_obj);
2285 if (ret != 0) {
2286 drm_gem_object_unreference(old_obj);
2287 return ret;
2288 }
2289
2290 break;
2291 }
2292
2293 /*
2294 * Zap this virtual mapping so we can set up a fence again
2295 * for this object next time we need it.
2296 */
2297 i915_gem_release_mmap(old_obj);
2298
2299 i = old_obj_priv->fence_reg;
2300 reg = &dev_priv->fence_regs[i];
2301
2302 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
2303 list_del_init(&old_obj_priv->fence_list);
2304
2305 drm_gem_object_unreference(old_obj);
2306 }
2307
2308 obj_priv->fence_reg = i;
2309 list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2310
2311 reg->obj = obj;
2312
2313 if (IS_I965G(dev))
2314 i965_write_fence_reg(reg);
2315 else if (IS_I9XX(dev))
2316 i915_write_fence_reg(reg);
2317 else
2318 i830_write_fence_reg(reg);
2319
2320 return 0;
2321 }
2322
2323 /**
2324 * i915_gem_clear_fence_reg - clear out fence register info
2325 * @obj: object to clear
2326 *
2327 * Zeroes out the fence register itself and clears out the associated
2328 * data structures in dev_priv and obj_priv.
2329 */
2330 static void
2331 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2332 {
2333 struct drm_device *dev = obj->dev;
2334 drm_i915_private_t *dev_priv = dev->dev_private;
2335 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2336
2337 if (IS_I965G(dev))
2338 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2339 else {
2340 uint32_t fence_reg;
2341
2342 if (obj_priv->fence_reg < 8)
2343 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2344 else
2345 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2346 8) * 4;
2347
2348 I915_WRITE(fence_reg, 0);
2349 }
2350
2351 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2352 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2353 list_del_init(&obj_priv->fence_list);
2354 }
2355
2356 /**
2357 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2358 * to the buffer to finish, and then resets the fence register.
2359 * @obj: tiled object holding a fence register.
2360 *
2361 * Zeroes out the fence register itself and clears out the associated
2362 * data structures in dev_priv and obj_priv.
2363 */
2364 int
2365 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2366 {
2367 struct drm_device *dev = obj->dev;
2368 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2369
2370 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2371 return 0;
2372
2373 /* On the i915, GPU access to tiled buffers is via a fence,
2374 * therefore we must wait for any outstanding access to complete
2375 * before clearing the fence.
2376 */
2377 if (!IS_I965G(dev)) {
2378 int ret;
2379
2380 i915_gem_object_flush_gpu_write_domain(obj);
2381 i915_gem_object_flush_gtt_write_domain(obj);
2382 ret = i915_gem_object_wait_rendering(obj);
2383 if (ret != 0)
2384 return ret;
2385 }
2386
2387 i915_gem_clear_fence_reg (obj);
2388
2389 return 0;
2390 }
2391
2392 /**
2393 * Finds free space in the GTT aperture and binds the object there.
2394 */
2395 static int
2396 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2397 {
2398 struct drm_device *dev = obj->dev;
2399 drm_i915_private_t *dev_priv = dev->dev_private;
2400 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2401 struct drm_mm_node *free_space;
2402 int page_count, ret;
2403
2404 if (dev_priv->mm.suspended)
2405 return -EBUSY;
2406 if (alignment == 0)
2407 alignment = i915_gem_get_gtt_alignment(obj);
2408 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2409 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2410 return -EINVAL;
2411 }
2412
2413 search_free:
2414 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2415 obj->size, alignment, 0);
2416 if (free_space != NULL) {
2417 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2418 alignment);
2419 if (obj_priv->gtt_space != NULL) {
2420 obj_priv->gtt_space->private = obj;
2421 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2422 }
2423 }
2424 if (obj_priv->gtt_space == NULL) {
2425 bool lists_empty;
2426
2427 /* If the gtt is empty and we're still having trouble
2428 * fitting our object in, we're out of memory.
2429 */
2430 #if WATCH_LRU
2431 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2432 #endif
2433 spin_lock(&dev_priv->mm.active_list_lock);
2434 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2435 list_empty(&dev_priv->mm.flushing_list) &&
2436 list_empty(&dev_priv->mm.active_list));
2437 spin_unlock(&dev_priv->mm.active_list_lock);
2438 if (lists_empty) {
2439 DRM_ERROR("GTT full, but LRU list empty\n");
2440 return -ENOSPC;
2441 }
2442
2443 ret = i915_gem_evict_something(dev);
2444 if (ret != 0) {
2445 if (ret != -ERESTARTSYS)
2446 DRM_ERROR("Failed to evict a buffer %d\n", ret);
2447 return ret;
2448 }
2449 goto search_free;
2450 }
2451
2452 #if WATCH_BUF
2453 DRM_INFO("Binding object of size %zd at 0x%08x\n",
2454 obj->size, obj_priv->gtt_offset);
2455 #endif
2456 ret = i915_gem_object_get_pages(obj);
2457 if (ret) {
2458 drm_mm_put_block(obj_priv->gtt_space);
2459 obj_priv->gtt_space = NULL;
2460 return ret;
2461 }
2462
2463 page_count = obj->size / PAGE_SIZE;
2464 /* Create an AGP memory structure pointing at our pages, and bind it
2465 * into the GTT.
2466 */
2467 obj_priv->agp_mem = drm_agp_bind_pages(dev,
2468 obj_priv->pages,
2469 page_count,
2470 obj_priv->gtt_offset,
2471 obj_priv->agp_type);
2472 if (obj_priv->agp_mem == NULL) {
2473 i915_gem_object_put_pages(obj);
2474 drm_mm_put_block(obj_priv->gtt_space);
2475 obj_priv->gtt_space = NULL;
2476 return -ENOMEM;
2477 }
2478 atomic_inc(&dev->gtt_count);
2479 atomic_add(obj->size, &dev->gtt_memory);
2480
2481 /* Assert that the object is not currently in any GPU domain. As it
2482 * wasn't in the GTT, there shouldn't be any way it could have been in
2483 * a GPU cache
2484 */
2485 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2486 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2487
2488 return 0;
2489 }
2490
2491 void
2492 i915_gem_clflush_object(struct drm_gem_object *obj)
2493 {
2494 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2495
2496 /* If we don't have a page list set up, then we're not pinned
2497 * to GPU, and we can ignore the cache flush because it'll happen
2498 * again at bind time.
2499 */
2500 if (obj_priv->pages == NULL)
2501 return;
2502
2503 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2504 }
2505
2506 /** Flushes any GPU write domain for the object if it's dirty. */
2507 static void
2508 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2509 {
2510 struct drm_device *dev = obj->dev;
2511 uint32_t seqno;
2512
2513 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2514 return;
2515
2516 /* Queue the GPU write cache flushing we need. */
2517 i915_gem_flush(dev, 0, obj->write_domain);
2518 seqno = i915_add_request(dev, NULL, obj->write_domain);
2519 obj->write_domain = 0;
2520 i915_gem_object_move_to_active(obj, seqno);
2521 }
2522
2523 /** Flushes the GTT write domain for the object if it's dirty. */
2524 static void
2525 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2526 {
2527 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2528 return;
2529
2530 /* No actual flushing is required for the GTT write domain. Writes
2531 * to it immediately go to main memory as far as we know, so there's
2532 * no chipset flush. It also doesn't land in render cache.
2533 */
2534 obj->write_domain = 0;
2535 }
2536
2537 /** Flushes the CPU write domain for the object if it's dirty. */
2538 static void
2539 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2540 {
2541 struct drm_device *dev = obj->dev;
2542
2543 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2544 return;
2545
2546 i915_gem_clflush_object(obj);
2547 drm_agp_chipset_flush(dev);
2548 obj->write_domain = 0;
2549 }
2550
2551 /**
2552 * Moves a single object to the GTT read, and possibly write domain.
2553 *
2554 * This function returns when the move is complete, including waiting on
2555 * flushes to occur.
2556 */
2557 int
2558 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2559 {
2560 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2561 int ret;
2562
2563 /* Not valid to be called on unbound objects. */
2564 if (obj_priv->gtt_space == NULL)
2565 return -EINVAL;
2566
2567 i915_gem_object_flush_gpu_write_domain(obj);
2568 /* Wait on any GPU rendering and flushing to occur. */
2569 ret = i915_gem_object_wait_rendering(obj);
2570 if (ret != 0)
2571 return ret;
2572
2573 /* If we're writing through the GTT domain, then CPU and GPU caches
2574 * will need to be invalidated at next use.
2575 */
2576 if (write)
2577 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2578
2579 i915_gem_object_flush_cpu_write_domain(obj);
2580
2581 /* It should now be out of any other write domains, and we can update
2582 * the domain values for our changes.
2583 */
2584 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2585 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2586 if (write) {
2587 obj->write_domain = I915_GEM_DOMAIN_GTT;
2588 obj_priv->dirty = 1;
2589 }
2590
2591 return 0;
2592 }
2593
2594 /**
2595 * Moves a single object to the CPU read, and possibly write domain.
2596 *
2597 * This function returns when the move is complete, including waiting on
2598 * flushes to occur.
2599 */
2600 static int
2601 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2602 {
2603 int ret;
2604
2605 i915_gem_object_flush_gpu_write_domain(obj);
2606 /* Wait on any GPU rendering and flushing to occur. */
2607 ret = i915_gem_object_wait_rendering(obj);
2608 if (ret != 0)
2609 return ret;
2610
2611 i915_gem_object_flush_gtt_write_domain(obj);
2612
2613 /* If we have a partially-valid cache of the object in the CPU,
2614 * finish invalidating it and free the per-page flags.
2615 */
2616 i915_gem_object_set_to_full_cpu_read_domain(obj);
2617
2618 /* Flush the CPU cache if it's still invalid. */
2619 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2620 i915_gem_clflush_object(obj);
2621
2622 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2623 }
2624
2625 /* It should now be out of any other write domains, and we can update
2626 * the domain values for our changes.
2627 */
2628 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2629
2630 /* If we're writing through the CPU, then the GPU read domains will
2631 * need to be invalidated at next use.
2632 */
2633 if (write) {
2634 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2635 obj->write_domain = I915_GEM_DOMAIN_CPU;
2636 }
2637
2638 return 0;
2639 }
2640
2641 /*
2642 * Set the next domain for the specified object. This
2643 * may not actually perform the necessary flushing/invaliding though,
2644 * as that may want to be batched with other set_domain operations
2645 *
2646 * This is (we hope) the only really tricky part of gem. The goal
2647 * is fairly simple -- track which caches hold bits of the object
2648 * and make sure they remain coherent. A few concrete examples may
2649 * help to explain how it works. For shorthand, we use the notation
2650 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2651 * a pair of read and write domain masks.
2652 *
2653 * Case 1: the batch buffer
2654 *
2655 * 1. Allocated
2656 * 2. Written by CPU
2657 * 3. Mapped to GTT
2658 * 4. Read by GPU
2659 * 5. Unmapped from GTT
2660 * 6. Freed
2661 *
2662 * Let's take these a step at a time
2663 *
2664 * 1. Allocated
2665 * Pages allocated from the kernel may still have
2666 * cache contents, so we set them to (CPU, CPU) always.
2667 * 2. Written by CPU (using pwrite)
2668 * The pwrite function calls set_domain (CPU, CPU) and
2669 * this function does nothing (as nothing changes)
2670 * 3. Mapped by GTT
2671 * This function asserts that the object is not
2672 * currently in any GPU-based read or write domains
2673 * 4. Read by GPU
2674 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2675 * As write_domain is zero, this function adds in the
2676 * current read domains (CPU+COMMAND, 0).
2677 * flush_domains is set to CPU.
2678 * invalidate_domains is set to COMMAND
2679 * clflush is run to get data out of the CPU caches
2680 * then i915_dev_set_domain calls i915_gem_flush to
2681 * emit an MI_FLUSH and drm_agp_chipset_flush
2682 * 5. Unmapped from GTT
2683 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2684 * flush_domains and invalidate_domains end up both zero
2685 * so no flushing/invalidating happens
2686 * 6. Freed
2687 * yay, done
2688 *
2689 * Case 2: The shared render buffer
2690 *
2691 * 1. Allocated
2692 * 2. Mapped to GTT
2693 * 3. Read/written by GPU
2694 * 4. set_domain to (CPU,CPU)
2695 * 5. Read/written by CPU
2696 * 6. Read/written by GPU
2697 *
2698 * 1. Allocated
2699 * Same as last example, (CPU, CPU)
2700 * 2. Mapped to GTT
2701 * Nothing changes (assertions find that it is not in the GPU)
2702 * 3. Read/written by GPU
2703 * execbuffer calls set_domain (RENDER, RENDER)
2704 * flush_domains gets CPU
2705 * invalidate_domains gets GPU
2706 * clflush (obj)
2707 * MI_FLUSH and drm_agp_chipset_flush
2708 * 4. set_domain (CPU, CPU)
2709 * flush_domains gets GPU
2710 * invalidate_domains gets CPU
2711 * wait_rendering (obj) to make sure all drawing is complete.
2712 * This will include an MI_FLUSH to get the data from GPU
2713 * to memory
2714 * clflush (obj) to invalidate the CPU cache
2715 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2716 * 5. Read/written by CPU
2717 * cache lines are loaded and dirtied
2718 * 6. Read written by GPU
2719 * Same as last GPU access
2720 *
2721 * Case 3: The constant buffer
2722 *
2723 * 1. Allocated
2724 * 2. Written by CPU
2725 * 3. Read by GPU
2726 * 4. Updated (written) by CPU again
2727 * 5. Read by GPU
2728 *
2729 * 1. Allocated
2730 * (CPU, CPU)
2731 * 2. Written by CPU
2732 * (CPU, CPU)
2733 * 3. Read by GPU
2734 * (CPU+RENDER, 0)
2735 * flush_domains = CPU
2736 * invalidate_domains = RENDER
2737 * clflush (obj)
2738 * MI_FLUSH
2739 * drm_agp_chipset_flush
2740 * 4. Updated (written) by CPU again
2741 * (CPU, CPU)
2742 * flush_domains = 0 (no previous write domain)
2743 * invalidate_domains = 0 (no new read domains)
2744 * 5. Read by GPU
2745 * (CPU+RENDER, 0)
2746 * flush_domains = CPU
2747 * invalidate_domains = RENDER
2748 * clflush (obj)
2749 * MI_FLUSH
2750 * drm_agp_chipset_flush
2751 */
2752 static void
2753 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2754 {
2755 struct drm_device *dev = obj->dev;
2756 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2757 uint32_t invalidate_domains = 0;
2758 uint32_t flush_domains = 0;
2759
2760 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2761 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2762
2763 #if WATCH_BUF
2764 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2765 __func__, obj,
2766 obj->read_domains, obj->pending_read_domains,
2767 obj->write_domain, obj->pending_write_domain);
2768 #endif
2769 /*
2770 * If the object isn't moving to a new write domain,
2771 * let the object stay in multiple read domains
2772 */
2773 if (obj->pending_write_domain == 0)
2774 obj->pending_read_domains |= obj->read_domains;
2775 else
2776 obj_priv->dirty = 1;
2777
2778 /*
2779 * Flush the current write domain if
2780 * the new read domains don't match. Invalidate
2781 * any read domains which differ from the old
2782 * write domain
2783 */
2784 if (obj->write_domain &&
2785 obj->write_domain != obj->pending_read_domains) {
2786 flush_domains |= obj->write_domain;
2787 invalidate_domains |=
2788 obj->pending_read_domains & ~obj->write_domain;
2789 }
2790 /*
2791 * Invalidate any read caches which may have
2792 * stale data. That is, any new read domains.
2793 */
2794 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
2795 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
2796 #if WATCH_BUF
2797 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
2798 __func__, flush_domains, invalidate_domains);
2799 #endif
2800 i915_gem_clflush_object(obj);
2801 }
2802
2803 /* The actual obj->write_domain will be updated with
2804 * pending_write_domain after we emit the accumulated flush for all
2805 * of our domain changes in execbuffers (which clears objects'
2806 * write_domains). So if we have a current write domain that we
2807 * aren't changing, set pending_write_domain to that.
2808 */
2809 if (flush_domains == 0 && obj->pending_write_domain == 0)
2810 obj->pending_write_domain = obj->write_domain;
2811 obj->read_domains = obj->pending_read_domains;
2812
2813 dev->invalidate_domains |= invalidate_domains;
2814 dev->flush_domains |= flush_domains;
2815 #if WATCH_BUF
2816 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
2817 __func__,
2818 obj->read_domains, obj->write_domain,
2819 dev->invalidate_domains, dev->flush_domains);
2820 #endif
2821 }
2822
2823 /**
2824 * Moves the object from a partially CPU read to a full one.
2825 *
2826 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
2827 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
2828 */
2829 static void
2830 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
2831 {
2832 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2833
2834 if (!obj_priv->page_cpu_valid)
2835 return;
2836
2837 /* If we're partially in the CPU read domain, finish moving it in.
2838 */
2839 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
2840 int i;
2841
2842 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
2843 if (obj_priv->page_cpu_valid[i])
2844 continue;
2845 drm_clflush_pages(obj_priv->pages + i, 1);
2846 }
2847 }
2848
2849 /* Free the page_cpu_valid mappings which are now stale, whether
2850 * or not we've got I915_GEM_DOMAIN_CPU.
2851 */
2852 kfree(obj_priv->page_cpu_valid);
2853 obj_priv->page_cpu_valid = NULL;
2854 }
2855
2856 /**
2857 * Set the CPU read domain on a range of the object.
2858 *
2859 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
2860 * not entirely valid. The page_cpu_valid member of the object flags which
2861 * pages have been flushed, and will be respected by
2862 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
2863 * of the whole object.
2864 *
2865 * This function returns when the move is complete, including waiting on
2866 * flushes to occur.
2867 */
2868 static int
2869 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
2870 uint64_t offset, uint64_t size)
2871 {
2872 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2873 int i, ret;
2874
2875 if (offset == 0 && size == obj->size)
2876 return i915_gem_object_set_to_cpu_domain(obj, 0);
2877
2878 i915_gem_object_flush_gpu_write_domain(obj);
2879 /* Wait on any GPU rendering and flushing to occur. */
2880 ret = i915_gem_object_wait_rendering(obj);
2881 if (ret != 0)
2882 return ret;
2883 i915_gem_object_flush_gtt_write_domain(obj);
2884
2885 /* If we're already fully in the CPU read domain, we're done. */
2886 if (obj_priv->page_cpu_valid == NULL &&
2887 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
2888 return 0;
2889
2890 /* Otherwise, create/clear the per-page CPU read domain flag if we're
2891 * newly adding I915_GEM_DOMAIN_CPU
2892 */
2893 if (obj_priv->page_cpu_valid == NULL) {
2894 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
2895 GFP_KERNEL);
2896 if (obj_priv->page_cpu_valid == NULL)
2897 return -ENOMEM;
2898 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
2899 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
2900
2901 /* Flush the cache on any pages that are still invalid from the CPU's
2902 * perspective.
2903 */
2904 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
2905 i++) {
2906 if (obj_priv->page_cpu_valid[i])
2907 continue;
2908
2909 drm_clflush_pages(obj_priv->pages + i, 1);
2910
2911 obj_priv->page_cpu_valid[i] = 1;
2912 }
2913
2914 /* It should now be out of any other write domains, and we can update
2915 * the domain values for our changes.
2916 */
2917 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2918
2919 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2920
2921 return 0;
2922 }
2923
2924 /**
2925 * Pin an object to the GTT and evaluate the relocations landing in it.
2926 */
2927 static int
2928 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
2929 struct drm_file *file_priv,
2930 struct drm_i915_gem_exec_object *entry,
2931 struct drm_i915_gem_relocation_entry *relocs)
2932 {
2933 struct drm_device *dev = obj->dev;
2934 drm_i915_private_t *dev_priv = dev->dev_private;
2935 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2936 int i, ret;
2937 void __iomem *reloc_page;
2938
2939 /* Choose the GTT offset for our buffer and put it there. */
2940 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
2941 if (ret)
2942 return ret;
2943
2944 entry->offset = obj_priv->gtt_offset;
2945
2946 /* Apply the relocations, using the GTT aperture to avoid cache
2947 * flushing requirements.
2948 */
2949 for (i = 0; i < entry->relocation_count; i++) {
2950 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
2951 struct drm_gem_object *target_obj;
2952 struct drm_i915_gem_object *target_obj_priv;
2953 uint32_t reloc_val, reloc_offset;
2954 uint32_t __iomem *reloc_entry;
2955
2956 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
2957 reloc->target_handle);
2958 if (target_obj == NULL) {
2959 i915_gem_object_unpin(obj);
2960 return -EBADF;
2961 }
2962 target_obj_priv = target_obj->driver_private;
2963
2964 /* The target buffer should have appeared before us in the
2965 * exec_object list, so it should have a GTT space bound by now.
2966 */
2967 if (target_obj_priv->gtt_space == NULL) {
2968 DRM_ERROR("No GTT space found for object %d\n",
2969 reloc->target_handle);
2970 drm_gem_object_unreference(target_obj);
2971 i915_gem_object_unpin(obj);
2972 return -EINVAL;
2973 }
2974
2975 if (reloc->offset > obj->size - 4) {
2976 DRM_ERROR("Relocation beyond object bounds: "
2977 "obj %p target %d offset %d size %d.\n",
2978 obj, reloc->target_handle,
2979 (int) reloc->offset, (int) obj->size);
2980 drm_gem_object_unreference(target_obj);
2981 i915_gem_object_unpin(obj);
2982 return -EINVAL;
2983 }
2984 if (reloc->offset & 3) {
2985 DRM_ERROR("Relocation not 4-byte aligned: "
2986 "obj %p target %d offset %d.\n",
2987 obj, reloc->target_handle,
2988 (int) reloc->offset);
2989 drm_gem_object_unreference(target_obj);
2990 i915_gem_object_unpin(obj);
2991 return -EINVAL;
2992 }
2993
2994 if (reloc->delta >= target_obj->size) {
2995 DRM_ERROR("Relocation beyond target object bounds: "
2996 "obj %p target %d delta %d size %d.\n",
2997 obj, reloc->target_handle,
2998 (int) reloc->delta, (int) target_obj->size);
2999 drm_gem_object_unreference(target_obj);
3000 i915_gem_object_unpin(obj);
3001 return -EINVAL;
3002 }
3003
3004 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3005 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3006 DRM_ERROR("reloc with read/write CPU domains: "
3007 "obj %p target %d offset %d "
3008 "read %08x write %08x",
3009 obj, reloc->target_handle,
3010 (int) reloc->offset,
3011 reloc->read_domains,
3012 reloc->write_domain);
3013 drm_gem_object_unreference(target_obj);
3014 i915_gem_object_unpin(obj);
3015 return -EINVAL;
3016 }
3017
3018 if (reloc->write_domain && target_obj->pending_write_domain &&
3019 reloc->write_domain != target_obj->pending_write_domain) {
3020 DRM_ERROR("Write domain conflict: "
3021 "obj %p target %d offset %d "
3022 "new %08x old %08x\n",
3023 obj, reloc->target_handle,
3024 (int) reloc->offset,
3025 reloc->write_domain,
3026 target_obj->pending_write_domain);
3027 drm_gem_object_unreference(target_obj);
3028 i915_gem_object_unpin(obj);
3029 return -EINVAL;
3030 }
3031
3032 #if WATCH_RELOC
3033 DRM_INFO("%s: obj %p offset %08x target %d "
3034 "read %08x write %08x gtt %08x "
3035 "presumed %08x delta %08x\n",
3036 __func__,
3037 obj,
3038 (int) reloc->offset,
3039 (int) reloc->target_handle,
3040 (int) reloc->read_domains,
3041 (int) reloc->write_domain,
3042 (int) target_obj_priv->gtt_offset,
3043 (int) reloc->presumed_offset,
3044 reloc->delta);
3045 #endif
3046
3047 target_obj->pending_read_domains |= reloc->read_domains;
3048 target_obj->pending_write_domain |= reloc->write_domain;
3049
3050 /* If the relocation already has the right value in it, no
3051 * more work needs to be done.
3052 */
3053 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3054 drm_gem_object_unreference(target_obj);
3055 continue;
3056 }
3057
3058 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3059 if (ret != 0) {
3060 drm_gem_object_unreference(target_obj);
3061 i915_gem_object_unpin(obj);
3062 return -EINVAL;
3063 }
3064
3065 /* Map the page containing the relocation we're going to
3066 * perform.
3067 */
3068 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3069 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3070 (reloc_offset &
3071 ~(PAGE_SIZE - 1)));
3072 reloc_entry = (uint32_t __iomem *)(reloc_page +
3073 (reloc_offset & (PAGE_SIZE - 1)));
3074 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3075
3076 #if WATCH_BUF
3077 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3078 obj, (unsigned int) reloc->offset,
3079 readl(reloc_entry), reloc_val);
3080 #endif
3081 writel(reloc_val, reloc_entry);
3082 io_mapping_unmap_atomic(reloc_page);
3083
3084 /* The updated presumed offset for this entry will be
3085 * copied back out to the user.
3086 */
3087 reloc->presumed_offset = target_obj_priv->gtt_offset;
3088
3089 drm_gem_object_unreference(target_obj);
3090 }
3091
3092 #if WATCH_BUF
3093 if (0)
3094 i915_gem_dump_object(obj, 128, __func__, ~0);
3095 #endif
3096 return 0;
3097 }
3098
3099 /** Dispatch a batchbuffer to the ring
3100 */
3101 static int
3102 i915_dispatch_gem_execbuffer(struct drm_device *dev,
3103 struct drm_i915_gem_execbuffer *exec,
3104 struct drm_clip_rect *cliprects,
3105 uint64_t exec_offset)
3106 {
3107 drm_i915_private_t *dev_priv = dev->dev_private;
3108 int nbox = exec->num_cliprects;
3109 int i = 0, count;
3110 uint32_t exec_start, exec_len;
3111 RING_LOCALS;
3112
3113 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3114 exec_len = (uint32_t) exec->batch_len;
3115
3116 count = nbox ? nbox : 1;
3117
3118 for (i = 0; i < count; i++) {
3119 if (i < nbox) {
3120 int ret = i915_emit_box(dev, cliprects, i,
3121 exec->DR1, exec->DR4);
3122 if (ret)
3123 return ret;
3124 }
3125
3126 if (IS_I830(dev) || IS_845G(dev)) {
3127 BEGIN_LP_RING(4);
3128 OUT_RING(MI_BATCH_BUFFER);
3129 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3130 OUT_RING(exec_start + exec_len - 4);
3131 OUT_RING(0);
3132 ADVANCE_LP_RING();
3133 } else {
3134 BEGIN_LP_RING(2);
3135 if (IS_I965G(dev)) {
3136 OUT_RING(MI_BATCH_BUFFER_START |
3137 (2 << 6) |
3138 MI_BATCH_NON_SECURE_I965);
3139 OUT_RING(exec_start);
3140 } else {
3141 OUT_RING(MI_BATCH_BUFFER_START |
3142 (2 << 6));
3143 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3144 }
3145 ADVANCE_LP_RING();
3146 }
3147 }
3148
3149 /* XXX breadcrumb */
3150 return 0;
3151 }
3152
3153 /* Throttle our rendering by waiting until the ring has completed our requests
3154 * emitted over 20 msec ago.
3155 *
3156 * Note that if we were to use the current jiffies each time around the loop,
3157 * we wouldn't escape the function with any frames outstanding if the time to
3158 * render a frame was over 20ms.
3159 *
3160 * This should get us reasonable parallelism between CPU and GPU but also
3161 * relatively low latency when blocking on a particular request to finish.
3162 */
3163 static int
3164 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3165 {
3166 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3167 int ret = 0;
3168 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3169
3170 mutex_lock(&dev->struct_mutex);
3171 while (!list_empty(&i915_file_priv->mm.request_list)) {
3172 struct drm_i915_gem_request *request;
3173
3174 request = list_first_entry(&i915_file_priv->mm.request_list,
3175 struct drm_i915_gem_request,
3176 client_list);
3177
3178 if (time_after_eq(request->emitted_jiffies, recent_enough))
3179 break;
3180
3181 ret = i915_wait_request(dev, request->seqno);
3182 if (ret != 0)
3183 break;
3184 }
3185 mutex_unlock(&dev->struct_mutex);
3186
3187 return ret;
3188 }
3189
3190 static int
3191 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list,
3192 uint32_t buffer_count,
3193 struct drm_i915_gem_relocation_entry **relocs)
3194 {
3195 uint32_t reloc_count = 0, reloc_index = 0, i;
3196 int ret;
3197
3198 *relocs = NULL;
3199 for (i = 0; i < buffer_count; i++) {
3200 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3201 return -EINVAL;
3202 reloc_count += exec_list[i].relocation_count;
3203 }
3204
3205 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3206 if (*relocs == NULL)
3207 return -ENOMEM;
3208
3209 for (i = 0; i < buffer_count; i++) {
3210 struct drm_i915_gem_relocation_entry __user *user_relocs;
3211
3212 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3213
3214 ret = copy_from_user(&(*relocs)[reloc_index],
3215 user_relocs,
3216 exec_list[i].relocation_count *
3217 sizeof(**relocs));
3218 if (ret != 0) {
3219 drm_free_large(*relocs);
3220 *relocs = NULL;
3221 return -EFAULT;
3222 }
3223
3224 reloc_index += exec_list[i].relocation_count;
3225 }
3226
3227 return 0;
3228 }
3229
3230 static int
3231 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list,
3232 uint32_t buffer_count,
3233 struct drm_i915_gem_relocation_entry *relocs)
3234 {
3235 uint32_t reloc_count = 0, i;
3236 int ret = 0;
3237
3238 for (i = 0; i < buffer_count; i++) {
3239 struct drm_i915_gem_relocation_entry __user *user_relocs;
3240 int unwritten;
3241
3242 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3243
3244 unwritten = copy_to_user(user_relocs,
3245 &relocs[reloc_count],
3246 exec_list[i].relocation_count *
3247 sizeof(*relocs));
3248
3249 if (unwritten) {
3250 ret = -EFAULT;
3251 goto err;
3252 }
3253
3254 reloc_count += exec_list[i].relocation_count;
3255 }
3256
3257 err:
3258 drm_free_large(relocs);
3259
3260 return ret;
3261 }
3262
3263 static int
3264 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer *exec,
3265 uint64_t exec_offset)
3266 {
3267 uint32_t exec_start, exec_len;
3268
3269 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3270 exec_len = (uint32_t) exec->batch_len;
3271
3272 if ((exec_start | exec_len) & 0x7)
3273 return -EINVAL;
3274
3275 if (!exec_start)
3276 return -EINVAL;
3277
3278 return 0;
3279 }
3280
3281 int
3282 i915_gem_execbuffer(struct drm_device *dev, void *data,
3283 struct drm_file *file_priv)
3284 {
3285 drm_i915_private_t *dev_priv = dev->dev_private;
3286 struct drm_i915_gem_execbuffer *args = data;
3287 struct drm_i915_gem_exec_object *exec_list = NULL;
3288 struct drm_gem_object **object_list = NULL;
3289 struct drm_gem_object *batch_obj;
3290 struct drm_i915_gem_object *obj_priv;
3291 struct drm_clip_rect *cliprects = NULL;
3292 struct drm_i915_gem_relocation_entry *relocs;
3293 int ret, ret2, i, pinned = 0;
3294 uint64_t exec_offset;
3295 uint32_t seqno, flush_domains, reloc_index;
3296 int pin_tries;
3297
3298 #if WATCH_EXEC
3299 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3300 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3301 #endif
3302
3303 if (args->buffer_count < 1) {
3304 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3305 return -EINVAL;
3306 }
3307 /* Copy in the exec list from userland */
3308 exec_list = drm_calloc_large(sizeof(*exec_list), args->buffer_count);
3309 object_list = drm_calloc_large(sizeof(*object_list), args->buffer_count);
3310 if (exec_list == NULL || object_list == NULL) {
3311 DRM_ERROR("Failed to allocate exec or object list "
3312 "for %d buffers\n",
3313 args->buffer_count);
3314 ret = -ENOMEM;
3315 goto pre_mutex_err;
3316 }
3317 ret = copy_from_user(exec_list,
3318 (struct drm_i915_relocation_entry __user *)
3319 (uintptr_t) args->buffers_ptr,
3320 sizeof(*exec_list) * args->buffer_count);
3321 if (ret != 0) {
3322 DRM_ERROR("copy %d exec entries failed %d\n",
3323 args->buffer_count, ret);
3324 goto pre_mutex_err;
3325 }
3326
3327 if (args->num_cliprects != 0) {
3328 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3329 GFP_KERNEL);
3330 if (cliprects == NULL)
3331 goto pre_mutex_err;
3332
3333 ret = copy_from_user(cliprects,
3334 (struct drm_clip_rect __user *)
3335 (uintptr_t) args->cliprects_ptr,
3336 sizeof(*cliprects) * args->num_cliprects);
3337 if (ret != 0) {
3338 DRM_ERROR("copy %d cliprects failed: %d\n",
3339 args->num_cliprects, ret);
3340 goto pre_mutex_err;
3341 }
3342 }
3343
3344 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3345 &relocs);
3346 if (ret != 0)
3347 goto pre_mutex_err;
3348
3349 mutex_lock(&dev->struct_mutex);
3350
3351 i915_verify_inactive(dev, __FILE__, __LINE__);
3352
3353 if (dev_priv->mm.wedged) {
3354 DRM_ERROR("Execbuf while wedged\n");
3355 mutex_unlock(&dev->struct_mutex);
3356 ret = -EIO;
3357 goto pre_mutex_err;
3358 }
3359
3360 if (dev_priv->mm.suspended) {
3361 DRM_ERROR("Execbuf while VT-switched.\n");
3362 mutex_unlock(&dev->struct_mutex);
3363 ret = -EBUSY;
3364 goto pre_mutex_err;
3365 }
3366
3367 /* Look up object handles */
3368 for (i = 0; i < args->buffer_count; i++) {
3369 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3370 exec_list[i].handle);
3371 if (object_list[i] == NULL) {
3372 DRM_ERROR("Invalid object handle %d at index %d\n",
3373 exec_list[i].handle, i);
3374 ret = -EBADF;
3375 goto err;
3376 }
3377
3378 obj_priv = object_list[i]->driver_private;
3379 if (obj_priv->in_execbuffer) {
3380 DRM_ERROR("Object %p appears more than once in object list\n",
3381 object_list[i]);
3382 ret = -EBADF;
3383 goto err;
3384 }
3385 obj_priv->in_execbuffer = true;
3386 }
3387
3388 /* Pin and relocate */
3389 for (pin_tries = 0; ; pin_tries++) {
3390 ret = 0;
3391 reloc_index = 0;
3392
3393 for (i = 0; i < args->buffer_count; i++) {
3394 object_list[i]->pending_read_domains = 0;
3395 object_list[i]->pending_write_domain = 0;
3396 ret = i915_gem_object_pin_and_relocate(object_list[i],
3397 file_priv,
3398 &exec_list[i],
3399 &relocs[reloc_index]);
3400 if (ret)
3401 break;
3402 pinned = i + 1;
3403 reloc_index += exec_list[i].relocation_count;
3404 }
3405 /* success */
3406 if (ret == 0)
3407 break;
3408
3409 /* error other than GTT full, or we've already tried again */
3410 if (ret != -ENOSPC || pin_tries >= 1) {
3411 if (ret != -ERESTARTSYS)
3412 DRM_ERROR("Failed to pin buffers %d\n", ret);
3413 goto err;
3414 }
3415
3416 /* unpin all of our buffers */
3417 for (i = 0; i < pinned; i++)
3418 i915_gem_object_unpin(object_list[i]);
3419 pinned = 0;
3420
3421 /* evict everyone we can from the aperture */
3422 ret = i915_gem_evict_everything(dev);
3423 if (ret)
3424 goto err;
3425 }
3426
3427 /* Set the pending read domains for the batch buffer to COMMAND */
3428 batch_obj = object_list[args->buffer_count-1];
3429 if (batch_obj->pending_write_domain) {
3430 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3431 ret = -EINVAL;
3432 goto err;
3433 }
3434 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3435
3436 /* Sanity check the batch buffer, prior to moving objects */
3437 exec_offset = exec_list[args->buffer_count - 1].offset;
3438 ret = i915_gem_check_execbuffer (args, exec_offset);
3439 if (ret != 0) {
3440 DRM_ERROR("execbuf with invalid offset/length\n");
3441 goto err;
3442 }
3443
3444 i915_verify_inactive(dev, __FILE__, __LINE__);
3445
3446 /* Zero the global flush/invalidate flags. These
3447 * will be modified as new domains are computed
3448 * for each object
3449 */
3450 dev->invalidate_domains = 0;
3451 dev->flush_domains = 0;
3452
3453 for (i = 0; i < args->buffer_count; i++) {
3454 struct drm_gem_object *obj = object_list[i];
3455
3456 /* Compute new gpu domains and update invalidate/flush */
3457 i915_gem_object_set_to_gpu_domain(obj);
3458 }
3459
3460 i915_verify_inactive(dev, __FILE__, __LINE__);
3461
3462 if (dev->invalidate_domains | dev->flush_domains) {
3463 #if WATCH_EXEC
3464 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3465 __func__,
3466 dev->invalidate_domains,
3467 dev->flush_domains);
3468 #endif
3469 i915_gem_flush(dev,
3470 dev->invalidate_domains,
3471 dev->flush_domains);
3472 if (dev->flush_domains)
3473 (void)i915_add_request(dev, file_priv,
3474 dev->flush_domains);
3475 }
3476
3477 for (i = 0; i < args->buffer_count; i++) {
3478 struct drm_gem_object *obj = object_list[i];
3479
3480 obj->write_domain = obj->pending_write_domain;
3481 }
3482
3483 i915_verify_inactive(dev, __FILE__, __LINE__);
3484
3485 #if WATCH_COHERENCY
3486 for (i = 0; i < args->buffer_count; i++) {
3487 i915_gem_object_check_coherency(object_list[i],
3488 exec_list[i].handle);
3489 }
3490 #endif
3491
3492 #if WATCH_EXEC
3493 i915_gem_dump_object(batch_obj,
3494 args->batch_len,
3495 __func__,
3496 ~0);
3497 #endif
3498
3499 /* Exec the batchbuffer */
3500 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
3501 if (ret) {
3502 DRM_ERROR("dispatch failed %d\n", ret);
3503 goto err;
3504 }
3505
3506 /*
3507 * Ensure that the commands in the batch buffer are
3508 * finished before the interrupt fires
3509 */
3510 flush_domains = i915_retire_commands(dev);
3511
3512 i915_verify_inactive(dev, __FILE__, __LINE__);
3513
3514 /*
3515 * Get a seqno representing the execution of the current buffer,
3516 * which we can wait on. We would like to mitigate these interrupts,
3517 * likely by only creating seqnos occasionally (so that we have
3518 * *some* interrupts representing completion of buffers that we can
3519 * wait on when trying to clear up gtt space).
3520 */
3521 seqno = i915_add_request(dev, file_priv, flush_domains);
3522 BUG_ON(seqno == 0);
3523 for (i = 0; i < args->buffer_count; i++) {
3524 struct drm_gem_object *obj = object_list[i];
3525
3526 i915_gem_object_move_to_active(obj, seqno);
3527 #if WATCH_LRU
3528 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3529 #endif
3530 }
3531 #if WATCH_LRU
3532 i915_dump_lru(dev, __func__);
3533 #endif
3534
3535 i915_verify_inactive(dev, __FILE__, __LINE__);
3536
3537 err:
3538 for (i = 0; i < pinned; i++)
3539 i915_gem_object_unpin(object_list[i]);
3540
3541 for (i = 0; i < args->buffer_count; i++) {
3542 if (object_list[i]) {
3543 obj_priv = object_list[i]->driver_private;
3544 obj_priv->in_execbuffer = false;
3545 }
3546 drm_gem_object_unreference(object_list[i]);
3547 }
3548
3549 mutex_unlock(&dev->struct_mutex);
3550
3551 if (!ret) {
3552 /* Copy the new buffer offsets back to the user's exec list. */
3553 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3554 (uintptr_t) args->buffers_ptr,
3555 exec_list,
3556 sizeof(*exec_list) * args->buffer_count);
3557 if (ret) {
3558 ret = -EFAULT;
3559 DRM_ERROR("failed to copy %d exec entries "
3560 "back to user (%d)\n",
3561 args->buffer_count, ret);
3562 }
3563 }
3564
3565 /* Copy the updated relocations out regardless of current error
3566 * state. Failure to update the relocs would mean that the next
3567 * time userland calls execbuf, it would do so with presumed offset
3568 * state that didn't match the actual object state.
3569 */
3570 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3571 relocs);
3572 if (ret2 != 0) {
3573 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3574
3575 if (ret == 0)
3576 ret = ret2;
3577 }
3578
3579 pre_mutex_err:
3580 drm_free_large(object_list);
3581 drm_free_large(exec_list);
3582 kfree(cliprects);
3583
3584 return ret;
3585 }
3586
3587 int
3588 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
3589 {
3590 struct drm_device *dev = obj->dev;
3591 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3592 int ret;
3593
3594 i915_verify_inactive(dev, __FILE__, __LINE__);
3595 if (obj_priv->gtt_space == NULL) {
3596 ret = i915_gem_object_bind_to_gtt(obj, alignment);
3597 if (ret != 0) {
3598 if (ret != -EBUSY && ret != -ERESTARTSYS)
3599 DRM_ERROR("Failure to bind: %d\n", ret);
3600 return ret;
3601 }
3602 }
3603 /*
3604 * Pre-965 chips need a fence register set up in order to
3605 * properly handle tiled surfaces.
3606 */
3607 if (!IS_I965G(dev) && obj_priv->tiling_mode != I915_TILING_NONE) {
3608 ret = i915_gem_object_get_fence_reg(obj);
3609 if (ret != 0) {
3610 if (ret != -EBUSY && ret != -ERESTARTSYS)
3611 DRM_ERROR("Failure to install fence: %d\n",
3612 ret);
3613 return ret;
3614 }
3615 }
3616 obj_priv->pin_count++;
3617
3618 /* If the object is not active and not pending a flush,
3619 * remove it from the inactive list
3620 */
3621 if (obj_priv->pin_count == 1) {
3622 atomic_inc(&dev->pin_count);
3623 atomic_add(obj->size, &dev->pin_memory);
3624 if (!obj_priv->active &&
3625 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
3626 !list_empty(&obj_priv->list))
3627 list_del_init(&obj_priv->list);
3628 }
3629 i915_verify_inactive(dev, __FILE__, __LINE__);
3630
3631 return 0;
3632 }
3633
3634 void
3635 i915_gem_object_unpin(struct drm_gem_object *obj)
3636 {
3637 struct drm_device *dev = obj->dev;
3638 drm_i915_private_t *dev_priv = dev->dev_private;
3639 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3640
3641 i915_verify_inactive(dev, __FILE__, __LINE__);
3642 obj_priv->pin_count--;
3643 BUG_ON(obj_priv->pin_count < 0);
3644 BUG_ON(obj_priv->gtt_space == NULL);
3645
3646 /* If the object is no longer pinned, and is
3647 * neither active nor being flushed, then stick it on
3648 * the inactive list
3649 */
3650 if (obj_priv->pin_count == 0) {
3651 if (!obj_priv->active &&
3652 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
3653 list_move_tail(&obj_priv->list,
3654 &dev_priv->mm.inactive_list);
3655 atomic_dec(&dev->pin_count);
3656 atomic_sub(obj->size, &dev->pin_memory);
3657 }
3658 i915_verify_inactive(dev, __FILE__, __LINE__);
3659 }
3660
3661 int
3662 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3663 struct drm_file *file_priv)
3664 {
3665 struct drm_i915_gem_pin *args = data;
3666 struct drm_gem_object *obj;
3667 struct drm_i915_gem_object *obj_priv;
3668 int ret;
3669
3670 mutex_lock(&dev->struct_mutex);
3671
3672 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3673 if (obj == NULL) {
3674 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
3675 args->handle);
3676 mutex_unlock(&dev->struct_mutex);
3677 return -EBADF;
3678 }
3679 obj_priv = obj->driver_private;
3680
3681 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
3682 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3683 args->handle);
3684 drm_gem_object_unreference(obj);
3685 mutex_unlock(&dev->struct_mutex);
3686 return -EINVAL;
3687 }
3688
3689 obj_priv->user_pin_count++;
3690 obj_priv->pin_filp = file_priv;
3691 if (obj_priv->user_pin_count == 1) {
3692 ret = i915_gem_object_pin(obj, args->alignment);
3693 if (ret != 0) {
3694 drm_gem_object_unreference(obj);
3695 mutex_unlock(&dev->struct_mutex);
3696 return ret;
3697 }
3698 }
3699
3700 /* XXX - flush the CPU caches for pinned objects
3701 * as the X server doesn't manage domains yet
3702 */
3703 i915_gem_object_flush_cpu_write_domain(obj);
3704 args->offset = obj_priv->gtt_offset;
3705 drm_gem_object_unreference(obj);
3706 mutex_unlock(&dev->struct_mutex);
3707
3708 return 0;
3709 }
3710
3711 int
3712 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3713 struct drm_file *file_priv)
3714 {
3715 struct drm_i915_gem_pin *args = data;
3716 struct drm_gem_object *obj;
3717 struct drm_i915_gem_object *obj_priv;
3718
3719 mutex_lock(&dev->struct_mutex);
3720
3721 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3722 if (obj == NULL) {
3723 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
3724 args->handle);
3725 mutex_unlock(&dev->struct_mutex);
3726 return -EBADF;
3727 }
3728
3729 obj_priv = obj->driver_private;
3730 if (obj_priv->pin_filp != file_priv) {
3731 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3732 args->handle);
3733 drm_gem_object_unreference(obj);
3734 mutex_unlock(&dev->struct_mutex);
3735 return -EINVAL;
3736 }
3737 obj_priv->user_pin_count--;
3738 if (obj_priv->user_pin_count == 0) {
3739 obj_priv->pin_filp = NULL;
3740 i915_gem_object_unpin(obj);
3741 }
3742
3743 drm_gem_object_unreference(obj);
3744 mutex_unlock(&dev->struct_mutex);
3745 return 0;
3746 }
3747
3748 int
3749 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3750 struct drm_file *file_priv)
3751 {
3752 struct drm_i915_gem_busy *args = data;
3753 struct drm_gem_object *obj;
3754 struct drm_i915_gem_object *obj_priv;
3755
3756 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3757 if (obj == NULL) {
3758 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
3759 args->handle);
3760 return -EBADF;
3761 }
3762
3763 mutex_lock(&dev->struct_mutex);
3764 /* Update the active list for the hardware's current position.
3765 * Otherwise this only updates on a delayed timer or when irqs are
3766 * actually unmasked, and our working set ends up being larger than
3767 * required.
3768 */
3769 i915_gem_retire_requests(dev);
3770
3771 obj_priv = obj->driver_private;
3772 /* Don't count being on the flushing list against the object being
3773 * done. Otherwise, a buffer left on the flushing list but not getting
3774 * flushed (because nobody's flushing that domain) won't ever return
3775 * unbusy and get reused by libdrm's bo cache. The other expected
3776 * consumer of this interface, OpenGL's occlusion queries, also specs
3777 * that the objects get unbusy "eventually" without any interference.
3778 */
3779 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
3780
3781 drm_gem_object_unreference(obj);
3782 mutex_unlock(&dev->struct_mutex);
3783 return 0;
3784 }
3785
3786 int
3787 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3788 struct drm_file *file_priv)
3789 {
3790 return i915_gem_ring_throttle(dev, file_priv);
3791 }
3792
3793 int i915_gem_init_object(struct drm_gem_object *obj)
3794 {
3795 struct drm_i915_gem_object *obj_priv;
3796
3797 obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
3798 if (obj_priv == NULL)
3799 return -ENOMEM;
3800
3801 /*
3802 * We've just allocated pages from the kernel,
3803 * so they've just been written by the CPU with
3804 * zeros. They'll need to be clflushed before we
3805 * use them with the GPU.
3806 */
3807 obj->write_domain = I915_GEM_DOMAIN_CPU;
3808 obj->read_domains = I915_GEM_DOMAIN_CPU;
3809
3810 obj_priv->agp_type = AGP_USER_MEMORY;
3811
3812 obj->driver_private = obj_priv;
3813 obj_priv->obj = obj;
3814 obj_priv->fence_reg = I915_FENCE_REG_NONE;
3815 INIT_LIST_HEAD(&obj_priv->list);
3816 INIT_LIST_HEAD(&obj_priv->fence_list);
3817
3818 return 0;
3819 }
3820
3821 void i915_gem_free_object(struct drm_gem_object *obj)
3822 {
3823 struct drm_device *dev = obj->dev;
3824 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3825
3826 while (obj_priv->pin_count > 0)
3827 i915_gem_object_unpin(obj);
3828
3829 if (obj_priv->phys_obj)
3830 i915_gem_detach_phys_object(dev, obj);
3831
3832 i915_gem_object_unbind(obj);
3833
3834 if (obj_priv->mmap_offset)
3835 i915_gem_free_mmap_offset(obj);
3836
3837 kfree(obj_priv->page_cpu_valid);
3838 kfree(obj_priv->bit_17);
3839 kfree(obj->driver_private);
3840 }
3841
3842 /** Unbinds all objects that are on the given buffer list. */
3843 static int
3844 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
3845 {
3846 struct drm_gem_object *obj;
3847 struct drm_i915_gem_object *obj_priv;
3848 int ret;
3849
3850 while (!list_empty(head)) {
3851 obj_priv = list_first_entry(head,
3852 struct drm_i915_gem_object,
3853 list);
3854 obj = obj_priv->obj;
3855
3856 if (obj_priv->pin_count != 0) {
3857 DRM_ERROR("Pinned object in unbind list\n");
3858 mutex_unlock(&dev->struct_mutex);
3859 return -EINVAL;
3860 }
3861
3862 ret = i915_gem_object_unbind(obj);
3863 if (ret != 0) {
3864 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
3865 ret);
3866 mutex_unlock(&dev->struct_mutex);
3867 return ret;
3868 }
3869 }
3870
3871
3872 return 0;
3873 }
3874
3875 int
3876 i915_gem_idle(struct drm_device *dev)
3877 {
3878 drm_i915_private_t *dev_priv = dev->dev_private;
3879 uint32_t seqno, cur_seqno, last_seqno;
3880 int stuck, ret;
3881
3882 mutex_lock(&dev->struct_mutex);
3883
3884 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
3885 mutex_unlock(&dev->struct_mutex);
3886 return 0;
3887 }
3888
3889 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3890 * We need to replace this with a semaphore, or something.
3891 */
3892 dev_priv->mm.suspended = 1;
3893
3894 /* Cancel the retire work handler, wait for it to finish if running
3895 */
3896 mutex_unlock(&dev->struct_mutex);
3897 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3898 mutex_lock(&dev->struct_mutex);
3899
3900 i915_kernel_lost_context(dev);
3901
3902 /* Flush the GPU along with all non-CPU write domains
3903 */
3904 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
3905 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
3906
3907 if (seqno == 0) {
3908 mutex_unlock(&dev->struct_mutex);
3909 return -ENOMEM;
3910 }
3911
3912 dev_priv->mm.waiting_gem_seqno = seqno;
3913 last_seqno = 0;
3914 stuck = 0;
3915 for (;;) {
3916 cur_seqno = i915_get_gem_seqno(dev);
3917 if (i915_seqno_passed(cur_seqno, seqno))
3918 break;
3919 if (last_seqno == cur_seqno) {
3920 if (stuck++ > 100) {
3921 DRM_ERROR("hardware wedged\n");
3922 dev_priv->mm.wedged = 1;
3923 DRM_WAKEUP(&dev_priv->irq_queue);
3924 break;
3925 }
3926 }
3927 msleep(10);
3928 last_seqno = cur_seqno;
3929 }
3930 dev_priv->mm.waiting_gem_seqno = 0;
3931
3932 i915_gem_retire_requests(dev);
3933
3934 spin_lock(&dev_priv->mm.active_list_lock);
3935 if (!dev_priv->mm.wedged) {
3936 /* Active and flushing should now be empty as we've
3937 * waited for a sequence higher than any pending execbuffer
3938 */
3939 WARN_ON(!list_empty(&dev_priv->mm.active_list));
3940 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
3941 /* Request should now be empty as we've also waited
3942 * for the last request in the list
3943 */
3944 WARN_ON(!list_empty(&dev_priv->mm.request_list));
3945 }
3946
3947 /* Empty the active and flushing lists to inactive. If there's
3948 * anything left at this point, it means that we're wedged and
3949 * nothing good's going to happen by leaving them there. So strip
3950 * the GPU domains and just stuff them onto inactive.
3951 */
3952 while (!list_empty(&dev_priv->mm.active_list)) {
3953 struct drm_i915_gem_object *obj_priv;
3954
3955 obj_priv = list_first_entry(&dev_priv->mm.active_list,
3956 struct drm_i915_gem_object,
3957 list);
3958 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3959 i915_gem_object_move_to_inactive(obj_priv->obj);
3960 }
3961 spin_unlock(&dev_priv->mm.active_list_lock);
3962
3963 while (!list_empty(&dev_priv->mm.flushing_list)) {
3964 struct drm_i915_gem_object *obj_priv;
3965
3966 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
3967 struct drm_i915_gem_object,
3968 list);
3969 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3970 i915_gem_object_move_to_inactive(obj_priv->obj);
3971 }
3972
3973
3974 /* Move all inactive buffers out of the GTT. */
3975 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
3976 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
3977 if (ret) {
3978 mutex_unlock(&dev->struct_mutex);
3979 return ret;
3980 }
3981
3982 i915_gem_cleanup_ringbuffer(dev);
3983 mutex_unlock(&dev->struct_mutex);
3984
3985 return 0;
3986 }
3987
3988 static int
3989 i915_gem_init_hws(struct drm_device *dev)
3990 {
3991 drm_i915_private_t *dev_priv = dev->dev_private;
3992 struct drm_gem_object *obj;
3993 struct drm_i915_gem_object *obj_priv;
3994 int ret;
3995
3996 /* If we need a physical address for the status page, it's already
3997 * initialized at driver load time.
3998 */
3999 if (!I915_NEED_GFX_HWS(dev))
4000 return 0;
4001
4002 obj = drm_gem_object_alloc(dev, 4096);
4003 if (obj == NULL) {
4004 DRM_ERROR("Failed to allocate status page\n");
4005 return -ENOMEM;
4006 }
4007 obj_priv = obj->driver_private;
4008 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4009
4010 ret = i915_gem_object_pin(obj, 4096);
4011 if (ret != 0) {
4012 drm_gem_object_unreference(obj);
4013 return ret;
4014 }
4015
4016 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
4017
4018 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
4019 if (dev_priv->hw_status_page == NULL) {
4020 DRM_ERROR("Failed to map status page.\n");
4021 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4022 i915_gem_object_unpin(obj);
4023 drm_gem_object_unreference(obj);
4024 return -EINVAL;
4025 }
4026 dev_priv->hws_obj = obj;
4027 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4028 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
4029 I915_READ(HWS_PGA); /* posting read */
4030 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
4031
4032 return 0;
4033 }
4034
4035 static void
4036 i915_gem_cleanup_hws(struct drm_device *dev)
4037 {
4038 drm_i915_private_t *dev_priv = dev->dev_private;
4039 struct drm_gem_object *obj;
4040 struct drm_i915_gem_object *obj_priv;
4041
4042 if (dev_priv->hws_obj == NULL)
4043 return;
4044
4045 obj = dev_priv->hws_obj;
4046 obj_priv = obj->driver_private;
4047
4048 kunmap(obj_priv->pages[0]);
4049 i915_gem_object_unpin(obj);
4050 drm_gem_object_unreference(obj);
4051 dev_priv->hws_obj = NULL;
4052
4053 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4054 dev_priv->hw_status_page = NULL;
4055
4056 /* Write high address into HWS_PGA when disabling. */
4057 I915_WRITE(HWS_PGA, 0x1ffff000);
4058 }
4059
4060 int
4061 i915_gem_init_ringbuffer(struct drm_device *dev)
4062 {
4063 drm_i915_private_t *dev_priv = dev->dev_private;
4064 struct drm_gem_object *obj;
4065 struct drm_i915_gem_object *obj_priv;
4066 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
4067 int ret;
4068 u32 head;
4069
4070 ret = i915_gem_init_hws(dev);
4071 if (ret != 0)
4072 return ret;
4073
4074 obj = drm_gem_object_alloc(dev, 128 * 1024);
4075 if (obj == NULL) {
4076 DRM_ERROR("Failed to allocate ringbuffer\n");
4077 i915_gem_cleanup_hws(dev);
4078 return -ENOMEM;
4079 }
4080 obj_priv = obj->driver_private;
4081
4082 ret = i915_gem_object_pin(obj, 4096);
4083 if (ret != 0) {
4084 drm_gem_object_unreference(obj);
4085 i915_gem_cleanup_hws(dev);
4086 return ret;
4087 }
4088
4089 /* Set up the kernel mapping for the ring. */
4090 ring->Size = obj->size;
4091 ring->tail_mask = obj->size - 1;
4092
4093 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4094 ring->map.size = obj->size;
4095 ring->map.type = 0;
4096 ring->map.flags = 0;
4097 ring->map.mtrr = 0;
4098
4099 drm_core_ioremap_wc(&ring->map, dev);
4100 if (ring->map.handle == NULL) {
4101 DRM_ERROR("Failed to map ringbuffer.\n");
4102 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4103 i915_gem_object_unpin(obj);
4104 drm_gem_object_unreference(obj);
4105 i915_gem_cleanup_hws(dev);
4106 return -EINVAL;
4107 }
4108 ring->ring_obj = obj;
4109 ring->virtual_start = ring->map.handle;
4110
4111 /* Stop the ring if it's running. */
4112 I915_WRITE(PRB0_CTL, 0);
4113 I915_WRITE(PRB0_TAIL, 0);
4114 I915_WRITE(PRB0_HEAD, 0);
4115
4116 /* Initialize the ring. */
4117 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
4118 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4119
4120 /* G45 ring initialization fails to reset head to zero */
4121 if (head != 0) {
4122 DRM_ERROR("Ring head not reset to zero "
4123 "ctl %08x head %08x tail %08x start %08x\n",
4124 I915_READ(PRB0_CTL),
4125 I915_READ(PRB0_HEAD),
4126 I915_READ(PRB0_TAIL),
4127 I915_READ(PRB0_START));
4128 I915_WRITE(PRB0_HEAD, 0);
4129
4130 DRM_ERROR("Ring head forced to zero "
4131 "ctl %08x head %08x tail %08x start %08x\n",
4132 I915_READ(PRB0_CTL),
4133 I915_READ(PRB0_HEAD),
4134 I915_READ(PRB0_TAIL),
4135 I915_READ(PRB0_START));
4136 }
4137
4138 I915_WRITE(PRB0_CTL,
4139 ((obj->size - 4096) & RING_NR_PAGES) |
4140 RING_NO_REPORT |
4141 RING_VALID);
4142
4143 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4144
4145 /* If the head is still not zero, the ring is dead */
4146 if (head != 0) {
4147 DRM_ERROR("Ring initialization failed "
4148 "ctl %08x head %08x tail %08x start %08x\n",
4149 I915_READ(PRB0_CTL),
4150 I915_READ(PRB0_HEAD),
4151 I915_READ(PRB0_TAIL),
4152 I915_READ(PRB0_START));
4153 return -EIO;
4154 }
4155
4156 /* Update our cache of the ring state */
4157 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4158 i915_kernel_lost_context(dev);
4159 else {
4160 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4161 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4162 ring->space = ring->head - (ring->tail + 8);
4163 if (ring->space < 0)
4164 ring->space += ring->Size;
4165 }
4166
4167 return 0;
4168 }
4169
4170 void
4171 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4172 {
4173 drm_i915_private_t *dev_priv = dev->dev_private;
4174
4175 if (dev_priv->ring.ring_obj == NULL)
4176 return;
4177
4178 drm_core_ioremapfree(&dev_priv->ring.map, dev);
4179
4180 i915_gem_object_unpin(dev_priv->ring.ring_obj);
4181 drm_gem_object_unreference(dev_priv->ring.ring_obj);
4182 dev_priv->ring.ring_obj = NULL;
4183 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4184
4185 i915_gem_cleanup_hws(dev);
4186 }
4187
4188 int
4189 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4190 struct drm_file *file_priv)
4191 {
4192 drm_i915_private_t *dev_priv = dev->dev_private;
4193 int ret;
4194
4195 if (drm_core_check_feature(dev, DRIVER_MODESET))
4196 return 0;
4197
4198 if (dev_priv->mm.wedged) {
4199 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4200 dev_priv->mm.wedged = 0;
4201 }
4202
4203 mutex_lock(&dev->struct_mutex);
4204 dev_priv->mm.suspended = 0;
4205
4206 ret = i915_gem_init_ringbuffer(dev);
4207 if (ret != 0) {
4208 mutex_unlock(&dev->struct_mutex);
4209 return ret;
4210 }
4211
4212 spin_lock(&dev_priv->mm.active_list_lock);
4213 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4214 spin_unlock(&dev_priv->mm.active_list_lock);
4215
4216 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4217 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4218 BUG_ON(!list_empty(&dev_priv->mm.request_list));
4219 mutex_unlock(&dev->struct_mutex);
4220
4221 drm_irq_install(dev);
4222
4223 return 0;
4224 }
4225
4226 int
4227 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4228 struct drm_file *file_priv)
4229 {
4230 if (drm_core_check_feature(dev, DRIVER_MODESET))
4231 return 0;
4232
4233 drm_irq_uninstall(dev);
4234 return i915_gem_idle(dev);
4235 }
4236
4237 void
4238 i915_gem_lastclose(struct drm_device *dev)
4239 {
4240 int ret;
4241
4242 if (drm_core_check_feature(dev, DRIVER_MODESET))
4243 return;
4244
4245 ret = i915_gem_idle(dev);
4246 if (ret)
4247 DRM_ERROR("failed to idle hardware: %d\n", ret);
4248 }
4249
4250 void
4251 i915_gem_load(struct drm_device *dev)
4252 {
4253 int i;
4254 drm_i915_private_t *dev_priv = dev->dev_private;
4255
4256 spin_lock_init(&dev_priv->mm.active_list_lock);
4257 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4258 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4259 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4260 INIT_LIST_HEAD(&dev_priv->mm.request_list);
4261 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4262 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4263 i915_gem_retire_work_handler);
4264 dev_priv->mm.next_gem_seqno = 1;
4265
4266 /* Old X drivers will take 0-2 for front, back, depth buffers */
4267 dev_priv->fence_reg_start = 3;
4268
4269 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4270 dev_priv->num_fence_regs = 16;
4271 else
4272 dev_priv->num_fence_regs = 8;
4273
4274 /* Initialize fence registers to zero */
4275 if (IS_I965G(dev)) {
4276 for (i = 0; i < 16; i++)
4277 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4278 } else {
4279 for (i = 0; i < 8; i++)
4280 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4281 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4282 for (i = 0; i < 8; i++)
4283 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4284 }
4285
4286 i915_gem_detect_bit_6_swizzle(dev);
4287 }
4288
4289 /*
4290 * Create a physically contiguous memory object for this object
4291 * e.g. for cursor + overlay regs
4292 */
4293 int i915_gem_init_phys_object(struct drm_device *dev,
4294 int id, int size)
4295 {
4296 drm_i915_private_t *dev_priv = dev->dev_private;
4297 struct drm_i915_gem_phys_object *phys_obj;
4298 int ret;
4299
4300 if (dev_priv->mm.phys_objs[id - 1] || !size)
4301 return 0;
4302
4303 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4304 if (!phys_obj)
4305 return -ENOMEM;
4306
4307 phys_obj->id = id;
4308
4309 phys_obj->handle = drm_pci_alloc(dev, size, 0, 0xffffffff);
4310 if (!phys_obj->handle) {
4311 ret = -ENOMEM;
4312 goto kfree_obj;
4313 }
4314 #ifdef CONFIG_X86
4315 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4316 #endif
4317
4318 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4319
4320 return 0;
4321 kfree_obj:
4322 kfree(phys_obj);
4323 return ret;
4324 }
4325
4326 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4327 {
4328 drm_i915_private_t *dev_priv = dev->dev_private;
4329 struct drm_i915_gem_phys_object *phys_obj;
4330
4331 if (!dev_priv->mm.phys_objs[id - 1])
4332 return;
4333
4334 phys_obj = dev_priv->mm.phys_objs[id - 1];
4335 if (phys_obj->cur_obj) {
4336 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4337 }
4338
4339 #ifdef CONFIG_X86
4340 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4341 #endif
4342 drm_pci_free(dev, phys_obj->handle);
4343 kfree(phys_obj);
4344 dev_priv->mm.phys_objs[id - 1] = NULL;
4345 }
4346
4347 void i915_gem_free_all_phys_object(struct drm_device *dev)
4348 {
4349 int i;
4350
4351 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4352 i915_gem_free_phys_object(dev, i);
4353 }
4354
4355 void i915_gem_detach_phys_object(struct drm_device *dev,
4356 struct drm_gem_object *obj)
4357 {
4358 struct drm_i915_gem_object *obj_priv;
4359 int i;
4360 int ret;
4361 int page_count;
4362
4363 obj_priv = obj->driver_private;
4364 if (!obj_priv->phys_obj)
4365 return;
4366
4367 ret = i915_gem_object_get_pages(obj);
4368 if (ret)
4369 goto out;
4370
4371 page_count = obj->size / PAGE_SIZE;
4372
4373 for (i = 0; i < page_count; i++) {
4374 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4375 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4376
4377 memcpy(dst, src, PAGE_SIZE);
4378 kunmap_atomic(dst, KM_USER0);
4379 }
4380 drm_clflush_pages(obj_priv->pages, page_count);
4381 drm_agp_chipset_flush(dev);
4382
4383 i915_gem_object_put_pages(obj);
4384 out:
4385 obj_priv->phys_obj->cur_obj = NULL;
4386 obj_priv->phys_obj = NULL;
4387 }
4388
4389 int
4390 i915_gem_attach_phys_object(struct drm_device *dev,
4391 struct drm_gem_object *obj, int id)
4392 {
4393 drm_i915_private_t *dev_priv = dev->dev_private;
4394 struct drm_i915_gem_object *obj_priv;
4395 int ret = 0;
4396 int page_count;
4397 int i;
4398
4399 if (id > I915_MAX_PHYS_OBJECT)
4400 return -EINVAL;
4401
4402 obj_priv = obj->driver_private;
4403
4404 if (obj_priv->phys_obj) {
4405 if (obj_priv->phys_obj->id == id)
4406 return 0;
4407 i915_gem_detach_phys_object(dev, obj);
4408 }
4409
4410
4411 /* create a new object */
4412 if (!dev_priv->mm.phys_objs[id - 1]) {
4413 ret = i915_gem_init_phys_object(dev, id,
4414 obj->size);
4415 if (ret) {
4416 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4417 goto out;
4418 }
4419 }
4420
4421 /* bind to the object */
4422 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4423 obj_priv->phys_obj->cur_obj = obj;
4424
4425 ret = i915_gem_object_get_pages(obj);
4426 if (ret) {
4427 DRM_ERROR("failed to get page list\n");
4428 goto out;
4429 }
4430
4431 page_count = obj->size / PAGE_SIZE;
4432
4433 for (i = 0; i < page_count; i++) {
4434 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4435 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4436
4437 memcpy(dst, src, PAGE_SIZE);
4438 kunmap_atomic(src, KM_USER0);
4439 }
4440
4441 i915_gem_object_put_pages(obj);
4442
4443 return 0;
4444 out:
4445 return ret;
4446 }
4447
4448 static int
4449 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4450 struct drm_i915_gem_pwrite *args,
4451 struct drm_file *file_priv)
4452 {
4453 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4454 void *obj_addr;
4455 int ret;
4456 char __user *user_data;
4457
4458 user_data = (char __user *) (uintptr_t) args->data_ptr;
4459 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4460
4461 DRM_DEBUG("obj_addr %p, %lld\n", obj_addr, args->size);
4462 ret = copy_from_user(obj_addr, user_data, args->size);
4463 if (ret)
4464 return -EFAULT;
4465
4466 drm_agp_chipset_flush(dev);
4467 return 0;
4468 }
4469
4470 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4471 {
4472 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4473
4474 /* Clean up our request list when the client is going away, so that
4475 * later retire_requests won't dereference our soon-to-be-gone
4476 * file_priv.
4477 */
4478 mutex_lock(&dev->struct_mutex);
4479 while (!list_empty(&i915_file_priv->mm.request_list))
4480 list_del_init(i915_file_priv->mm.request_list.next);
4481 mutex_unlock(&dev->struct_mutex);
4482 }
4483
|
This page was automatically generated by the
LXR engine.
|