drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35
  36 static u32 i915_gem_get_seqno(struct drm_device *dev)
  37 {
  38         drm_i915_private_t *dev_priv = dev->dev_private;
  39         u32 seqno;
  40
  41         seqno = dev_priv->next_seqno;
  42
  43         /* reserve 0 for non-seqno */
  44         if (++dev_priv->next_seqno == 0)
  45                 dev_priv->next_seqno = 1;
  46
  47         return seqno;
  48 }
  49
  50 static void
  51 render_ring_flush(struct drm_device *dev,
  52                 struct intel_ring_buffer *ring,
  53                 u32     invalidate_domains,
  54                 u32     flush_domains)
  55 {
  56         drm_i915_private_t *dev_priv = dev->dev_private;
  57         u32 cmd;
  58
  59 #if WATCH_EXEC
  60         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  61                   invalidate_domains, flush_domains);
  62 #endif
  63
  64         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
  65                                      invalidate_domains, flush_domains);
  66
  67         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  68                 /*
  69                  * read/write caches:
  70                  *
  71                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  72                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  73                  * also flushed at 2d versus 3d pipeline switches.
  74                  *
  75                  * read-only caches:
  76                  *
  77                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  78                  * MI_READ_FLUSH is set, and is always flushed on 965.
  79                  *
  80                  * I915_GEM_DOMAIN_COMMAND may not exist?
  81                  *
  82                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  83                  * invalidated when MI_EXE_FLUSH is set.
  84                  *
  85                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  86                  * invalidated with every MI_FLUSH.
  87                  *
  88                  * TLBs:
  89                  *
  90                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  91                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  92                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  93                  * are flushed at any MI_FLUSH.
  94                  */
  95
  96                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  97                 if ((invalidate_domains|flush_domains) &
  98                     I915_GEM_DOMAIN_RENDER)
  99                         cmd &= ~MI_NO_WRITE_FLUSH;
 100                 if (!IS_I965G(dev)) {
 101                         /*
 102                          * On the 965, the sampler cache always gets flushed
 103                          * and this bit is reserved.
 104                          */
 105                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 106                                 cmd |= MI_READ_FLUSH;
 107                 }
 108                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 109                         cmd |= MI_EXE_FLUSH;
 110
 111 #if WATCH_EXEC
 112                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 113 #endif
 114                 intel_ring_begin(dev, ring, 2);
 115                 intel_ring_emit(dev, ring, cmd);
 116                 intel_ring_emit(dev, ring, MI_NOOP);
 117                 intel_ring_advance(dev, ring);
 118         }
 119 }
 120
 121 static unsigned int render_ring_get_head(struct drm_device *dev,
 122                 struct intel_ring_buffer *ring)
 123 {
 124         drm_i915_private_t *dev_priv = dev->dev_private;
 125         return I915_READ(PRB0_HEAD) & HEAD_ADDR;
 126 }
 127
 128 static unsigned int render_ring_get_tail(struct drm_device *dev,
 129                 struct intel_ring_buffer *ring)
 130 {
 131         drm_i915_private_t *dev_priv = dev->dev_private;
 132         return I915_READ(PRB0_TAIL) & TAIL_ADDR;
 133 }
 134
 135 static unsigned int render_ring_get_active_head(struct drm_device *dev,
 136                 struct intel_ring_buffer *ring)
 137 {
 138         drm_i915_private_t *dev_priv = dev->dev_private;
 139         u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
 140
 141         return I915_READ(acthd_reg);
 142 }
 143
 144 static void render_ring_advance_ring(struct drm_device *dev,
 145                 struct intel_ring_buffer *ring)
 146 {
 147         drm_i915_private_t *dev_priv = dev->dev_private;
 148         I915_WRITE(PRB0_TAIL, ring->tail);
 149 }
 150
 151 static int init_ring_common(struct drm_device *dev,
 152                 struct intel_ring_buffer *ring)
 153 {
 154         u32 head;
 155         drm_i915_private_t *dev_priv = dev->dev_private;
 156         struct drm_i915_gem_object *obj_priv;
 157         obj_priv = to_intel_bo(ring->gem_object);
 158
 159         /* Stop the ring if it's running. */
 160         I915_WRITE(ring->regs.ctl, 0);
 161         I915_WRITE(ring->regs.head, 0);
 162         I915_WRITE(ring->regs.tail, 0);
 163
 164         /* Initialize the ring. */
 165         I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
 166         head = ring->get_head(dev, ring);
 167
 168         /* G45 ring initialization fails to reset head to zero */
 169         if (head != 0) {
 170                 DRM_ERROR("%s head not reset to zero "
 171                                 "ctl %08x head %08x tail %08x start %08x\n",
 172                                 ring->name,
 173                                 I915_READ(ring->regs.ctl),
 174                                 I915_READ(ring->regs.head),
 175                                 I915_READ(ring->regs.tail),
 176                                 I915_READ(ring->regs.start));
 177
 178                 I915_WRITE(ring->regs.head, 0);
 179
 180                 DRM_ERROR("%s head forced to zero "
 181                                 "ctl %08x head %08x tail %08x start %08x\n",
 182                                 ring->name,
 183                                 I915_READ(ring->regs.ctl),
 184                                 I915_READ(ring->regs.head),
 185                                 I915_READ(ring->regs.tail),
 186                                 I915_READ(ring->regs.start));
 187         }
 188
 189         I915_WRITE(ring->regs.ctl,
 190                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
 191                         | RING_NO_REPORT | RING_VALID);
 192
 193         head = I915_READ(ring->regs.head) & HEAD_ADDR;
 194         /* If the head is still not zero, the ring is dead */
 195         if (head != 0) {
 196                 DRM_ERROR("%s initialization failed "
 197                                 "ctl %08x head %08x tail %08x start %08x\n",
 198                                 ring->name,
 199                                 I915_READ(ring->regs.ctl),
 200                                 I915_READ(ring->regs.head),
 201                                 I915_READ(ring->regs.tail),
 202                                 I915_READ(ring->regs.start));
 203                 return -EIO;
 204         }
 205
 206         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 207                 i915_kernel_lost_context(dev);
 208         else {
 209                 ring->head = ring->get_head(dev, ring);
 210                 ring->tail = ring->get_tail(dev, ring);
 211                 ring->space = ring->head - (ring->tail + 8);
 212                 if (ring->space < 0)
 213                         ring->space += ring->size;
 214         }
 215         return 0;
 216 }
 217
 218 static int init_render_ring(struct drm_device *dev,
 219                 struct intel_ring_buffer *ring)
 220 {
 221         drm_i915_private_t *dev_priv = dev->dev_private;
 222         int ret = init_ring_common(dev, ring);
 223         if (IS_I9XX(dev) && !IS_GEN3(dev)) {
 224                 I915_WRITE(MI_MODE,
 225                                 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
 226         }
 227         return ret;
 228 }
 229
 230 #define PIPE_CONTROL_FLUSH(addr)                                        \
 231 do {                                                                    \
 232         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
 233                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
 234         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
 235         OUT_RING(0);                                                    \
 236         OUT_RING(0);                                                    \
 237 } while (0)
 238
 239 /**
 240  * Creates a new sequence number, emitting a write of it to the status page
 241  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 242  *
 243  * Must be called with struct_lock held.
 244  *
 245  * Returned sequence numbers are nonzero on success.
 246  */
 247 static u32
 248 render_ring_add_request(struct drm_device *dev,
 249                 struct intel_ring_buffer *ring,
 250                 struct drm_file *file_priv,
 251                 u32 flush_domains)
 252 {
 253         drm_i915_private_t *dev_priv = dev->dev_private;
 254         u32 seqno;
 255
 256         seqno = i915_gem_get_seqno(dev);
 257
 258         if (IS_GEN6(dev)) {
 259                 BEGIN_LP_RING(6);
 260                 OUT_RING(GFX_OP_PIPE_CONTROL | 3);
 261                 OUT_RING(PIPE_CONTROL_QW_WRITE |
 262                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
 263                          PIPE_CONTROL_NOTIFY);
 264                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 265                 OUT_RING(seqno);
 266                 OUT_RING(0);
 267                 OUT_RING(0);
 268                 ADVANCE_LP_RING();
 269         } else if (HAS_PIPE_CONTROL(dev)) {
 270                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 271
 272                 /*
 273                  * Workaround qword write incoherence by flushing the
 274                  * PIPE_NOTIFY buffers out to memory before requesting
 275                  * an interrupt.
 276                  */
 277                 BEGIN_LP_RING(32);
 278                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 279                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 280                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 281                 OUT_RING(seqno);
 282                 OUT_RING(0);
 283                 PIPE_CONTROL_FLUSH(scratch_addr);
 284                 scratch_addr += 128; /* write to separate cachelines */
 285                 PIPE_CONTROL_FLUSH(scratch_addr);
 286                 scratch_addr += 128;
 287                 PIPE_CONTROL_FLUSH(scratch_addr);
 288                 scratch_addr += 128;
 289                 PIPE_CONTROL_FLUSH(scratch_addr);
 290                 scratch_addr += 128;
 291                 PIPE_CONTROL_FLUSH(scratch_addr);
 292                 scratch_addr += 128;
 293                 PIPE_CONTROL_FLUSH(scratch_addr);
 294                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 295                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 296                          PIPE_CONTROL_NOTIFY);
 297                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 298                 OUT_RING(seqno);
 299                 OUT_RING(0);
 300                 ADVANCE_LP_RING();
 301         } else {
 302                 BEGIN_LP_RING(4);
 303                 OUT_RING(MI_STORE_DWORD_INDEX);
 304                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 305                 OUT_RING(seqno);
 306
 307                 OUT_RING(MI_USER_INTERRUPT);
 308                 ADVANCE_LP_RING();
 309         }
 310         return seqno;
 311 }
 312
 313 static u32
 314 render_ring_get_gem_seqno(struct drm_device *dev,
 315                 struct intel_ring_buffer *ring)
 316 {
 317         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 318         if (HAS_PIPE_CONTROL(dev))
 319                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
 320         else
 321                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 322 }
 323
 324 static void
 325 render_ring_get_user_irq(struct drm_device *dev,
 326                 struct intel_ring_buffer *ring)
 327 {
 328         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 329         unsigned long irqflags;
 330
 331         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 332         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 333                 if (HAS_PCH_SPLIT(dev))
 334                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 335                 else
 336                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 337         }
 338         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 339 }
 340
 341 static void
 342 render_ring_put_user_irq(struct drm_device *dev,
 343                 struct intel_ring_buffer *ring)
 344 {
 345         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 346         unsigned long irqflags;
 347
 348         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 349         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 350         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 351                 if (HAS_PCH_SPLIT(dev))
 352                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 353                 else
 354                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 355         }
 356         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 357 }
 358
 359 static void render_setup_status_page(struct drm_device *dev,
 360         struct  intel_ring_buffer *ring)
 361 {
 362         drm_i915_private_t *dev_priv = dev->dev_private;
 363         if (IS_GEN6(dev)) {
 364                 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
 365                 I915_READ(HWS_PGA_GEN6); /* posting read */
 366         } else {
 367                 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
 368                 I915_READ(HWS_PGA); /* posting read */
 369         }
 370
 371 }
 372
 373 void
 374 bsd_ring_flush(struct drm_device *dev,
 375                 struct intel_ring_buffer *ring,
 376                 u32     invalidate_domains,
 377                 u32     flush_domains)
 378 {
 379         intel_ring_begin(dev, ring, 2);
 380         intel_ring_emit(dev, ring, MI_FLUSH);
 381         intel_ring_emit(dev, ring, MI_NOOP);
 382         intel_ring_advance(dev, ring);
 383 }
 384
 385 static inline unsigned int bsd_ring_get_head(struct drm_device *dev,
 386                 struct intel_ring_buffer *ring)
 387 {
 388         drm_i915_private_t *dev_priv = dev->dev_private;
 389         return I915_READ(BSD_RING_HEAD) & HEAD_ADDR;
 390 }
 391
 392 static inline unsigned int bsd_ring_get_tail(struct drm_device *dev,
 393                 struct intel_ring_buffer *ring)
 394 {
 395         drm_i915_private_t *dev_priv = dev->dev_private;
 396         return I915_READ(BSD_RING_TAIL) & TAIL_ADDR;
 397 }
 398
 399 static inline unsigned int bsd_ring_get_active_head(struct drm_device *dev,
 400                 struct intel_ring_buffer *ring)
 401 {
 402         drm_i915_private_t *dev_priv = dev->dev_private;
 403         return I915_READ(BSD_RING_ACTHD);
 404 }
 405
 406 static inline void bsd_ring_advance_ring(struct drm_device *dev,
 407                 struct intel_ring_buffer *ring)
 408 {
 409         drm_i915_private_t *dev_priv = dev->dev_private;
 410         I915_WRITE(BSD_RING_TAIL, ring->tail);
 411 }
 412
 413 static int init_bsd_ring(struct drm_device *dev,
 414                 struct intel_ring_buffer *ring)
 415 {
 416         return init_ring_common(dev, ring);
 417 }
 418
 419 static u32
 420 bsd_ring_add_request(struct drm_device *dev,
 421                 struct intel_ring_buffer *ring,
 422                 struct drm_file *file_priv,
 423                 u32 flush_domains)
 424 {
 425         u32 seqno;
 426
 427         seqno = i915_gem_get_seqno(dev);
 428
 429         intel_ring_begin(dev, ring, 4);
 430         intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
 431         intel_ring_emit(dev, ring,
 432                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 433         intel_ring_emit(dev, ring, seqno);
 434         intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
 435         intel_ring_advance(dev, ring);
 436
 437         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
 438
 439         return seqno;
 440 }
 441
 442 static void bsd_setup_status_page(struct drm_device *dev,
 443                 struct  intel_ring_buffer *ring)
 444 {
 445         drm_i915_private_t *dev_priv = dev->dev_private;
 446         I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr);
 447         I915_READ(BSD_HWS_PGA);
 448 }
 449
 450 static void
 451 bsd_ring_get_user_irq(struct drm_device *dev,
 452                 struct intel_ring_buffer *ring)
 453 {
 454         /* do nothing */
 455 }
 456 static void
 457 bsd_ring_put_user_irq(struct drm_device *dev,
 458                 struct intel_ring_buffer *ring)
 459 {
 460         /* do nothing */
 461 }
 462
 463 static u32
 464 bsd_ring_get_gem_seqno(struct drm_device *dev,
 465                 struct intel_ring_buffer *ring)
 466 {
 467         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 468 }
 469
 470 static int
 471 bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 472                 struct intel_ring_buffer *ring,
 473                 struct drm_i915_gem_execbuffer2 *exec,
 474                 struct drm_clip_rect *cliprects,
 475                 uint64_t exec_offset)
 476 {
 477         uint32_t exec_start;
 478         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 479         intel_ring_begin(dev, ring, 2);
 480         intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
 481                         (2 << 6) | MI_BATCH_NON_SECURE_I965);
 482         intel_ring_emit(dev, ring, exec_start);
 483         intel_ring_advance(dev, ring);
 484         return 0;
 485 }
 486
 487
 488 static int
 489 render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 490                 struct intel_ring_buffer *ring,
 491                 struct drm_i915_gem_execbuffer2 *exec,
 492                 struct drm_clip_rect *cliprects,
 493                 uint64_t exec_offset)
 494 {
 495         drm_i915_private_t *dev_priv = dev->dev_private;
 496         int nbox = exec->num_cliprects;
 497         int i = 0, count;
 498         uint32_t exec_start, exec_len;
 499         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 500         exec_len = (uint32_t) exec->batch_len;
 501
 502         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 503
 504         count = nbox ? nbox : 1;
 505
 506         for (i = 0; i < count; i++) {
 507                 if (i < nbox) {
 508                         int ret = i915_emit_box(dev, cliprects, i,
 509                                                 exec->DR1, exec->DR4);
 510                         if (ret)
 511                                 return ret;
 512                 }
 513
 514                 if (IS_I830(dev) || IS_845G(dev)) {
 515                         intel_ring_begin(dev, ring, 4);
 516                         intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
 517                         intel_ring_emit(dev, ring,
 518                                         exec_start | MI_BATCH_NON_SECURE);
 519                         intel_ring_emit(dev, ring, exec_start + exec_len - 4);
 520                         intel_ring_emit(dev, ring, 0);
 521                 } else {
 522                         intel_ring_begin(dev, ring, 4);
 523                         if (IS_I965G(dev)) {
 524                                 intel_ring_emit(dev, ring,
 525                                                 MI_BATCH_BUFFER_START | (2 << 6)
 526                                                 | MI_BATCH_NON_SECURE_I965);
 527                                 intel_ring_emit(dev, ring, exec_start);
 528                         } else {
 529                                 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
 530                                                 | (2 << 6));
 531                                 intel_ring_emit(dev, ring, exec_start |
 532                                                 MI_BATCH_NON_SECURE);
 533                         }
 534                 }
 535                 intel_ring_advance(dev, ring);
 536         }
 537
 538         /* XXX breadcrumb */
 539         return 0;
 540 }
 541
 542 static void cleanup_status_page(struct drm_device *dev,
 543                 struct intel_ring_buffer *ring)
 544 {
 545         drm_i915_private_t *dev_priv = dev->dev_private;
 546         struct drm_gem_object *obj;
 547         struct drm_i915_gem_object *obj_priv;
 548
 549         obj = ring->status_page.obj;
 550         if (obj == NULL)
 551                 return;
 552         obj_priv = to_intel_bo(obj);
 553
 554         kunmap(obj_priv->pages[0]);
 555         i915_gem_object_unpin(obj);
 556         drm_gem_object_unreference(obj);
 557         ring->status_page.obj = NULL;
 558
 559         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 560 }
 561
 562 static int init_status_page(struct drm_device *dev,
 563                 struct intel_ring_buffer *ring)
 564 {
 565         drm_i915_private_t *dev_priv = dev->dev_private;
 566         struct drm_gem_object *obj;
 567         struct drm_i915_gem_object *obj_priv;
 568         int ret;
 569
 570         obj = i915_gem_alloc_object(dev, 4096);
 571         if (obj == NULL) {
 572                 DRM_ERROR("Failed to allocate status page\n");
 573                 ret = -ENOMEM;
 574                 goto err;
 575         }
 576         obj_priv = to_intel_bo(obj);
 577         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 578
 579         ret = i915_gem_object_pin(obj, 4096);
 580         if (ret != 0) {
 581                 goto err_unref;
 582         }
 583
 584         ring->status_page.gfx_addr = obj_priv->gtt_offset;
 585         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
 586         if (ring->status_page.page_addr == NULL) {
 587                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 588                 goto err_unpin;
 589         }
 590         ring->status_page.obj = obj;
 591         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 592
 593         ring->setup_status_page(dev, ring);
 594         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 595                         ring->name, ring->status_page.gfx_addr);
 596
 597         return 0;
 598
 599 err_unpin:
 600         i915_gem_object_unpin(obj);
 601 err_unref:
 602         drm_gem_object_unreference(obj);
 603 err:
 604         return ret;
 605 }
 606
 607
 608 int intel_init_ring_buffer(struct drm_device *dev,
 609                 struct intel_ring_buffer *ring)
 610 {
 611         int ret;
 612         struct drm_i915_gem_object *obj_priv;
 613         struct drm_gem_object *obj;
 614         ring->dev = dev;
 615
 616         if (I915_NEED_GFX_HWS(dev)) {
 617                 ret = init_status_page(dev, ring);
 618                 if (ret)
 619                         return ret;
 620         }
 621
 622         obj = i915_gem_alloc_object(dev, ring->size);
 623         if (obj == NULL) {
 624                 DRM_ERROR("Failed to allocate ringbuffer\n");
 625                 ret = -ENOMEM;
 626                 goto cleanup;
 627         }
 628
 629         ring->gem_object = obj;
 630
 631         ret = i915_gem_object_pin(obj, ring->alignment);
 632         if (ret != 0) {
 633                 drm_gem_object_unreference(obj);
 634                 goto cleanup;
 635         }
 636
 637         obj_priv = to_intel_bo(obj);
 638         ring->map.size = ring->size;
 639         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 640         ring->map.type = 0;
 641         ring->map.flags = 0;
 642         ring->map.mtrr = 0;
 643
 644         drm_core_ioremap_wc(&ring->map, dev);
 645         if (ring->map.handle == NULL) {
 646                 DRM_ERROR("Failed to map ringbuffer.\n");
 647                 i915_gem_object_unpin(obj);
 648                 drm_gem_object_unreference(obj);
 649                 ret = -EINVAL;
 650                 goto cleanup;
 651         }
 652
 653         ring->virtual_start = ring->map.handle;
 654         ret = ring->init(dev, ring);
 655         if (ret != 0) {
 656                 intel_cleanup_ring_buffer(dev, ring);
 657                 return ret;
 658         }
 659
 660         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 661                 i915_kernel_lost_context(dev);
 662         else {
 663                 ring->head = ring->get_head(dev, ring);
 664                 ring->tail = ring->get_tail(dev, ring);
 665                 ring->space = ring->head - (ring->tail + 8);
 666                 if (ring->space < 0)
 667                         ring->space += ring->size;
 668         }
 669         INIT_LIST_HEAD(&ring->active_list);
 670         INIT_LIST_HEAD(&ring->request_list);
 671         return ret;
 672 cleanup:
 673         cleanup_status_page(dev, ring);
 674         return ret;
 675 }
 676
 677 void intel_cleanup_ring_buffer(struct drm_device *dev,
 678                 struct intel_ring_buffer *ring)
 679 {
 680         if (ring->gem_object == NULL)
 681                 return;
 682
 683         drm_core_ioremapfree(&ring->map, dev);
 684
 685         i915_gem_object_unpin(ring->gem_object);
 686         drm_gem_object_unreference(ring->gem_object);
 687         ring->gem_object = NULL;
 688         cleanup_status_page(dev, ring);
 689 }
 690
 691 int intel_wrap_ring_buffer(struct drm_device *dev,
 692                 struct intel_ring_buffer *ring)
 693 {
 694         unsigned int *virt;
 695         int rem;
 696         rem = ring->size - ring->tail;
 697
 698         if (ring->space < rem) {
 699                 int ret = intel_wait_ring_buffer(dev, ring, rem);
 700                 if (ret)
 701                         return ret;
 702         }
 703
 704         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 705         rem /= 8;
 706         while (rem--) {
 707                 *virt++ = MI_NOOP;
 708                 *virt++ = MI_NOOP;
 709         }
 710
 711         ring->tail = 0;
 712         ring->space = ring->head - 8;
 713
 714         return 0;
 715 }
 716
 717 int intel_wait_ring_buffer(struct drm_device *dev,
 718                 struct intel_ring_buffer *ring, int n)
 719 {
 720         unsigned long end;
 721
 722         trace_i915_ring_wait_begin (dev);
 723         end = jiffies + 3 * HZ;
 724         do {
 725                 ring->head = ring->get_head(dev, ring);
 726                 ring->space = ring->head - (ring->tail + 8);
 727                 if (ring->space < 0)
 728                         ring->space += ring->size;
 729                 if (ring->space >= n) {
 730                         trace_i915_ring_wait_end (dev);
 731                         return 0;
 732                 }
 733
 734                 if (dev->primary->master) {
 735                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 736                         if (master_priv->sarea_priv)
 737                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 738                 }
 739
 740                 yield();
 741         } while (!time_after(jiffies, end));
 742         trace_i915_ring_wait_end (dev);
 743         return -EBUSY;
 744 }
 745
 746 void intel_ring_begin(struct drm_device *dev,
 747                 struct intel_ring_buffer *ring, int num_dwords)
 748 {
 749         int n = 4*num_dwords;
 750         if (unlikely(ring->tail + n > ring->size))
 751                 intel_wrap_ring_buffer(dev, ring);
 752         if (unlikely(ring->space < n))
 753                 intel_wait_ring_buffer(dev, ring, n);
 754
 755         ring->space -= n;
 756 }
 757
 758 void intel_ring_advance(struct drm_device *dev,
 759                 struct intel_ring_buffer *ring)
 760 {
 761         ring->tail &= ring->size - 1;
 762         ring->advance_ring(dev, ring);
 763 }
 764
 765 void intel_fill_struct(struct drm_device *dev,
 766                 struct intel_ring_buffer *ring,
 767                 void *data,
 768                 unsigned int len)
 769 {
 770         unsigned int *virt = ring->virtual_start + ring->tail;
 771         BUG_ON((len&~(4-1)) != 0);
 772         intel_ring_begin(dev, ring, len/4);
 773         memcpy(virt, data, len);
 774         ring->tail += len;
 775         ring->tail &= ring->size - 1;
 776         ring->space -= len;
 777         intel_ring_advance(dev, ring);
 778 }
 779
 780 struct intel_ring_buffer render_ring = {
 781         .name                   = "render ring",
 782         .regs                   = {
 783                 .ctl = PRB0_CTL,
 784                 .head = PRB0_HEAD,
 785                 .tail = PRB0_TAIL,
 786                 .start = PRB0_START
 787         },
 788         .ring_flag              = I915_EXEC_RENDER,
 789         .size                   = 32 * PAGE_SIZE,
 790         .alignment              = PAGE_SIZE,
 791         .virtual_start          = NULL,
 792         .dev                    = NULL,
 793         .gem_object             = NULL,
 794         .head                   = 0,
 795         .tail                   = 0,
 796         .space                  = 0,
 797         .user_irq_refcount      = 0,
 798         .irq_gem_seqno          = 0,
 799         .waiting_gem_seqno      = 0,
 800         .setup_status_page      = render_setup_status_page,
 801         .init                   = init_render_ring,
 802         .get_head               = render_ring_get_head,
 803         .get_tail               = render_ring_get_tail,
 804         .get_active_head        = render_ring_get_active_head,
 805         .advance_ring           = render_ring_advance_ring,
 806         .flush                  = render_ring_flush,
 807         .add_request            = render_ring_add_request,
 808         .get_gem_seqno          = render_ring_get_gem_seqno,
 809         .user_irq_get           = render_ring_get_user_irq,
 810         .user_irq_put           = render_ring_put_user_irq,
 811         .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
 812         .status_page            = {NULL, 0, NULL},
 813         .map                    = {0,}
 814 };
 815
 816 /* ring buffer for bit-stream decoder */
 817
 818 struct intel_ring_buffer bsd_ring = {
 819         .name                   = "bsd ring",
 820         .regs                   = {
 821                 .ctl = BSD_RING_CTL,
 822                 .head = BSD_RING_HEAD,
 823                 .tail = BSD_RING_TAIL,
 824                 .start = BSD_RING_START
 825         },
 826         .ring_flag              = I915_EXEC_BSD,
 827         .size                   = 32 * PAGE_SIZE,
 828         .alignment              = PAGE_SIZE,
 829         .virtual_start          = NULL,
 830         .dev                    = NULL,
 831         .gem_object             = NULL,
 832         .head                   = 0,
 833         .tail                   = 0,
 834         .space                  = 0,
 835         .user_irq_refcount      = 0,
 836         .irq_gem_seqno          = 0,
 837         .waiting_gem_seqno      = 0,
 838         .setup_status_page      = bsd_setup_status_page,
 839         .init                   = init_bsd_ring,
 840         .get_head               = bsd_ring_get_head,
 841         .get_tail               = bsd_ring_get_tail,
 842         .get_active_head        = bsd_ring_get_active_head,
 843         .advance_ring           = bsd_ring_advance_ring,
 844         .flush                  = bsd_ring_flush,
 845         .add_request            = bsd_ring_add_request,
 846         .get_gem_seqno          = bsd_ring_get_gem_seqno,
 847         .user_irq_get           = bsd_ring_get_user_irq,
 848         .user_irq_put           = bsd_ring_put_user_irq,
 849         .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer,
 850         .status_page            = {NULL, 0, NULL},
 851         .map                    = {0,}
 852 };