joy/android_kernel_sony_sdm845
Watch
1
Fork
You've already forked android_kernel_sony_sdm845
 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge tag 'drm-intel-next-2016-08-08' of git://anongit.freedesktop.org/drm-intel into drm-next

Browse source 
- refactor ddi buffer programming a bit (Ville)
- large-scale renaming to untangle naming in the gem code (Chris)
- rework vma/active tracking for accurately reaping idle mappings of shared
  objects (Chris)
- misc dp sst/mst probing corner case fixes (Ville)
- tons of cleanup&tunings all around in gem
- lockless (rcu-protected) request lookup, plus use it everywhere for
  non(b)locking waits (Chris)
- pipe crc debugfs fixes (Rodrigo)
- random fixes all over

* tag 'drm-intel-next-2016-08-08' of git://anongit.freedesktop.org/drm-intel: (222 commits)
  drm/i915: Update DRIVER_DATE to 20160808
  drm/i915: fix aliasing_ppgtt leak
  drm/i915: Update comment before i915_spin_request
  drm/i915: Use drm official vblank_no_hw_counter callback.
  drm/i915: Fix copy_to_user usage for pipe_crc
  Revert "drm/i915: Track active streams also for DP SST"
  drm/i915: fix WaInsertDummyPushConstPs
  drm/i915: Assert that the request hasn't been retired
  drm/i915: Repack fence tiling mode and stride into a single integer
  drm/i915: Document and reject invalid tiling modes
  drm/i915: Remove locking for get_tiling
  drm/i915: Remove pinned check from madvise ioctl
  drm/i915: Reduce locking inside swfinish ioctl
  drm/i915: Remove (struct_mutex) locking for busy-ioctl
  drm/i915: Remove (struct_mutex) locking for wait-ioctl
  drm/i915: Do a nonblocking wait first in pread/pwrite
  drm/i915: Remove unused no-shrinker-steal
  drm/i915: Tidy generation of the GTT mmap offset
  drm/i915/shrinker: Wait before acquiring struct_mutex under oom
  drm/i915: Simplify do_idling() (Ironlake vt-d w/a)
  ...
This commit is contained in:
Dave Airlie 2016-08-15 16:53:57 +10:00
commit fc93ff608b
62 changed files with 6280 additions and 5999 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Documentation/gpu/i915.rst
View file

@ -70,6 +70,9 @@ Frontbuffer Tracking
.. kernel-doc:: drivers/gpu/drm/i915/intel_frontbuffer.c
:doc: frontbuffer tracking
.. kernel-doc:: drivers/gpu/drm/i915/intel_frontbuffer.h
:internal:
.. kernel-doc:: drivers/gpu/drm/i915/intel_frontbuffer.c
:internal:

3
drivers/gpu/drm/i915/Makefile
View file

@ -25,7 +25,6 @@ i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o
i915-y += i915_cmd_parser.o \
i915_gem_batch_pool.o \
i915_gem_context.o \
i915_gem_debug.o \
i915_gem_dmabuf.o \
i915_gem_evict.o \
i915_gem_execbuffer.o \
@ -33,6 +32,7 @@ i915-y += i915_cmd_parser.o \
i915_gem_gtt.o \
i915_gem.o \
i915_gem_render_state.o \
i915_gem_request.o \
i915_gem_shrinker.o \
i915_gem_stolen.o \
i915_gem_tiling.o \
@ -40,6 +40,7 @@ i915-y += i915_cmd_parser.o \
i915_gpu_error.o \
i915_trace_points.o \
intel_breadcrumbs.o \
intel_engine_cs.o \
intel_lrc.o \
intel_mocs.o \
intel_ringbuffer.o \

74
drivers/gpu/drm/i915/i915_cmd_parser.c
View file

@ -62,23 +62,23 @@
* The parser always rejects such commands.
*
* The majority of the problematic commands fall in the MI_* range, with only a
* few specific commands on each ring (e.g. PIPE_CONTROL and MI_FLUSH_DW).
* few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW).
*
* Implementation:
* Each ring maintains tables of commands and registers which the parser uses in
* scanning batch buffers submitted to that ring.
* Each engine maintains tables of commands and registers which the parser
* uses in scanning batch buffers submitted to that engine.
*
* Since the set of commands that the parser must check for is significantly
* smaller than the number of commands supported, the parser tables contain only
* those commands required by the parser. This generally works because command
* opcode ranges have standard command length encodings. So for commands that
* the parser does not need to check, it can easily skip them. This is
* implemented via a per-ring length decoding vfunc.
* implemented via a per-engine length decoding vfunc.
*
* Unfortunately, there are a number of commands that do not follow the standard
* length encoding for their opcode range, primarily amongst the MI_* commands.
* To handle this, the parser provides a way to define explicit "skip" entries
* in the per-ring command tables.
* in the per-engine command tables.
*
* Other command table entries map fairly directly to high level categories
* mentioned above: rejected, master-only, register whitelist. The parser
@ -603,7 +603,7 @@ static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)
return 0;
}
static bool validate_cmds_sorted(struct intel_engine_cs *engine,
static bool validate_cmds_sorted(const struct intel_engine_cs *engine,
const struct drm_i915_cmd_table *cmd_tables,
int cmd_table_count)
{
@ -624,8 +624,10 @@ static bool validate_cmds_sorted(struct intel_engine_cs *engine,
u32 curr = desc->cmd.value & desc->cmd.mask;
if (curr < previous) {
DRM_ERROR("CMD: table not sorted ring=%d table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
engine->id, i, j, curr, previous);
DRM_ERROR("CMD: %s [%d] command table not sorted: "
"table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
engine->name, engine->id,
i, j, curr, previous);
ret = false;
}
@ -636,7 +638,7 @@ static bool validate_cmds_sorted(struct intel_engine_cs *engine,
return ret;
}
static bool check_sorted(int ring_id,
static bool check_sorted(const struct intel_engine_cs *engine,
const struct drm_i915_reg_descriptor *reg_table,
int reg_count)
{
@ -648,8 +650,10 @@ static bool check_sorted(int ring_id,
u32 curr = i915_mmio_reg_offset(reg_table[i].addr);
if (curr < previous) {
DRM_ERROR("CMD: table not sorted ring=%d entry=%d reg=0x%08X prev=0x%08X\n",
ring_id, i, curr, previous);
DRM_ERROR("CMD: %s [%d] register table not sorted: "
"entry=%d reg=0x%08X prev=0x%08X\n",
engine->name, engine->id,
i, curr, previous);
ret = false;
}
@ -666,7 +670,7 @@ static bool validate_regs_sorted(struct intel_engine_cs *engine)
for (i = 0; i < engine->reg_table_count; i++) {
table = &engine->reg_tables[i];
if (!check_sorted(engine->id, table->regs, table->num_regs))
if (!check_sorted(engine, table->regs, table->num_regs))
return false;
}
@ -736,7 +740,7 @@ static void fini_hash_table(struct intel_engine_cs *engine)
}
/**
* i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer
* intel_engine_init_cmd_parser() - set cmd parser related fields for an engine
* @engine: the engine to initialize
*
* Optionally initializes fields related to batch buffer command parsing in the
@ -745,7 +749,7 @@ static void fini_hash_table(struct intel_engine_cs *engine)
*
* Return: non-zero if initialization fails
*/
int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
int intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
{
const struct drm_i915_cmd_table *cmd_tables;
int cmd_table_count;
@ -806,8 +810,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
break;
default:
DRM_ERROR("CMD: cmd_parser_init with unknown ring: %d\n",
engine->id);
MISSING_CASE(engine->id);
BUG();
}
@ -829,13 +832,13 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
}
/**
* i915_cmd_parser_fini_ring() - clean up cmd parser related fields
* intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields
* @engine: the engine to clean up
*
* Releases any resources related to command parsing that may have been
* initialized for the specified ring.
* initialized for the specified engine.
*/
void i915_cmd_parser_fini_ring(struct intel_engine_cs *engine)
void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine)
{
if (!engine->needs_cmd_parser)
return;
@ -866,9 +869,9 @@ find_cmd_in_table(struct intel_engine_cs *engine,
* Returns a pointer to a descriptor for the command specified by cmd_header.
*
* The caller must supply space for a default descriptor via the default_desc
* parameter. If no descriptor for the specified command exists in the ring's
* parameter. If no descriptor for the specified command exists in the engine's
* command parser tables, this function fills in default_desc based on the
* ring's default length encoding and returns default_desc.
* engine's default length encoding and returns default_desc.
*/
static const struct drm_i915_cmd_descriptor*
find_cmd(struct intel_engine_cs *engine,
@ -1023,15 +1026,16 @@ unpin_src:
}
/**
* i915_needs_cmd_parser() - should a given ring use software command parsing?
* intel_engine_needs_cmd_parser() - should a given engine use software
* command parsing?
* @engine: the engine in question
*
* Only certain platforms require software batch buffer command parsing, and
* only when enabled via module parameter.
*
* Return: true if the ring requires software command parsing
* Return: true if the engine requires software command parsing
*/
bool i915_needs_cmd_parser(struct intel_engine_cs *engine)
bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine)
{
if (!engine->needs_cmd_parser)
return false;
@ -1078,8 +1082,8 @@ static bool check_cmd(const struct intel_engine_cs *engine,
reg_addr);
if (!reg) {
DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (ring=%d)\n",
reg_addr, *cmd, engine->id);
DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (exec_id=%d)\n",
reg_addr, *cmd, engine->exec_id);
return false;
}
@ -1159,11 +1163,11 @@ static bool check_cmd(const struct intel_engine_cs *engine,
desc->bits[i].mask;
if (dword != desc->bits[i].expected) {
DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (ring=%d)\n",
DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (exec_id=%d)\n",
*cmd,
desc->bits[i].mask,
desc->bits[i].expected,
dword, engine->id);
dword, engine->exec_id);
return false;
}
}
@ -1189,12 +1193,12 @@ static bool check_cmd(const struct intel_engine_cs *engine,
* Return: non-zero if the parser finds violations or otherwise fails; -EACCES
* if the batch appears legal but should use hardware parsing
*/
int i915_parse_cmds(struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj,
struct drm_i915_gem_object *shadow_batch_obj,
u32 batch_start_offset,
u32 batch_len,
bool is_master)
int intel_engine_cmd_parser(struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj,
struct drm_i915_gem_object *shadow_batch_obj,
u32 batch_start_offset,
u32 batch_len,
bool is_master)
{
u32 *cmd, *batch_base, *batch_end;
struct drm_i915_cmd_descriptor default_desc = { 0 };
@ -1295,7 +1299,7 @@ int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
/* If the command parser is not enabled, report 0 - unsupported */
for_each_engine(engine, dev_priv) {
if (i915_needs_cmd_parser(engine)) {
if (intel_engine_needs_cmd_parser(engine)) {
active = true;
break;
}

189
drivers/gpu/drm/i915/i915_debugfs.c
View file

@ -91,7 +91,7 @@ static int i915_capabilities(struct seq_file *m, void *data)
static char get_active_flag(struct drm_i915_gem_object *obj)
{
return obj->active ? '*' : ' ';
return i915_gem_object_is_active(obj) ? '*' : ' ';
}
static char get_pin_flag(struct drm_i915_gem_object *obj)
@ -101,7 +101,7 @@ static char get_pin_flag(struct drm_i915_gem_object *obj)
static char get_tiling_flag(struct drm_i915_gem_object *obj)
{
switch (obj->tiling_mode) {
switch (i915_gem_object_get_tiling(obj)) {
default:
case I915_TILING_NONE: return ' ';
case I915_TILING_X: return 'X';
@ -125,7 +125,7 @@ static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
struct i915_vma *vma;
list_for_each_entry(vma, &obj->vma_list, obj_link) {
if (vma->is_ggtt && drm_mm_node_allocated(&vma->node))
if (i915_vma_is_ggtt(vma) && drm_mm_node_allocated(&vma->node))
size += vma->node.size;
}
@ -138,6 +138,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct intel_engine_cs *engine;
struct i915_vma *vma;
unsigned int frontbuffer_bits;
int pin_count = 0;
enum intel_engine_id id;
@ -155,17 +156,20 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
obj->base.write_domain);
for_each_engine_id(engine, dev_priv, id)
seq_printf(m, "%x ",
i915_gem_request_get_seqno(obj->last_read_req[id]));
i915_gem_active_get_seqno(&obj->last_read[id],
&obj->base.dev->struct_mutex));
seq_printf(m, "] %x %x%s%s%s",
i915_gem_request_get_seqno(obj->last_write_req),
i915_gem_request_get_seqno(obj->last_fenced_req),
i915_gem_active_get_seqno(&obj->last_write,
&obj->base.dev->struct_mutex),
i915_gem_active_get_seqno(&obj->last_fence,
&obj->base.dev->struct_mutex),
i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
obj->dirty ? " dirty" : "",
obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
if (obj->base.name)
seq_printf(m, " (name: %d)", obj->base.name);
list_for_each_entry(vma, &obj->vma_list, obj_link) {
if (vma->pin_count > 0)
if (i915_vma_is_pinned(vma))
pin_count++;
}
seq_printf(m, " (pinned x %d)", pin_count);
@ -174,10 +178,13 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
if (obj->fence_reg != I915_FENCE_REG_NONE)
seq_printf(m, " (fence: %d)", obj->fence_reg);
list_for_each_entry(vma, &obj->vma_list, obj_link) {
if (!drm_mm_node_allocated(&vma->node))
continue;
seq_printf(m, " (%sgtt offset: %08llx, size: %08llx",
vma->is_ggtt ? "g" : "pp",
i915_vma_is_ggtt(vma) ? "g" : "pp",
vma->node.start, vma->node.size);
if (vma->is_ggtt)
if (i915_vma_is_ggtt(vma))
seq_printf(m, ", type: %u", vma->ggtt_view.type);
seq_puts(m, ")");
}
@ -192,11 +199,15 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
*t = '\0';
seq_printf(m, " (%s mappable)", s);
}
if (obj->last_write_req != NULL)
seq_printf(m, " (%s)",
i915_gem_request_get_engine(obj->last_write_req)->name);
if (obj->frontbuffer_bits)
seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits);
engine = i915_gem_active_get_engine(&obj->last_write,
&obj->base.dev->struct_mutex);
if (engine)
seq_printf(m, " (%s)", engine->name);
frontbuffer_bits = atomic_read(&obj->frontbuffer_bits);
if (frontbuffer_bits)
seq_printf(m, " (frontbuffer: 0x%03x)", frontbuffer_bits);
}
static int i915_gem_object_list_info(struct seq_file *m, void *data)
@ -338,47 +349,30 @@ static int per_file_stats(int id, void *ptr, void *data)
stats->count++;
stats->total += obj->base.size;
if (!obj->bind_count)
stats->unbound += obj->base.size;
if (obj->base.name || obj->base.dma_buf)
stats->shared += obj->base.size;
if (USES_FULL_PPGTT(obj->base.dev)) {
list_for_each_entry(vma, &obj->vma_list, obj_link) {
struct i915_hw_ppgtt *ppgtt;
list_for_each_entry(vma, &obj->vma_list, obj_link) {
if (!drm_mm_node_allocated(&vma->node))
continue;
if (!drm_mm_node_allocated(&vma->node))
if (i915_vma_is_ggtt(vma)) {
stats->global += vma->node.size;
} else {
struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vma->vm);
if (ppgtt->base.file != stats->file_priv)
continue;
if (vma->is_ggtt) {
stats->global += obj->base.size;
continue;
}
ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base);
if (ppgtt->file_priv != stats->file_priv)
continue;
if (obj->active) /* XXX per-vma statistic */
stats->active += obj->base.size;
else
stats->inactive += obj->base.size;
return 0;
}
} else {
if (i915_gem_obj_ggtt_bound(obj)) {
stats->global += obj->base.size;
if (obj->active)
stats->active += obj->base.size;
else
stats->inactive += obj->base.size;
return 0;
}
if (i915_vma_is_active(vma))
stats->active += vma->node.size;
else
stats->inactive += vma->node.size;
}
if (!list_empty(&obj->global_list))
stats->unbound += obj->base.size;
return 0;
}
@ -425,8 +419,8 @@ static int per_file_ctx_stats(int id, void *ptr, void *data)
for (n = 0; n < ARRAY_SIZE(ctx->engine); n++) {
if (ctx->engine[n].state)
per_file_stats(0, ctx->engine[n].state, data);
if (ctx->engine[n].ringbuf)
per_file_stats(0, ctx->engine[n].ringbuf->obj, data);
if (ctx->engine[n].ring)
per_file_stats(0, ctx->engine[n].ring->obj, data);
}
return 0;
@ -754,13 +748,13 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
int count;
count = 0;
list_for_each_entry(req, &engine->request_list, list)
list_for_each_entry(req, &engine->request_list, link)
count++;
if (count == 0)
continue;
seq_printf(m, "%s requests: %d\n", engine->name, count);
list_for_each_entry(req, &engine->request_list, list) {
list_for_each_entry(req, &engine->request_list, link) {
struct task_struct *task;
rcu_read_lock();
@ -768,7 +762,7 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
if (req->pid)
task = pid_task(req->pid, PIDTYPE_PID);
seq_printf(m, " %x @ %d: %s [%d]\n",
req->seqno,
req->fence.seqno,
(int) (jiffies - req->emitted_jiffies),
task ? task->comm : "<unknown>",
task ? task->pid : -1);
@ -1205,8 +1199,6 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
intel_runtime_pm_get(dev_priv);
flush_delayed_work(&dev_priv->rps.delayed_resume_work);
if (IS_GEN5(dev)) {
u16 rgvswctl = I915_READ16(MEMSWCTL);
u16 rgvstat = I915_READ16(MEMSTAT_ILK);
@ -1381,6 +1373,8 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq));
seq_printf(m, "Min freq: %d MHz\n",
intel_gpu_freq(dev_priv, dev_priv->rps.min_freq));
seq_printf(m, "Boost freq: %d MHz\n",
intel_gpu_freq(dev_priv, dev_priv->rps.boost_freq));
seq_printf(m, "Max freq: %d MHz\n",
intel_gpu_freq(dev_priv, dev_priv->rps.max_freq));
seq_printf(m,
@ -1419,7 +1413,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
intel_runtime_pm_get(dev_priv);
for_each_engine_id(engine, dev_priv, id) {
acthd[id] = intel_ring_get_active_head(engine);
acthd[id] = intel_engine_get_active_head(engine);
seqno[id] = intel_engine_get_seqno(engine);
}
@ -1602,6 +1596,7 @@ static int gen6_drpc_info(struct seq_file *m)
struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = to_i915(dev);
u32 rpmodectl1, gt_core_status, rcctl1, rc6vids = 0;
u32 gen9_powergate_enable = 0, gen9_powergate_status = 0;
unsigned forcewake_count;
int count = 0, ret;
@ -1629,6 +1624,10 @@ static int gen6_drpc_info(struct seq_file *m)
rpmodectl1 = I915_READ(GEN6_RP_CONTROL);
rcctl1 = I915_READ(GEN6_RC_CONTROL);
if (INTEL_INFO(dev)->gen >= 9) {
gen9_powergate_enable = I915_READ(GEN9_PG_ENABLE);
gen9_powergate_status = I915_READ(GEN9_PWRGT_DOMAIN_STATUS);
}
mutex_unlock(&dev->struct_mutex);
mutex_lock(&dev_priv->rps.hw_lock);
sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
@ -1647,6 +1646,12 @@ static int gen6_drpc_info(struct seq_file *m)
yesno(rcctl1 & GEN6_RC_CTL_RC1e_ENABLE));
seq_printf(m, "RC6 Enabled: %s\n",
yesno(rcctl1 & GEN6_RC_CTL_RC6_ENABLE));
if (INTEL_INFO(dev)->gen >= 9) {
seq_printf(m, "Render Well Gating Enabled: %s\n",
yesno(gen9_powergate_enable & GEN9_RENDER_PG_ENABLE));
seq_printf(m, "Media Well Gating Enabled: %s\n",
yesno(gen9_powergate_enable & GEN9_MEDIA_PG_ENABLE));
}
seq_printf(m, "Deep RC6 Enabled: %s\n",
yesno(rcctl1 & GEN6_RC_CTL_RC6p_ENABLE));
seq_printf(m, "Deepest RC6 Enabled: %s\n",
@ -1675,6 +1680,14 @@ static int gen6_drpc_info(struct seq_file *m)
seq_printf(m, "Core Power Down: %s\n",
yesno(gt_core_status & GEN6_CORE_CPD_STATE_MASK));
if (INTEL_INFO(dev)->gen >= 9) {
seq_printf(m, "Render Power Well: %s\n",
(gen9_powergate_status &
GEN9_PWRGT_RENDER_STATUS_MASK) ? "Up" : "Down");
seq_printf(m, "Media Power Well: %s\n",
(gen9_powergate_status &
GEN9_PWRGT_MEDIA_STATUS_MASK) ? "Up" : "Down");
}
/* Not exactly sure what this is */
seq_printf(m, "RC6 \"Locked to RPn\" residency since boot: %u\n",
@ -1692,7 +1705,7 @@ static int gen6_drpc_info(struct seq_file *m)
GEN6_DECODE_RC6_VID(((rc6vids >> 8) & 0xff)));
seq_printf(m, "RC6++ voltage: %dmV\n",
GEN6_DECODE_RC6_VID(((rc6vids >> 16) & 0xff)));
return 0;
return i915_forcewake_domains(m, NULL);
}
static int i915_drpc_info(struct seq_file *m, void *unused)
@ -1896,8 +1909,6 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
intel_runtime_pm_get(dev_priv);
flush_delayed_work(&dev_priv->rps.delayed_resume_work);
ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
if (ret)
goto out;
@ -2019,12 +2030,11 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
return 0;
}
static void describe_ctx_ringbuf(struct seq_file *m,
struct intel_ringbuffer *ringbuf)
static void describe_ctx_ring(struct seq_file *m, struct intel_ring *ring)
{
seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, last head: %d)",
ringbuf->space, ringbuf->head, ringbuf->tail,
ringbuf->last_retired_head);
ring->space, ring->head, ring->tail,
ring->last_retired_head);
}
static int i915_context_status(struct seq_file *m, void *unused)
@ -2068,8 +2078,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
seq_putc(m, ce->initialised ? 'I' : 'i');
if (ce->state)
describe_obj(m, ce->state);
if (ce->ringbuf)
describe_ctx_ringbuf(m, ce->ringbuf);
if (ce->ring)
describe_ctx_ring(m, ce->ring);
seq_putc(m, '\n');
}
@ -2467,13 +2477,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
list_empty(&file_priv->rps.link) ? "" : ", active");
rcu_read_unlock();
}
seq_printf(m, "Semaphore boosts: %d%s\n",
dev_priv->rps.semaphores.boosts,
list_empty(&dev_priv->rps.semaphores.link) ? "" : ", active");
seq_printf(m, "MMIO flip boosts: %d%s\n",
dev_priv->rps.mmioflips.boosts,
list_empty(&dev_priv->rps.mmioflips.link) ? "" : ", active");
seq_printf(m, "Kernel boosts: %d\n", dev_priv->rps.boosts);
seq_printf(m, "Kernel (anonymous) boosts: %d\n", dev_priv->rps.boosts);
spin_unlock(&dev_priv->rps.client_lock);
mutex_unlock(&dev->filelist_mutex);
@ -3228,7 +3232,7 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
enum intel_engine_id id;
int j, ret;
if (!i915_semaphore_is_enabled(dev_priv)) {
if (!i915.semaphores) {
seq_puts(m, "Semaphores are disabled\n");
return 0;
}
@ -3621,7 +3625,6 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
while (n_entries > 0) {
struct intel_pipe_crc_entry *entry =
&pipe_crc->entries[pipe_crc->tail];
int ret;
if (CIRC_CNT(pipe_crc->head, pipe_crc->tail,
INTEL_PIPE_CRC_ENTRIES_NR) < 1)
@ -3638,8 +3641,7 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
spin_unlock_irq(&pipe_crc->lock);
ret = copy_to_user(user_buf, buf, PIPE_CRC_LINE_LEN);
if (ret == PIPE_CRC_LINE_LEN)
if (copy_to_user(user_buf, buf, PIPE_CRC_LINE_LEN))
return -EFAULT;
user_buf += PIPE_CRC_LINE_LEN;
@ -4921,7 +4923,7 @@ i915_drop_caches_set(void *data, u64 val)
return ret;
if (val & DROP_ACTIVE) {
ret = i915_gem_wait_for_idle(dev_priv);
ret = i915_gem_wait_for_idle(dev_priv, true);
if (ret)
goto unlock;
}
@ -4950,20 +4952,11 @@ i915_max_freq_get(void *data, u64 *val)
{
struct drm_device *dev = data;
struct drm_i915_private *dev_priv = to_i915(dev);
int ret;
if (INTEL_INFO(dev)->gen < 6)
return -ENODEV;
flush_delayed_work(&dev_priv->rps.delayed_resume_work);
ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
if (ret)
return ret;
*val = intel_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit);
mutex_unlock(&dev_priv->rps.hw_lock);
return 0;
}
@ -4978,8 +4971,6 @@ i915_max_freq_set(void *data, u64 val)
if (INTEL_INFO(dev)->gen < 6)
return -ENODEV;
flush_delayed_work(&dev_priv->rps.delayed_resume_work);
DRM_DEBUG_DRIVER("Manually setting max freq to %llu\n", val);
ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
@ -5017,20 +5008,11 @@ i915_min_freq_get(void *data, u64 *val)
{
struct drm_device *dev = data;
struct drm_i915_private *dev_priv = to_i915(dev);
int ret;
if (INTEL_INFO(dev)->gen < 6)
if (INTEL_GEN(dev_priv) < 6)
return -ENODEV;
flush_delayed_work(&dev_priv->rps.delayed_resume_work);
ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
if (ret)
return ret;
*val = intel_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit);
mutex_unlock(&dev_priv->rps.hw_lock);
return 0;
}
@ -5042,11 +5024,9 @@ i915_min_freq_set(void *data, u64 val)
u32 hw_max, hw_min;
int ret;
if (INTEL_INFO(dev)->gen < 6)
if (INTEL_GEN(dev_priv) < 6)
return -ENODEV;
flush_delayed_work(&dev_priv->rps.delayed_resume_work);
DRM_DEBUG_DRIVER("Manually setting min freq to %llu\n", val);
ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
@ -5268,7 +5248,8 @@ static void broadwell_sseu_device_status(struct drm_device *dev,
static int i915_sseu_status(struct seq_file *m, void *unused)
{
struct drm_info_node *node = (struct drm_info_node *) m->private;
struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = to_i915(node->minor->dev);
struct drm_device *dev = &dev_priv->drm;
struct sseu_dev_status stat;
if (INTEL_INFO(dev)->gen < 8)
@ -5298,6 +5279,9 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
seq_puts(m, "SSEU Device Status\n");
memset(&stat, 0, sizeof(stat));
intel_runtime_pm_get(dev_priv);
if (IS_CHERRYVIEW(dev)) {
cherryview_sseu_device_status(dev, &stat);
} else if (IS_BROADWELL(dev)) {
@ -5305,6 +5289,9 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
} else if (INTEL_INFO(dev)->gen >= 9) {
gen9_sseu_device_status(dev, &stat);
}
intel_runtime_pm_put(dev_priv);
seq_printf(m, " Enabled Slice Total: %u\n",
stat.slice_total);
seq_printf(m, " Enabled Subslice Total: %u\n",

85
drivers/gpu/drm/i915/i915_drv.c
View file

@ -228,27 +228,6 @@ static void intel_detect_pch(struct drm_device *dev)
pci_dev_put(pch);
}
bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv)
{
if (INTEL_GEN(dev_priv) < 6)
return false;
if (i915.semaphores >= 0)
return i915.semaphores;
/* TODO: make semaphores and Execlists play nicely together */
if (i915.enable_execlists)
return false;
#ifdef CONFIG_INTEL_IOMMU
/* Enable semaphores on SNB when IO remapping is off */
if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped)
return false;
#endif
return true;
}
static int i915_getparam(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
@ -324,7 +303,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
value = 1;
break;
case I915_PARAM_HAS_SEMAPHORES:
value = i915_semaphore_is_enabled(dev_priv);
value = i915.semaphores;
break;
case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
value = 1;
@ -999,6 +978,9 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
i915.enable_ppgtt =
intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores));
}
/**
@ -1011,8 +993,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
{
struct drm_device *dev = &dev_priv->drm;
struct i915_ggtt *ggtt = &dev_priv->ggtt;
uint32_t aperture_size;
int ret;
if (i915_inject_load_failure())
@ -1022,16 +1002,10 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
intel_sanitize_options(dev_priv);
ret = i915_ggtt_init_hw(dev);
ret = i915_ggtt_probe_hw(dev_priv);
if (ret)
return ret;
ret = i915_ggtt_enable_hw(dev);
if (ret) {
DRM_ERROR("failed to enable GGTT\n");
goto out_ggtt;
}
/* WARNING: Apparently we must kick fbdev drivers before vgacon,
* otherwise the vga fbdev driver falls over. */
ret = i915_kick_out_firmware_fb(dev_priv);
@ -1046,6 +1020,16 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
goto out_ggtt;
}
ret = i915_ggtt_init_hw(dev_priv);
if (ret)
return ret;
ret = i915_ggtt_enable_hw(dev_priv);
if (ret) {
DRM_ERROR("failed to enable GGTT\n");
goto out_ggtt;
}
pci_set_master(dev->pdev);
/* overlay on gen2 is broken and can't address above 1G */
@ -1058,7 +1042,6 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
}
}
/* 965GM sometimes incorrectly writes to hardware status page (HWS)
* using 32bit addressing, overwriting memory if HWS is located
* above 4GB.
@ -1077,19 +1060,6 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
}
}
aperture_size = ggtt->mappable_end;
ggtt->mappable =
io_mapping_create_wc(ggtt->mappable_base,
aperture_size);
if (!ggtt->mappable) {
ret = -EIO;
goto out_ggtt;
}
ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
aperture_size);
pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
PM_QOS_DEFAULT_VALUE);
@ -1118,7 +1088,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
return 0;
out_ggtt:
i915_ggtt_cleanup_hw(dev);
i915_ggtt_cleanup_hw(dev_priv);
return ret;
}
@ -1130,15 +1100,12 @@ out_ggtt:
static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
{
struct drm_device *dev = &dev_priv->drm;
struct i915_ggtt *ggtt = &dev_priv->ggtt;
if (dev->pdev->msi_enabled)
pci_disable_msi(dev->pdev);
pm_qos_remove_request(&dev_priv->pm_qos);
arch_phys_wc_del(ggtt->mtrr);
io_mapping_free(ggtt->mappable);
i915_ggtt_cleanup_hw(dev);
i915_ggtt_cleanup_hw(dev_priv);
}
/**
@ -1343,7 +1310,7 @@ void i915_driver_unload(struct drm_device *dev)
i915_destroy_error_state(dev);
/* Flush any outstanding unpin_work. */
flush_workqueue(dev_priv->wq);
drain_workqueue(dev_priv->wq);
intel_guc_fini(dev);
i915_gem_fini(dev);
@ -1458,8 +1425,6 @@ static int i915_drm_suspend(struct drm_device *dev)
intel_guc_suspend(dev);
intel_suspend_gt_powersave(dev_priv);
intel_display_suspend(dev);
intel_dp_mst_suspend(dev);
@ -1586,15 +1551,13 @@ static int i915_drm_resume(struct drm_device *dev)
disable_rpm_wakeref_asserts(dev_priv);
ret = i915_ggtt_enable_hw(dev);
ret = i915_ggtt_enable_hw(dev_priv);
if (ret)
DRM_ERROR("failed to re-enable GGTT\n");
intel_csr_ucode_resume(dev_priv);
mutex_lock(&dev->struct_mutex);
i915_gem_restore_gtt_mappings(dev);
mutex_unlock(&dev->struct_mutex);
i915_gem_resume(dev);
i915_restore_state(dev);
intel_opregion_setup(dev_priv);
@ -1652,6 +1615,7 @@ static int i915_drm_resume(struct drm_device *dev)
intel_opregion_notify_adapter(dev_priv, PCI_D0);
intel_autoenable_gt_powersave(dev_priv);
drm_kms_helper_poll_enable(dev);
enable_rpm_wakeref_asserts(dev_priv);
@ -1778,8 +1742,6 @@ int i915_reset(struct drm_i915_private *dev_priv)
unsigned reset_counter;
int ret;
intel_reset_gt_powersave(dev_priv);
mutex_lock(&dev->struct_mutex);
/* Clear any previous failed attempts at recovery. Time to try again. */
@ -1835,8 +1797,7 @@ int i915_reset(struct drm_i915_private *dev_priv)
* previous concerns that it doesn't respond well to some forms
* of re-init after reset.
*/
if (INTEL_INFO(dev)->gen > 5)
intel_enable_gt_powersave(dev_priv);
intel_autoenable_gt_powersave(dev_priv);
return 0;
@ -2462,7 +2423,6 @@ static int intel_runtime_resume(struct device *device)
* we can do is to hope that things will still work (and disable RPM).
*/
i915_gem_init_swizzling(dev);
gen6_update_ring_freq(dev_priv);
intel_runtime_pm_enable_interrupts(dev_priv);
@ -2618,6 +2578,7 @@ static struct drm_driver driver = {
.postclose = i915_driver_postclose,
.set_busid = drm_pci_set_busid,
.gem_close_object = i915_gem_close_object,
.gem_free_object = i915_gem_free_object,
.gem_vm_ops = &i915_gem_vm_ops,

515
drivers/gpu/drm/i915/i915_drv.h
View file

@ -61,6 +61,7 @@
#include "i915_gem.h"
#include "i915_gem_gtt.h"
#include "i915_gem_render_state.h"
#include "i915_gem_request.h"
#include "intel_gvt.h"
@ -69,7 +70,7 @@
#define DRIVER_NAME "i915"
#define DRIVER_DESC "Intel Graphics"
#define DRIVER_DATE "20160711"
#define DRIVER_DATE "20160808"
#undef WARN_ON
/* Many gcc seem to no see through this and fall over :( */
@ -401,7 +402,7 @@ struct drm_i915_file_private {
unsigned boosts;
} rps;
unsigned int bsd_ring;
unsigned int bsd_engine;
};
/* Used by dp and fdi links */
@ -431,8 +432,6 @@ void intel_link_compute_m_n(int bpp, int nlanes,
#define DRIVER_MINOR 6
#define DRIVER_PATCHLEVEL 0
#define WATCH_LISTS 0
struct opregion_header;
struct opregion_acpi;
struct opregion_swsci;
@ -511,13 +510,13 @@ struct drm_i915_error_state {
struct intel_display_error_state *display;
struct drm_i915_error_object *semaphore_obj;
struct drm_i915_error_ring {
bool valid;
struct drm_i915_error_engine {
int engine_id;
/* Software tracked state */
bool waiting;
int num_waiters;
int hangcheck_score;
enum intel_ring_hangcheck_action hangcheck_action;
enum intel_engine_hangcheck_action hangcheck_action;
int num_requests;
/* our own tracking of ring head and tail */
@ -577,7 +576,7 @@ struct drm_i915_error_state {
pid_t pid;
char comm[TASK_COMM_LEN];
} ring[I915_NUM_ENGINES];
} engine[I915_NUM_ENGINES];
struct drm_i915_error_buffer {
u32 size;
@ -592,7 +591,7 @@ struct drm_i915_error_state {
u32 dirty:1;
u32 purgeable:1;
u32 userptr:1;
s32 ring:4;
s32 engine:4;
u32 cache_level:3;
} **active_bo, **pinned_bo;
@ -893,7 +892,7 @@ struct i915_gem_context {
struct intel_context {
struct drm_i915_gem_object *state;
struct intel_ringbuffer *ringbuf;
struct intel_ring *ring;
struct i915_vma *lrc_vma;
uint32_t *lrc_reg_state;
u64 lrc_desc;
@ -908,6 +907,7 @@ struct i915_gem_context {
struct list_head link;
u8 remap_slice;
bool closed:1;
};
enum fb_op_origin {
@ -1173,6 +1173,7 @@ struct intel_gen6_power_mgmt {
u8 max_freq_softlimit; /* Max frequency permitted by the driver */
u8 max_freq; /* Maximum frequency, RP0 if not overclocking */
u8 min_freq; /* AKA RPn. Minimum frequency */
u8 boost_freq; /* Frequency to request when wait boosting */
u8 idle_freq; /* Frequency to request when we are idle */
u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */
u8 rp1_freq; /* "less than" RP0 power/freqency */
@ -1190,11 +1191,9 @@ struct intel_gen6_power_mgmt {
bool client_boost;
bool enabled;
struct delayed_work delayed_resume_work;
struct delayed_work autoenable_work;
unsigned boosts;
struct intel_rps_client semaphores, mmioflips;
/* manual wa residency calculations */
struct intel_rps_ei up_ei, down_ei;
@ -1319,7 +1318,6 @@ struct i915_gem_mm {
struct notifier_block oom_notifier;
struct notifier_block vmap_notifier;
struct shrinker shrinker;
bool shrinker_no_lock_stealing;
/** LRU list of objects with fence regs on them. */
struct list_head fence_list;
@ -1331,7 +1329,7 @@ struct i915_gem_mm {
bool interruptible;
/* the indicator for dispatch video commands on two BSD rings */
unsigned int bsd_ring_dispatch_index;
unsigned int bsd_engine_dispatch_index;
/** Bit 6 swizzling required for X tiling */
uint32_t bit_6_swizzle_x;
@ -1670,7 +1668,7 @@ struct intel_pipe_crc {
};
struct i915_frontbuffer_tracking {
struct mutex lock;
spinlock_t lock;
/*
* Tracking bits for delayed frontbuffer flushing du to gpu activity or
@ -1705,18 +1703,6 @@ struct i915_virtual_gpu {
bool active;
};
struct i915_execbuffer_params {
struct drm_device *dev;
struct drm_file *file;
uint32_t dispatch_flags;
uint32_t args_batch_start_offset;
uint64_t batch_obj_vm_offset;
struct intel_engine_cs *engine;
struct drm_i915_gem_object *batch_obj;
struct i915_gem_context *ctx;
struct drm_i915_gem_request *request;
};
/* used in computing the new watermarks state */
struct intel_wm_config {
unsigned int num_pipes_active;
@ -1769,7 +1755,7 @@ struct drm_i915_private {
struct i915_gem_context *kernel_context;
struct intel_engine_cs engine[I915_NUM_ENGINES];
struct drm_i915_gem_object *semaphore_obj;
uint32_t last_seqno, next_seqno;
u32 next_seqno;
struct drm_dma_handle *status_page_dmah;
struct resource mch_res;
@ -2016,12 +2002,7 @@ struct drm_i915_private {
/* Abstract the submission mechanism (legacy ringbuffer or execlists) away */
struct {
int (*execbuf_submit)(struct i915_execbuffer_params *params,
struct drm_i915_gem_execbuffer2 *args,
struct list_head *vmas);
int (*init_engines)(struct drm_device *dev);
void (*cleanup_engine)(struct intel_engine_cs *engine);
void (*stop_engine)(struct intel_engine_cs *engine);
/**
* Is the GPU currently considered idle, or busy executing
@ -2144,8 +2125,6 @@ struct drm_i915_gem_object_ops {
*/
#define INTEL_MAX_SPRITE_BITS_PER_PIPE 5
#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8
#define INTEL_FRONTBUFFER_BITS \
(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES)
#define INTEL_FRONTBUFFER_PRIMARY(pipe) \
(1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
#define INTEL_FRONTBUFFER_CURSOR(pipe) \
@ -2169,18 +2148,21 @@ struct drm_i915_gem_object {
struct drm_mm_node *stolen;
struct list_head global_list;
struct list_head engine_list[I915_NUM_ENGINES];
/** Used in execbuf to temporarily hold a ref */
struct list_head obj_exec_link;
struct list_head batch_pool_link;
unsigned long flags;
/**
* This is set if the object is on the active lists (has pending
* rendering and so a non-zero seqno), and is not set if it i s on
* inactive (ready to be unbound) list.
*/
unsigned int active:I915_NUM_ENGINES;
#define I915_BO_ACTIVE_SHIFT 0
#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1)
#define __I915_BO_ACTIVE(bo) \
((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
/**
* This is set if the object has been written to since last bound
@ -2200,10 +2182,6 @@ struct drm_i915_gem_object {
*/
unsigned int madv:2;
/**
* Current tiling mode for the object.
*/
unsigned int tiling_mode:2;
/**
* Whether the tiling parameters for the currently associated fence
* register have changed. Note that for the purposes of tracking
@ -2234,9 +2212,17 @@ struct drm_i915_gem_object {
unsigned int cache_level:3;
unsigned int cache_dirty:1;
unsigned int frontbuffer_bits:INTEL_FRONTBUFFER_BITS;
atomic_t frontbuffer_bits;
/** Current tiling stride for the object, if it's tiled. */
unsigned int tiling_and_stride;
#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */
#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)
#define STRIDE_MASK (~TILING_MASK)
unsigned int has_wc_mmap;
/** Count of VMA actually bound by this object */
unsigned int bind_count;
unsigned int pin_display;
struct sg_table *pages;
@ -2256,14 +2242,10 @@ struct drm_i915_gem_object {
* requests on one ring where the write request is older than the
* read request. This allows for the CPU to read from an active
* buffer by only waiting for the write to complete.
* */
struct drm_i915_gem_request *last_read_req[I915_NUM_ENGINES];
struct drm_i915_gem_request *last_write_req;
/** Breadcrumb of last fenced GPU access to the buffer. */
struct drm_i915_gem_request *last_fenced_req;
/** Current tiling stride for the object, if it's tiled. */
uint32_t stride;
*/
struct i915_gem_active last_read[I915_NUM_ENGINES];
struct i915_gem_active last_write;
struct i915_gem_active last_fence;
/** References from framebuffers, locks out tiling changes. */
unsigned long framebuffer_references;
@ -2287,7 +2269,56 @@ struct drm_i915_gem_object {
} userptr;
};
};
#define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base)
static inline struct drm_i915_gem_object *
to_intel_bo(struct drm_gem_object *gem)
{
/* Assert that to_intel_bo(NULL) == NULL */
BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));
return container_of(gem, struct drm_i915_gem_object, base);
}
static inline struct drm_i915_gem_object *
i915_gem_object_lookup(struct drm_file *file, u32 handle)
{
return to_intel_bo(drm_gem_object_lookup(file, handle));
}
__deprecated
extern struct drm_gem_object *
drm_gem_object_lookup(struct drm_file *file, u32 handle);
__attribute__((nonnull))
static inline struct drm_i915_gem_object *
i915_gem_object_get(struct drm_i915_gem_object *obj)
{
drm_gem_object_reference(&obj->base);
return obj;
}
__deprecated
extern void drm_gem_object_reference(struct drm_gem_object *);
__attribute__((nonnull))
static inline void
i915_gem_object_put(struct drm_i915_gem_object *obj)
{
drm_gem_object_unreference(&obj->base);
}
__deprecated
extern void drm_gem_object_unreference(struct drm_gem_object *);
__attribute__((nonnull))
static inline void
i915_gem_object_put_unlocked(struct drm_i915_gem_object *obj)
{
drm_gem_object_unreference_unlocked(&obj->base);
}
__deprecated
extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
static inline bool
i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
@ -2295,6 +2326,55 @@ i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
}
static inline unsigned long
i915_gem_object_get_active(const struct drm_i915_gem_object *obj)
{
return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK;
}
static inline bool
i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
{
return i915_gem_object_get_active(obj);
}
static inline void
i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine)
{
obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT);
}
static inline void
i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine)
{
obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT);
}
static inline bool
i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
int engine)
{
return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
}
static inline unsigned int
i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
{
return obj->tiling_and_stride & TILING_MASK;
}
static inline bool
i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)
{
return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
}
static inline unsigned int
i915_gem_object_get_stride(struct drm_i915_gem_object *obj)
{
return obj->tiling_and_stride & STRIDE_MASK;
}
/*
* Optimised SGL iterator for GEM objects
*/
@ -2365,171 +2445,6 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg)
(((__iter).curr += PAGE_SIZE) < (__iter).max) || \
((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
/**
* Request queue structure.
*
* The request queue allows us to note sequence numbers that have been emitted
* and may be associated with active buffers to be retired.
*
* By keeping this list, we can avoid having to do questionable sequence
* number comparisons on buffer last_read|write_seqno. It also allows an
* emission time to be associated with the request for tracking how far ahead
* of the GPU the submission is.
*
* The requests are reference counted, so upon creation they should have an
* initial reference taken using kref_init
*/
struct drm_i915_gem_request {
struct kref ref;
/** On Which ring this request was generated */
struct drm_i915_private *i915;
struct intel_engine_cs *engine;
struct intel_signal_node signaling;
/** GEM sequence number associated with the previous request,
* when the HWS breadcrumb is equal to this the GPU is processing
* this request.
*/
u32 previous_seqno;
/** GEM sequence number associated with this request,
* when the HWS breadcrumb is equal or greater than this the GPU
* has finished processing this request.
*/
u32 seqno;
/** Position in the ringbuffer of the start of the request */
u32 head;
/**
* Position in the ringbuffer of the start of the postfix.
* This is required to calculate the maximum available ringbuffer
* space without overwriting the postfix.
*/
u32 postfix;
/** Position in the ringbuffer of the end of the whole request */
u32 tail;
/** Preallocate space in the ringbuffer for the emitting the request */
u32 reserved_space;
/**
* Context and ring buffer related to this request
* Contexts are refcounted, so when this request is associated with a
* context, we must increment the context's refcount, to guarantee that
* it persists while any request is linked to it. Requests themselves
* are also refcounted, so the request will only be freed when the last
* reference to it is dismissed, and the code in
* i915_gem_request_free() will then decrement the refcount on the
* context.
*/
struct i915_gem_context *ctx;
struct intel_ringbuffer *ringbuf;
/**
* Context related to the previous request.
* As the contexts are accessed by the hardware until the switch is
* completed to a new context, the hardware may still be writing
* to the context object after the breadcrumb is visible. We must
* not unpin/unbind/prune that object whilst still active and so
* we keep the previous context pinned until the following (this)
* request is retired.
*/
struct i915_gem_context *previous_context;
/** Batch buffer related to this request if any (used for
error state dump only) */
struct drm_i915_gem_object *batch_obj;
/** Time at which this request was emitted, in jiffies. */
unsigned long emitted_jiffies;
/** global list entry for this request */
struct list_head list;
struct drm_i915_file_private *file_priv;
/** file_priv list entry for this request */
struct list_head client_list;
/** process identifier submitting this request */
struct pid *pid;
/**
* The ELSP only accepts two elements at a time, so we queue
* context/tail pairs on a given queue (ring->execlist_queue) until the
* hardware is available. The queue serves a double purpose: we also use
* it to keep track of the up to 2 contexts currently in the hardware
* (usually one in execution and the other queued up by the GPU): We
* only remove elements from the head of the queue when the hardware
* informs us that an element has been completed.
*
* All accesses to the queue are mediated by a spinlock
* (ring->execlist_lock).
*/
/** Execlist link in the submission queue.*/
struct list_head execlist_link;
/** Execlists no. of times this request has been sent to the ELSP */
int elsp_submitted;
/** Execlists context hardware id. */
unsigned ctx_hw_id;
};
struct drm_i915_gem_request * __must_check
i915_gem_request_alloc(struct intel_engine_cs *engine,
struct i915_gem_context *ctx);
void i915_gem_request_free(struct kref *req_ref);
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
struct drm_file *file);
static inline uint32_t
i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
{
return req ? req->seqno : 0;
}
static inline struct intel_engine_cs *
i915_gem_request_get_engine(struct drm_i915_gem_request *req)
{
return req ? req->engine : NULL;
}
static inline struct drm_i915_gem_request *
i915_gem_request_reference(struct drm_i915_gem_request *req)
{
if (req)
kref_get(&req->ref);
return req;
}
static inline void
i915_gem_request_unreference(struct drm_i915_gem_request *req)
{
kref_put(&req->ref, i915_gem_request_free);
}
static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
struct drm_i915_gem_request *src)
{
if (src)
i915_gem_request_reference(src);
if (*pdst)
i915_gem_request_unreference(*pdst);
*pdst = src;
}
/*
* XXX: i915_gem_request_completed should be here but currently needs the
* definition of i915_seqno_passed() which is below. It will be moved in
* a later patch when the call to i915_seqno_passed() is obsoleted...
*/
/*
* A command that requires special handling by the command parser.
*/
@ -2617,8 +2532,9 @@ struct drm_i915_cmd_descriptor {
/*
* A table of commands requiring special handling by the command parser.
*
* Each ring has an array of tables. Each table consists of an array of command
* descriptors, which must be sorted with command opcodes in ascending order.
* Each engine has an array of tables. Each table consists of an array of
* command descriptors, which must be sorted with command opcodes in
* ascending order.
*/
struct drm_i915_cmd_table {
const struct drm_i915_cmd_descriptor *table;
@ -2932,6 +2848,8 @@ extern int i915_resume_switcheroo(struct drm_device *dev);
int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
int enable_ppgtt);
bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value);
/* i915_drv.c */
void __printf(3, 4)
__i915_printk(struct drm_i915_private *dev_priv, const char *level,
@ -3107,11 +3025,6 @@ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
void i915_gem_execbuffer_move_to_active(struct list_head *vmas,
struct drm_i915_gem_request *req);
int i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
struct drm_i915_gem_execbuffer2 *args,
struct list_head *vmas);
int i915_gem_execbuffer(struct drm_device *dev, void *data,
struct drm_file *file_priv);
int i915_gem_execbuffer2(struct drm_device *dev, void *data,
@ -3150,40 +3063,24 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
size_t size);
struct drm_i915_gem_object *i915_gem_object_create_from_data(
struct drm_device *dev, const void *data, size_t size);
void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
void i915_gem_free_object(struct drm_gem_object *obj);
void i915_gem_vma_destroy(struct i915_vma *vma);
/* Flags used by pin/bind&friends. */
#define PIN_MAPPABLE (1<<0)
#define PIN_NONBLOCK (1<<1)
#define PIN_GLOBAL (1<<2)
#define PIN_OFFSET_BIAS (1<<3)
#define PIN_USER (1<<4)
#define PIN_UPDATE (1<<5)
#define PIN_ZONE_4G (1<<6)
#define PIN_HIGH (1<<7)
#define PIN_OFFSET_FIXED (1<<8)
#define PIN_OFFSET_MASK (~4095)
int __must_check
i915_gem_object_pin(struct drm_i915_gem_object *obj,
struct i915_address_space *vm,
uint32_t alignment,
uint64_t flags);
int __must_check
i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
const struct i915_ggtt_view *view,
uint32_t alignment,
uint64_t flags);
u64 size,
u64 alignment,
u64 flags);
int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
u32 flags);
void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
int __must_check i915_vma_unbind(struct i915_vma *vma);
/*
* BEWARE: Do not use the function below unless you can _absolutely_
* _guarantee_ VMA in question is _not in use_ anywhere.
*/
int __must_check __i915_vma_unbind_no_wait(struct i915_vma *vma);
void i915_vma_close(struct i915_vma *vma);
void i915_vma_destroy(struct i915_vma *vma);
int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
@ -3285,10 +3182,10 @@ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
int i915_gem_object_sync(struct drm_i915_gem_object *obj,
struct intel_engine_cs *to,
struct drm_i915_gem_request **to_req);
struct drm_i915_gem_request *to);
void i915_vma_move_to_active(struct i915_vma *vma,
struct drm_i915_gem_request *req);
struct drm_i915_gem_request *req,
unsigned int flags);
int i915_gem_dumb_create(struct drm_file *file_priv,
struct drm_device *dev,
struct drm_mode_create_dumb *args);
@ -3299,44 +3196,12 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
struct drm_i915_gem_object *new,
unsigned frontbuffer_bits);
/**
* Returns true if seq1 is later than seq2.
*/
static inline bool
i915_seqno_passed(uint32_t seq1, uint32_t seq2)
{
return (int32_t)(seq1 - seq2) >= 0;
}
static inline bool i915_gem_request_started(const struct drm_i915_gem_request *req)
{
return i915_seqno_passed(intel_engine_get_seqno(req->engine),
req->previous_seqno);
}
static inline bool i915_gem_request_completed(const struct drm_i915_gem_request *req)
{
return i915_seqno_passed(intel_engine_get_seqno(req->engine),
req->seqno);
}
bool __i915_spin_request(const struct drm_i915_gem_request *request,
int state, unsigned long timeout_us);
static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
int state, unsigned long timeout_us)
{
return (i915_gem_request_started(request) &&
__i915_spin_request(request, state, timeout_us));
}
int __must_check i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno);
int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
struct drm_i915_gem_request *
i915_gem_find_active_request(struct intel_engine_cs *engine);
void i915_gem_retire_requests(struct drm_i915_private *dev_priv);
void i915_gem_retire_requests_ring(struct intel_engine_cs *engine);
static inline u32 i915_reset_counter(struct i915_gpu_error *error)
{
@ -3381,24 +3246,13 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error)
void i915_gem_reset(struct drm_device *dev);
bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
int __must_check i915_gem_init(struct drm_device *dev);
int i915_gem_init_engines(struct drm_device *dev);
int __must_check i915_gem_init_hw(struct drm_device *dev);
void i915_gem_init_swizzling(struct drm_device *dev);
void i915_gem_cleanup_engines(struct drm_device *dev);
int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv);
int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
bool interruptible);
int __must_check i915_gem_suspend(struct drm_device *dev);
void __i915_add_request(struct drm_i915_gem_request *req,
struct drm_i915_gem_object *batch_obj,
bool flush_caches);
#define i915_add_request(req) \
__i915_add_request(req, NULL, true)
#define i915_add_request_no_flush(req) \
__i915_add_request(req, NULL, false)
int __i915_wait_request(struct drm_i915_gem_request *req,
bool interruptible,
s64 *timeout,
struct intel_rps_client *rps);
int __must_check i915_wait_request(struct drm_i915_gem_request *req);
void i915_gem_resume(struct drm_device *dev);
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
int __must_check
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
@ -3419,11 +3273,10 @@ int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
int i915_gem_open(struct drm_device *dev, struct drm_file *file);
void i915_gem_release(struct drm_device *dev, struct drm_file *file);
uint32_t
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode);
uint32_t
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
int tiling_mode, bool fenced);
u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv, u64 size,
int tiling_mode);
u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
int tiling_mode, bool fenced);
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
enum i915_cache_level cache_level);
@ -3444,7 +3297,6 @@ i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *o)
return i915_gem_obj_ggtt_offset_view(o, &i915_ggtt_view_normal);
}
bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o);
bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
const struct i915_ggtt_view *view);
bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
@ -3478,7 +3330,6 @@ i915_vm_to_ppgtt(struct i915_address_space *vm)
return container_of(vm, struct i915_hw_ppgtt, base);
}
static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj)
{
return i915_gem_obj_ggtt_bound_view(obj, &i915_ggtt_view_normal);
@ -3487,18 +3338,6 @@ static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj)
unsigned long
i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj);
static inline int __must_check
i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
uint32_t alignment,
unsigned flags)
{
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct i915_ggtt *ggtt = &dev_priv->ggtt;
return i915_gem_object_pin(obj, &ggtt->base,
alignment, flags | PIN_GLOBAL);
}
void i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
const struct i915_ggtt_view *view);
static inline void
@ -3528,6 +3367,7 @@ void i915_gem_context_reset(struct drm_device *dev);
int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
int i915_switch_context(struct drm_i915_gem_request *req);
int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
void i915_gem_context_free(struct kref *ctx_ref);
struct drm_i915_gem_object *
i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
@ -3548,12 +3388,14 @@ i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
return ctx;
}
static inline void i915_gem_context_reference(struct i915_gem_context *ctx)
static inline struct i915_gem_context *
i915_gem_context_get(struct i915_gem_context *ctx)
{
kref_get(&ctx->ref);
return ctx;
}
static inline void i915_gem_context_unreference(struct i915_gem_context *ctx)
static inline void i915_gem_context_put(struct i915_gem_context *ctx)
{
lockdep_assert_held(&ctx->i915->drm.struct_mutex);
kref_put(&ctx->ref, i915_gem_context_free);
@ -3576,13 +3418,10 @@ int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
/* i915_gem_evict.c */
int __must_check i915_gem_evict_something(struct drm_device *dev,
struct i915_address_space *vm,
int min_size,
unsigned alignment,
int __must_check i915_gem_evict_something(struct i915_address_space *vm,
u64 min_size, u64 alignment,
unsigned cache_level,
unsigned long start,
unsigned long end,
u64 start, u64 end,
unsigned flags);
int __must_check i915_gem_evict_for_vma(struct i915_vma *target);
int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
@ -3634,16 +3473,9 @@ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_objec
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
obj->tiling_mode != I915_TILING_NONE;
i915_gem_object_is_tiled(obj);
}
/* i915_gem_debug.c */
#if WATCH_LISTS
int i915_verify_lists(struct drm_device *dev);
#else
#define i915_verify_lists(dev) 0
#endif
/* i915_debugfs.c */
#ifdef CONFIG_DEBUG_FS
int i915_debugfs_register(struct drm_i915_private *dev_priv);
@ -3684,15 +3516,15 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
/* i915_cmd_parser.c */
int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
int i915_cmd_parser_init_ring(struct intel_engine_cs *engine);
void i915_cmd_parser_fini_ring(struct intel_engine_cs *engine);
bool i915_needs_cmd_parser(struct intel_engine_cs *engine);
int i915_parse_cmds(struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj,
struct drm_i915_gem_object *shadow_batch_obj,
u32 batch_start_offset,
u32 batch_len,
bool is_master);
int intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine);
int intel_engine_cmd_parser(struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj,
struct drm_i915_gem_object *shadow_batch_obj,
u32 batch_start_offset,
u32 batch_len,
bool is_master);
/* i915_suspend.c */
extern int i915_save_state(struct drm_device *dev);
@ -3800,7 +3632,6 @@ extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
bool enable);
extern bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv);
int i915_reg_read_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);

2544
drivers/gpu/drm/i915/i915_gem.c
View file

File diff suppressed because it is too large Load diff

32
drivers/gpu/drm/i915/i915_gem_batch_pool.c
View file

@ -41,15 +41,15 @@
/**
* i915_gem_batch_pool_init() - initialize a batch buffer pool
* @dev: the drm device
* @engine: the associated request submission engine
* @pool: the batch buffer pool
*/
void i915_gem_batch_pool_init(struct drm_device *dev,
void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
struct i915_gem_batch_pool *pool)
{
int n;
pool->dev = dev;
pool->engine = engine;
for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
INIT_LIST_HEAD(&pool->cache_list[n]);
@ -65,18 +65,17 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
{
int n;
WARN_ON(!mutex_is_locked(&pool->dev->struct_mutex));
lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
while (!list_empty(&pool->cache_list[n])) {
struct drm_i915_gem_object *obj =
list_first_entry(&pool->cache_list[n],
struct drm_i915_gem_object,
batch_pool_link);
struct drm_i915_gem_object *obj, *next;
list_del(&obj->batch_pool_link);
drm_gem_object_unreference(&obj->base);
}
list_for_each_entry_safe(obj, next,
&pool->cache_list[n],
batch_pool_link)
i915_gem_object_put(obj);
INIT_LIST_HEAD(&pool->cache_list[n]);
}
}
@ -102,7 +101,7 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
struct list_head *list;
int n;
WARN_ON(!mutex_is_locked(&pool->dev->struct_mutex));
lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
/* Compute a power-of-two bucket, but throw everything greater than
* 16KiB into the same bucket: i.e. the the buckets hold objects of
@ -115,13 +114,14 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
/* The batches are strictly LRU ordered */
if (tmp->active)
if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
&tmp->base.dev->struct_mutex))
break;
/* While we're looping, do some clean up */
if (tmp->madv == __I915_MADV_PURGED) {
list_del(&tmp->batch_pool_link);
drm_gem_object_unreference(&tmp->base);
i915_gem_object_put(tmp);
continue;
}
@ -134,7 +134,7 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
if (obj == NULL) {
int ret;
obj = i915_gem_object_create(pool->dev, size);
obj = i915_gem_object_create(&pool->engine->i915->drm, size);
if (IS_ERR(obj))
return obj;

6
drivers/gpu/drm/i915/i915_gem_batch_pool.h
View file

@ -27,13 +27,15 @@
#include "i915_drv.h"
struct intel_engine_cs;
struct i915_gem_batch_pool {
struct drm_device *dev;
struct intel_engine_cs *engine;
struct list_head cache_list[4];
};
/* i915_gem_batch_pool.c */
void i915_gem_batch_pool_init(struct drm_device *dev,
void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
struct i915_gem_batch_pool *pool);
void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool);
struct drm_i915_gem_object*

183
drivers/gpu/drm/i915/i915_gem_context.c
View file

@ -134,21 +134,6 @@ static int get_context_size(struct drm_i915_private *dev_priv)
return ret;
}
static void i915_gem_context_clean(struct i915_gem_context *ctx)
{
struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
struct i915_vma *vma, *next;
if (!ppgtt)
return;
list_for_each_entry_safe(vma, next, &ppgtt->base.inactive_list,
vm_link) {
if (WARN_ON(__i915_vma_unbind_no_wait(vma)))
break;
}
}
void i915_gem_context_free(struct kref *ctx_ref)
{
struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
@ -156,13 +141,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
lockdep_assert_held(&ctx->i915->drm.struct_mutex);
trace_i915_context_free(ctx);
/*
* This context is going away and we need to remove all VMAs still
* around. This is to handle imported shared objects for which
* destructor did not run when their handles were closed.
*/
i915_gem_context_clean(ctx);
GEM_BUG_ON(!ctx->closed);
i915_ppgtt_put(ctx->ppgtt);
@ -173,10 +152,10 @@ void i915_gem_context_free(struct kref *ctx_ref)
continue;
WARN_ON(ce->pin_count);
if (ce->ringbuf)
intel_ringbuffer_free(ce->ringbuf);
if (ce->ring)
intel_ring_free(ce->ring);
drm_gem_object_unreference(&ce->state->base);
i915_gem_object_put(ce->state);
}
list_del(&ctx->link);
@ -216,7 +195,7 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
/* Failure shouldn't ever happen this early */
if (WARN_ON(ret)) {
drm_gem_object_unreference(&obj->base);
i915_gem_object_put(obj);
return ERR_PTR(ret);
}
}
@ -224,6 +203,37 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
return obj;
}
static void i915_ppgtt_close(struct i915_address_space *vm)
{
struct list_head *phases[] = {
&vm->active_list,
&vm->inactive_list,
&vm->unbound_list,
NULL,
}, **phase;
GEM_BUG_ON(vm->closed);
vm->closed = true;
for (phase = phases; *phase; phase++) {
struct i915_vma *vma, *vn;
list_for_each_entry_safe(vma, vn, *phase, vm_link)
if (!i915_vma_is_closed(vma))
i915_vma_close(vma);
}
}
static void context_close(struct i915_gem_context *ctx)
{
GEM_BUG_ON(ctx->closed);
ctx->closed = true;
if (ctx->ppgtt)
i915_ppgtt_close(&ctx->ppgtt->base);
ctx->file_priv = ERR_PTR(-EBADF);
i915_gem_context_put(ctx);
}
static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out)
{
int ret;
@ -305,7 +315,7 @@ __create_hw_context(struct drm_device *dev,
return ctx;
err_out:
i915_gem_context_unreference(ctx);
context_close(ctx);
return ERR_PTR(ret);
}
@ -327,13 +337,14 @@ i915_gem_create_context(struct drm_device *dev,
return ctx;
if (USES_FULL_PPGTT(dev)) {
struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv);
struct i915_hw_ppgtt *ppgtt =
i915_ppgtt_create(to_i915(dev), file_priv);
if (IS_ERR(ppgtt)) {
DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
PTR_ERR(ppgtt));
idr_remove(&file_priv->context_idr, ctx->user_handle);
i915_gem_context_unreference(ctx);
context_close(ctx);
return ERR_CAST(ppgtt);
}
@ -390,7 +401,7 @@ static void i915_gem_context_unpin(struct i915_gem_context *ctx,
if (ce->state)
i915_gem_object_ggtt_unpin(ce->state);
i915_gem_context_unreference(ctx);
i915_gem_context_put(ctx);
}
}
@ -504,7 +515,7 @@ void i915_gem_context_fini(struct drm_device *dev)
lockdep_assert_held(&dev->struct_mutex);
i915_gem_context_unreference(dctx);
context_close(dctx);
dev_priv->kernel_context = NULL;
ida_destroy(&dev_priv->context_hw_ida);
@ -514,8 +525,7 @@ static int context_idr_cleanup(int id, void *p, void *data)
{
struct i915_gem_context *ctx = p;
ctx->file_priv = ERR_PTR(-EBADF);
i915_gem_context_unreference(ctx);
context_close(ctx);
return 0;
}
@ -552,11 +562,12 @@ static inline int
mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
{
struct drm_i915_private *dev_priv = req->i915;
struct intel_ring *ring = req->ring;
struct intel_engine_cs *engine = req->engine;
u32 flags = hw_flags | MI_MM_SPACE_GTT;
const int num_rings =
/* Use an extended w/a on ivb+ if signalling from other rings */
i915_semaphore_is_enabled(dev_priv) ?
i915.semaphores ?
hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1 :
0;
int len, ret;
@ -567,7 +578,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
* itlb_before_ctx_switch.
*/
if (IS_GEN6(dev_priv)) {
ret = engine->flush(req, I915_GEM_GPU_DOMAINS, 0);
ret = engine->emit_flush(req, EMIT_INVALIDATE);
if (ret)
return ret;
}
@ -589,64 +600,64 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
if (INTEL_GEN(dev_priv) >= 7) {
intel_ring_emit(engine, MI_ARB_ON_OFF | MI_ARB_DISABLE);
intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE);
if (num_rings) {
struct intel_engine_cs *signaller;
intel_ring_emit(engine,
intel_ring_emit(ring,
MI_LOAD_REGISTER_IMM(num_rings));
for_each_engine(signaller, dev_priv) {
if (signaller == engine)
continue;
intel_ring_emit_reg(engine,
intel_ring_emit_reg(ring,
RING_PSMI_CTL(signaller->mmio_base));
intel_ring_emit(engine,
intel_ring_emit(ring,
_MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
}
}
}
intel_ring_emit(engine, MI_NOOP);
intel_ring_emit(engine, MI_SET_CONTEXT);
intel_ring_emit(engine,
intel_ring_emit(ring, MI_NOOP);
intel_ring_emit(ring, MI_SET_CONTEXT);
intel_ring_emit(ring,
i915_gem_obj_ggtt_offset(req->ctx->engine[RCS].state) |
flags);
/*
* w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
* WaMiSetContext_Hang:snb,ivb,vlv
*/
intel_ring_emit(engine, MI_NOOP);
intel_ring_emit(ring, MI_NOOP);
if (INTEL_GEN(dev_priv) >= 7) {
if (num_rings) {
struct intel_engine_cs *signaller;
i915_reg_t last_reg = {}; /* keep gcc quiet */
intel_ring_emit(engine,
intel_ring_emit(ring,
MI_LOAD_REGISTER_IMM(num_rings));
for_each_engine(signaller, dev_priv) {
if (signaller == engine)
continue;
last_reg = RING_PSMI_CTL(signaller->mmio_base);
intel_ring_emit_reg(engine, last_reg);
intel_ring_emit(engine,
intel_ring_emit_reg(ring, last_reg);
intel_ring_emit(ring,
_MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
}
/* Insert a delay before the next switch! */
intel_ring_emit(engine,
intel_ring_emit(ring,
MI_STORE_REGISTER_MEM |
MI_SRM_LRM_GLOBAL_GTT);
intel_ring_emit_reg(engine, last_reg);
intel_ring_emit(engine, engine->scratch.gtt_offset);
intel_ring_emit(engine, MI_NOOP);
intel_ring_emit_reg(ring, last_reg);
intel_ring_emit(ring, engine->scratch.gtt_offset);
intel_ring_emit(ring, MI_NOOP);
}
intel_ring_emit(engine, MI_ARB_ON_OFF | MI_ARB_ENABLE);
intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_ENABLE);
}
intel_ring_advance(engine);
intel_ring_advance(ring);
return ret;
}
@ -654,7 +665,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
static int remap_l3(struct drm_i915_gem_request *req, int slice)
{
u32 *remap_info = req->i915->l3_parity.remap_info[slice];
struct intel_engine_cs *engine = req->engine;
struct intel_ring *ring = req->ring;
int i, ret;
if (!remap_info)
@ -669,13 +680,13 @@ static int remap_l3(struct drm_i915_gem_request *req, int slice)
* here because no other code should access these registers other than
* at initialization time.
*/
intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4));
intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4));
for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
intel_ring_emit(engine, remap_info[i]);
intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
intel_ring_emit(ring, remap_info[i]);
}
intel_ring_emit(engine, MI_NOOP);
intel_ring_advance(engine);
intel_ring_emit(ring, MI_NOOP);
intel_ring_advance(ring);
return 0;
}
@ -752,9 +763,8 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
return 0;
/* Trying to pin first makes error handling easier. */
ret = i915_gem_obj_ggtt_pin(to->engine[RCS].state,
to->ggtt_alignment,
0);
ret = i915_gem_object_ggtt_pin(to->engine[RCS].state, NULL, 0,
to->ggtt_alignment, 0);
if (ret)
return ret;
@ -814,8 +824,8 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
* MI_SET_CONTEXT instead of when the next seqno has completed.
*/
if (from != NULL) {
from->engine[RCS].state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->engine[RCS].state), req);
struct drm_i915_gem_object *obj = from->engine[RCS].state;
/* As long as MI_SET_CONTEXT is serializing, ie. it flushes the
* whole damn pipeline, we don't need to explicitly mark the
* object dirty. The only exception is that the context must be
@ -823,14 +833,14 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
* able to defer doing this until we know the object would be
* swapped, but there is no way to do that yet.
*/
from->engine[RCS].state->dirty = 1;
obj->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
i915_vma_move_to_active(i915_gem_obj_to_ggtt(obj), req, 0);
/* obj is kept alive until the next request by its active ref */
i915_gem_object_ggtt_unpin(from->engine[RCS].state);
i915_gem_context_unreference(from);
i915_gem_object_ggtt_unpin(obj);
i915_gem_context_put(from);
}
i915_gem_context_reference(to);
engine->last_context = to;
engine->last_context = i915_gem_context_get(to);
/* GEN8 does *not* require an explicit reload if the PDPs have been
* setup, and we do not wish to move them.
@ -894,8 +904,9 @@ int i915_switch_context(struct drm_i915_gem_request *req)
{
struct intel_engine_cs *engine = req->engine;
WARN_ON(i915.enable_execlists);
lockdep_assert_held(&req->i915->drm.struct_mutex);
if (i915.enable_execlists)
return 0;
if (!req->ctx->engine[engine->id].state) {
struct i915_gem_context *to = req->ctx;
@ -914,10 +925,9 @@ int i915_switch_context(struct drm_i915_gem_request *req)
}
if (to != engine->last_context) {
i915_gem_context_reference(to);
if (engine->last_context)
i915_gem_context_unreference(engine->last_context);
engine->last_context = to;
i915_gem_context_put(engine->last_context);
engine->last_context = i915_gem_context_get(to);
}
return 0;
@ -926,6 +936,33 @@ int i915_switch_context(struct drm_i915_gem_request *req)
return do_rcs_switch(req);
}
int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
for_each_engine(engine, dev_priv) {
struct drm_i915_gem_request *req;
int ret;
if (engine->last_context == NULL)
continue;
if (engine->last_context == dev_priv->kernel_context)
continue;
req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
if (IS_ERR(req))
return PTR_ERR(req);
ret = i915_switch_context(req);
i915_add_request_no_flush(req);
if (ret)
return ret;
}
return 0;
}
static bool contexts_enabled(struct drm_device *dev)
{
return i915.enable_execlists || to_i915(dev)->hw_context_size;
@ -985,7 +1022,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
}
idr_remove(&file_priv->context_idr, ctx->user_handle);
i915_gem_context_unreference(ctx);
context_close(ctx);
mutex_unlock(&dev->struct_mutex);
DRM_DEBUG_DRIVER("HW context %d destroyed\n", args->ctx_id);

70
drivers/gpu/drm/i915/i915_gem_debug.c
View file

@ -1,70 +0,0 @@
/*
* Copyright © 2008 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*
* Authors:
* Keith Packard <keithp@keithp.com>
*
*/
#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
#if WATCH_LISTS
int
i915_verify_lists(struct drm_device *dev)
{
static int warned;
struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_gem_object *obj;
struct intel_engine_cs *engine;
int err = 0;
if (warned)
return 0;
for_each_engine(engine, dev_priv) {
list_for_each_entry(obj, &engine->active_list,
engine_list[engine->id]) {
if (obj->base.dev != dev ||
!atomic_read(&obj->base.refcount.refcount)) {
DRM_ERROR("%s: freed active obj %p\n",
engine->name, obj);
err++;
break;
} else if (!obj->active ||
obj->last_read_req[engine->id] == NULL) {
DRM_ERROR("%s: invalid active obj %p\n",
engine->name, obj);
err++;
} else if (obj->base.write_domain) {
DRM_ERROR("%s: invalid write obj %p (w %x)\n",
engine->name,
obj, obj->base.write_domain);
err++;
}
}
}
return warned = err;
}
#endif /* WATCH_LIST */

73
drivers/gpu/drm/i915/i915_gem_dmabuf.c
View file

@ -23,9 +23,13 @@
* Authors:
* Dave Airlie <airlied@redhat.com>
*/
#include <drm/drmP.h>
#include "i915_drv.h"
#include <linux/dma-buf.h>
#include <linux/reservation.h>
#include <drm/drmP.h>
#include "i915_drv.h"
static struct drm_i915_gem_object *dma_buf_to_obj(struct dma_buf *buf)
{
@ -218,25 +222,73 @@ static const struct dma_buf_ops i915_dmabuf_ops = {
.end_cpu_access = i915_gem_end_cpu_access,
};
static void export_fences(struct drm_i915_gem_object *obj,
struct dma_buf *dma_buf)
{
struct reservation_object *resv = dma_buf->resv;
struct drm_i915_gem_request *req;
unsigned long active;
int idx;
active = __I915_BO_ACTIVE(obj);
if (!active)
return;
/* Serialise with execbuf to prevent concurrent fence-loops */
mutex_lock(&obj->base.dev->struct_mutex);
/* Mark the object for future fences before racily adding old fences */
obj->base.dma_buf = dma_buf;
ww_mutex_lock(&resv->lock, NULL);
for_each_active(active, idx) {
req = i915_gem_active_get(&obj->last_read[idx],
&obj->base.dev->struct_mutex);
if (!req)
continue;
if (reservation_object_reserve_shared(resv) == 0)
reservation_object_add_shared_fence(resv, &req->fence);
i915_gem_request_put(req);
}
req = i915_gem_active_get(&obj->last_write,
&obj->base.dev->struct_mutex);
if (req) {
reservation_object_add_excl_fence(resv, &req->fence);
i915_gem_request_put(req);
}
ww_mutex_unlock(&resv->lock);
mutex_unlock(&obj->base.dev->struct_mutex);
}
struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
struct drm_gem_object *gem_obj, int flags)
{
struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
struct dma_buf *dma_buf;
exp_info.ops = &i915_dmabuf_ops;
exp_info.size = gem_obj->size;
exp_info.flags = flags;
exp_info.priv = gem_obj;
if (obj->ops->dmabuf_export) {
int ret = obj->ops->dmabuf_export(obj);
if (ret)
return ERR_PTR(ret);
}
return dma_buf_export(&exp_info);
dma_buf = dma_buf_export(&exp_info);
if (IS_ERR(dma_buf))
return dma_buf;
export_fences(obj, dma_buf);
return dma_buf;
}
static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
@ -278,8 +330,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
* Importing dmabuf exported from out own gem increases
* refcount on gem itself instead of f_count of dmabuf.
*/
drm_gem_object_reference(&obj->base);
return &obj->base;
return &i915_gem_object_get(obj)->base;
}
}
@ -300,6 +351,16 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops);
obj->base.import_attach = attach;
/* We use GTT as shorthand for a coherent domain, one that is
* neither in the GPU cache nor in the CPU cache, where all
* writes are immediately visible in memory. (That's not strictly
* true, but it's close! There are internal buffers such as the
* write-combined buffer or a delay through the chipset for GTT
* writes that do require us to treat GTT as a separate cache domain.)
*/
obj->base.read_domains = I915_GEM_DOMAIN_GTT;
obj->base.write_domain = 0;
return &obj->base;
fail_detach:

187
drivers/gpu/drm/i915/i915_gem_evict.c
View file

@ -33,41 +33,23 @@
#include "intel_drv.h"
#include "i915_trace.h"
static int switch_to_pinned_context(struct drm_i915_private *dev_priv)
static bool
gpu_is_idle(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
if (i915.enable_execlists)
return 0;
for_each_engine(engine, dev_priv) {
struct drm_i915_gem_request *req;
int ret;
if (engine->last_context == NULL)
continue;
if (engine->last_context == dev_priv->kernel_context)
continue;
req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
if (IS_ERR(req))
return PTR_ERR(req);
ret = i915_switch_context(req);
i915_add_request_no_flush(req);
if (ret)
return ret;
if (intel_engine_is_active(engine))
return false;
}
return 0;
return true;
}
static bool
mark_free(struct i915_vma *vma, struct list_head *unwind)
{
if (vma->pin_count)
if (i915_vma_is_pinned(vma))
return false;
if (WARN_ON(!list_empty(&vma->exec_list)))
@ -79,7 +61,6 @@ mark_free(struct i915_vma *vma, struct list_head *unwind)
/**
* i915_gem_evict_something - Evict vmas to make room for binding a new one
* @dev: drm_device
* @vm: address space to evict from
* @min_size: size of the desired free space
* @alignment: alignment constraint of the desired free space
@ -102,42 +83,37 @@ mark_free(struct i915_vma *vma, struct list_head *unwind)
* memory in e.g. the shrinker.
*/
int
i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
int min_size, unsigned alignment, unsigned cache_level,
unsigned long start, unsigned long end,
i915_gem_evict_something(struct i915_address_space *vm,
u64 min_size, u64 alignment,
unsigned cache_level,
u64 start, u64 end,
unsigned flags)
{
struct list_head eviction_list, unwind_list;
struct i915_vma *vma;
int ret = 0;
int pass = 0;
struct drm_i915_private *dev_priv = to_i915(vm->dev);
struct list_head eviction_list;
struct list_head *phases[] = {
&vm->inactive_list,
&vm->active_list,
NULL,
}, **phase;
struct i915_vma *vma, *next;
int ret;
trace_i915_gem_evict(dev, min_size, alignment, flags);
trace_i915_gem_evict(vm, min_size, alignment, flags);
/*
* The goal is to evict objects and amalgamate space in LRU order.
* The oldest idle objects reside on the inactive list, which is in
* retirement order. The next objects to retire are those on the (per
* ring) active list that do not have an outstanding flush. Once the
* hardware reports completion (the seqno is updated after the
* batchbuffer has been finished) the clean buffer objects would
* be retired to the inactive list. Any dirty objects would be added
* to the tail of the flushing list. So after processing the clean
* active objects we need to emit a MI_FLUSH to retire the flushing
* list, hence the retirement order of the flushing list is in
* advance of the dirty objects on the active lists.
* retirement order. The next objects to retire are those in flight,
* on the active list, again in retirement order.
*
* The retirement sequence is thus:
* 1. Inactive objects (already retired)
* 2. Clean active objects
* 3. Flushing list
* 4. Dirty active objects.
* 2. Active objects (will stall on unbinding)
*
* On each list, the oldest objects lie at the HEAD with the freshest
* object on the TAIL.
*/
INIT_LIST_HEAD(&unwind_list);
if (start != 0 || end != vm->total) {
drm_mm_init_scan_with_range(&vm->mm, min_size,
alignment, cache_level,
@ -145,96 +121,84 @@ i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
} else
drm_mm_init_scan(&vm->mm, min_size, alignment, cache_level);
search_again:
/* First see if there is a large enough contiguous idle region... */
list_for_each_entry(vma, &vm->inactive_list, vm_link) {
if (mark_free(vma, &unwind_list))
goto found;
}
if (flags & PIN_NONBLOCK)
goto none;
phases[1] = NULL;
/* Now merge in the soon-to-be-expired objects... */
list_for_each_entry(vma, &vm->active_list, vm_link) {
if (mark_free(vma, &unwind_list))
goto found;
}
search_again:
INIT_LIST_HEAD(&eviction_list);
phase = phases;
do {
list_for_each_entry(vma, *phase, vm_link)
if (mark_free(vma, &eviction_list))
goto found;
} while (*++phase);
none:
/* Nothing found, clean up and bail out! */
while (!list_empty(&unwind_list)) {
vma = list_first_entry(&unwind_list,
struct i915_vma,
exec_list);
list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
ret = drm_mm_scan_remove_block(&vma->node);
BUG_ON(ret);
list_del_init(&vma->exec_list);
INIT_LIST_HEAD(&vma->exec_list);
}
/* Can we unpin some objects such as idle hw contents,
* or pending flips?
* or pending flips? But since only the GGTT has global entries
* such as scanouts, rinbuffers and contexts, we can skip the
* purge when inspecting per-process local address spaces.
*/
if (flags & PIN_NONBLOCK)
if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK)
return -ENOSPC;
/* Only idle the GPU and repeat the search once */
if (pass++ == 0) {
struct drm_i915_private *dev_priv = to_i915(dev);
if (i915_is_ggtt(vm)) {
ret = switch_to_pinned_context(dev_priv);
if (ret)
return ret;
}
ret = i915_gem_wait_for_idle(dev_priv);
if (ret)
return ret;
i915_gem_retire_requests(dev_priv);
goto search_again;
if (gpu_is_idle(dev_priv)) {
/* If we still have pending pageflip completions, drop
* back to userspace to give our workqueues time to
* acquire our locks and unpin the old scanouts.
*/
return intel_has_pending_fb_unpin(vm->dev) ? -EAGAIN : -ENOSPC;
}
/* If we still have pending pageflip completions, drop
* back to userspace to give our workqueues time to
* acquire our locks and unpin the old scanouts.
/* Not everything in the GGTT is tracked via vma (otherwise we
* could evict as required with minimal stalling) so we are forced
* to idle the GPU and explicitly retire outstanding requests in
* the hopes that we can then remove contexts and the like only
* bound by their active reference.
*/
return intel_has_pending_fb_unpin(dev) ? -EAGAIN : -ENOSPC;
ret = i915_gem_switch_to_kernel_context(dev_priv);
if (ret)
return ret;
ret = i915_gem_wait_for_idle(dev_priv, true);
if (ret)
return ret;
i915_gem_retire_requests(dev_priv);
goto search_again;
found:
/* drm_mm doesn't allow any other other operations while
* scanning, therefore store to be evicted objects on a
* temporary list. */
INIT_LIST_HEAD(&eviction_list);
while (!list_empty(&unwind_list)) {
vma = list_first_entry(&unwind_list,
struct i915_vma,
exec_list);
if (drm_mm_scan_remove_block(&vma->node)) {
list_move(&vma->exec_list, &eviction_list);
drm_gem_object_reference(&vma->obj->base);
continue;
}
list_del_init(&vma->exec_list);
* scanning, therefore store to-be-evicted objects on a
* temporary list and take a reference for all before
* calling unbind (which may remove the active reference
* of any of our objects, thus corrupting the list).
*/
list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
if (drm_mm_scan_remove_block(&vma->node))
__i915_vma_pin(vma);
else
list_del_init(&vma->exec_list);
}
/* Unbinding will emit any required flushes */
while (!list_empty(&eviction_list)) {
struct drm_gem_object *obj;
vma = list_first_entry(&eviction_list,
struct i915_vma,
exec_list);
obj = &vma->obj->base;
list_del_init(&vma->exec_list);
__i915_vma_unpin(vma);
if (ret == 0)
ret = i915_vma_unbind(vma);
drm_gem_object_unreference(obj);
}
return ret;
}
@ -256,8 +220,8 @@ i915_gem_evict_for_vma(struct i915_vma *target)
vma = container_of(node, typeof(*vma), node);
if (vma->pin_count) {
if (!vma->exec_entry || (vma->pin_count > 1))
if (i915_vma_is_pinned(vma)) {
if (!vma->exec_entry || i915_vma_pin_count(vma) > 1)
/* Object is pinned for some other use */
return -EBUSY;
@ -303,22 +267,21 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
struct drm_i915_private *dev_priv = to_i915(vm->dev);
if (i915_is_ggtt(vm)) {
ret = switch_to_pinned_context(dev_priv);
ret = i915_gem_switch_to_kernel_context(dev_priv);
if (ret)
return ret;
}
ret = i915_gem_wait_for_idle(dev_priv);
ret = i915_gem_wait_for_idle(dev_priv, true);
if (ret)
return ret;
i915_gem_retire_requests(dev_priv);
WARN_ON(!list_empty(&vm->active_list));
}
list_for_each_entry_safe(vma, next, &vm->inactive_list, vm_link)
if (vma->pin_count == 0)
if (!i915_vma_is_pinned(vma))
WARN_ON(i915_vma_unbind(vma));
return 0;

478
drivers/gpu/drm/i915/i915_gem_execbuffer.c
View file

@ -26,21 +26,38 @@
*
*/
#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
#include "i915_trace.h"
#include "intel_drv.h"
#include <linux/dma_remapping.h>
#include <linux/reservation.h>
#include <linux/uaccess.h>
#define __EXEC_OBJECT_HAS_PIN (1<<31)
#define __EXEC_OBJECT_HAS_FENCE (1<<30)
#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
#define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
#include "i915_gem_dmabuf.h"
#include "i915_trace.h"
#include "intel_drv.h"
#include "intel_frontbuffer.h"
#define __EXEC_OBJECT_HAS_PIN (1<<31)
#define __EXEC_OBJECT_HAS_FENCE (1<<30)
#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
#define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
#define __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
#define BATCH_OFFSET_BIAS (256*1024)
struct i915_execbuffer_params {
struct drm_device *dev;
struct drm_file *file;
struct i915_vma *batch;
u32 dispatch_flags;
u32 args_batch_start_offset;
struct intel_engine_cs *engine;
struct i915_gem_context *ctx;
struct drm_i915_gem_request *request;
};
struct eb_vmas {
struct list_head vmas;
int and;
@ -89,6 +106,26 @@ eb_reset(struct eb_vmas *eb)
memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
}
static struct i915_vma *
eb_get_batch(struct eb_vmas *eb)
{
struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
/*
* SNA is doing fancy tricks with compressing batch buffers, which leads
* to negative relocation deltas. Usually that works out ok since the
* relocate address is still positive, except when the batch is placed
* very low in the GTT. Ensure this doesn't happen.
*
* Note that actual hangs have only been observed on gen7, but for
* paranoia do it everywhere.
*/
if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
return vma;
}
static int
eb_lookup_vmas(struct eb_vmas *eb,
struct drm_i915_gem_exec_object2 *exec,
@ -122,7 +159,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
goto err;
}
drm_gem_object_reference(&obj->base);
i915_gem_object_get(obj);
list_add_tail(&obj->obj_exec_link, &objects);
}
spin_unlock(&file->table_lock);
@ -175,7 +212,7 @@ err:
struct drm_i915_gem_object,
obj_exec_link);
list_del_init(&obj->obj_exec_link);
drm_gem_object_unreference(&obj->base);
i915_gem_object_put(obj);
}
/*
* Objects already transfered to the vmas list will be unreferenced by
@ -219,7 +256,7 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
i915_gem_object_unpin_fence(obj);
if (entry->flags & __EXEC_OBJECT_HAS_PIN)
vma->pin_count--;
__i915_vma_unpin(vma);
entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
}
@ -234,7 +271,7 @@ static void eb_destroy(struct eb_vmas *eb)
exec_list);
list_del_init(&vma->exec_list);
i915_gem_execbuffer_unreserve_vma(vma);
drm_gem_object_unreference(&vma->obj->base);
i915_gem_object_put(vma->obj);
}
kfree(eb);
}
@ -399,6 +436,20 @@ relocate_entry_clflush(struct drm_i915_gem_object *obj,
return 0;
}
static bool object_is_idle(struct drm_i915_gem_object *obj)
{
unsigned long active = i915_gem_object_get_active(obj);
int idx;
for_each_active(active, idx) {
if (!i915_gem_active_is_idle(&obj->last_read[idx],
&obj->base.dev->struct_mutex))
return false;
}
return true;
}
static int
i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
struct eb_vmas *eb,
@ -482,7 +533,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
}
/* We can't wait for rendering with pagefaults disabled */
if (obj->active && pagefault_disabled())
if (pagefault_disabled() && !object_is_idle(obj))
return -EFAULT;
if (use_cpu_reloc(obj))
@ -626,12 +677,16 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
flags |= PIN_HIGH;
}
ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
if ((ret == -ENOSPC || ret == -E2BIG) &&
ret = i915_vma_pin(vma,
entry->pad_to_size,
entry->alignment,
flags);
if ((ret == -ENOSPC || ret == -E2BIG) &&
only_mappable_for_reloc(entry->flags))
ret = i915_gem_object_pin(obj, vma->vm,
entry->alignment,
flags & ~PIN_MAPPABLE);
ret = i915_vma_pin(vma,
entry->pad_to_size,
entry->alignment,
flags & ~PIN_MAPPABLE);
if (ret)
return ret;
@ -667,7 +722,7 @@ need_reloc_mappable(struct i915_vma *vma)
if (entry->relocation_count == 0)
return false;
if (!vma->is_ggtt)
if (!i915_vma_is_ggtt(vma))
return false;
/* See also use_cpu_reloc() */
@ -686,12 +741,16 @@ eb_vma_misplaced(struct i915_vma *vma)
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
struct drm_i915_gem_object *obj = vma->obj;
WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP && !vma->is_ggtt);
WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
!i915_vma_is_ggtt(vma));
if (entry->alignment &&
vma->node.start & (entry->alignment - 1))
return true;
if (vma->node.size < entry->pad_to_size)
return true;
if (entry->flags & EXEC_OBJECT_PINNED &&
vma->node.start != entry->offset)
return true;
@ -725,8 +784,6 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
int retry;
i915_gem_retire_requests_ring(engine);
vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
INIT_LIST_HEAD(&ordered_vmas);
@ -746,7 +803,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
need_fence =
entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
obj->tiling_mode != I915_TILING_NONE;
i915_gem_object_is_tiled(obj);
need_mappable = need_fence || need_reloc_mappable(vma);
if (entry->flags & EXEC_OBJECT_PINNED)
@ -843,7 +900,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
list_del_init(&vma->exec_list);
i915_gem_execbuffer_unreserve_vma(vma);
drm_gem_object_unreference(&vma->obj->base);
i915_gem_object_put(vma->obj);
}
mutex_unlock(&dev->struct_mutex);
@ -937,11 +994,21 @@ err:
return ret;
}
static unsigned int eb_other_engines(struct drm_i915_gem_request *req)
{
unsigned int mask;
mask = ~intel_engine_flag(req->engine) & I915_BO_ACTIVE_MASK;
mask <<= I915_BO_ACTIVE_SHIFT;
return mask;
}
static int
i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
struct list_head *vmas)
{
const unsigned other_rings = ~intel_engine_flag(req->engine);
const unsigned int other_rings = eb_other_engines(req);
struct i915_vma *vma;
uint32_t flush_domains = 0;
bool flush_chipset = false;
@ -950,8 +1017,8 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
list_for_each_entry(vma, vmas, exec_list) {
struct drm_i915_gem_object *obj = vma->obj;
if (obj->active & other_rings) {
ret = i915_gem_object_sync(obj, req->engine, &req);
if (obj->flags & other_rings) {
ret = i915_gem_object_sync(obj, req);
if (ret)
return ret;
}
@ -968,10 +1035,8 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
if (flush_domains & I915_GEM_DOMAIN_GTT)
wmb();
/* Unconditionally invalidate gpu caches and ensure that we do flush
* any residual writes from the previous batch.
*/
return intel_ring_invalidate_all_caches(req);
/* Unconditionally invalidate GPU caches and TLBs. */
return req->engine->emit_flush(req, EMIT_INVALIDATE);
}
static bool
@ -1007,6 +1072,9 @@ validate_exec_list(struct drm_device *dev,
unsigned invalid_flags;
int i;
/* INTERNAL flags must not overlap with external ones */
BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
if (USES_FULL_PPGTT(dev))
invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
@ -1036,6 +1104,14 @@ validate_exec_list(struct drm_device *dev,
if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
return -EINVAL;
/* pad_to_size was once a reserved field, so sanitize it */
if (exec[i].flags & EXEC_OBJECT_PAD_TO_SIZE) {
if (offset_in_page(exec[i].pad_to_size))
return -EINVAL;
} else {
exec[i].pad_to_size = 0;
}
/* First check for malicious input causing overflow in
* the worst case where we need to allocate the entire
* relocation tree as a single array.
@ -1086,66 +1162,106 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
return ctx;
}
void
void i915_vma_move_to_active(struct i915_vma *vma,
struct drm_i915_gem_request *req,
unsigned int flags)
{
struct drm_i915_gem_object *obj = vma->obj;
const unsigned int idx = req->engine->id;
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
obj->dirty = 1; /* be paranoid */
/* Add a reference if we're newly entering the active list.
* The order in which we add operations to the retirement queue is
* vital here: mark_active adds to the start of the callback list,
* such that subsequent callbacks are called first. Therefore we
* add the active reference first and queue for it to be dropped
* *last*.
*/
if (!i915_gem_object_is_active(obj))
i915_gem_object_get(obj);
i915_gem_object_set_active(obj, idx);
i915_gem_active_set(&obj->last_read[idx], req);
if (flags & EXEC_OBJECT_WRITE) {
i915_gem_active_set(&obj->last_write, req);
intel_fb_obj_invalidate(obj, ORIGIN_CS);
/* update for the implicit flush after a batch */
obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
}
if (flags & EXEC_OBJECT_NEEDS_FENCE) {
i915_gem_active_set(&obj->last_fence, req);
if (flags & __EXEC_OBJECT_HAS_FENCE) {
struct drm_i915_private *dev_priv = req->i915;
list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
&dev_priv->mm.fence_list);
}
}
i915_vma_set_active(vma, idx);
i915_gem_active_set(&vma->last_read[idx], req);
list_move_tail(&vma->vm_link, &vma->vm->active_list);
}
static void eb_export_fence(struct drm_i915_gem_object *obj,
struct drm_i915_gem_request *req,
unsigned int flags)
{
struct reservation_object *resv;
resv = i915_gem_object_get_dmabuf_resv(obj);
if (!resv)
return;
/* Ignore errors from failing to allocate the new fence, we can't
* handle an error right now. Worst case should be missed
* synchronisation leading to rendering corruption.
*/
ww_mutex_lock(&resv->lock, NULL);
if (flags & EXEC_OBJECT_WRITE)
reservation_object_add_excl_fence(resv, &req->fence);
else if (reservation_object_reserve_shared(resv) == 0)
reservation_object_add_shared_fence(resv, &req->fence);
ww_mutex_unlock(&resv->lock);
}
static void
i915_gem_execbuffer_move_to_active(struct list_head *vmas,
struct drm_i915_gem_request *req)
{
struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
struct i915_vma *vma;
list_for_each_entry(vma, vmas, exec_list) {
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
struct drm_i915_gem_object *obj = vma->obj;
u32 old_read = obj->base.read_domains;
u32 old_write = obj->base.write_domain;
obj->dirty = 1; /* be paranoid */
obj->base.write_domain = obj->base.pending_write_domain;
if (obj->base.write_domain == 0)
if (obj->base.write_domain)
vma->exec_entry->flags |= EXEC_OBJECT_WRITE;
else
obj->base.pending_read_domains |= obj->base.read_domains;
obj->base.read_domains = obj->base.pending_read_domains;
i915_vma_move_to_active(vma, req);
if (obj->base.write_domain) {
i915_gem_request_assign(&obj->last_write_req, req);
intel_fb_obj_invalidate(obj, ORIGIN_CS);
/* update for the implicit flush after a batch */
obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
}
if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
i915_gem_request_assign(&obj->last_fenced_req, req);
if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
struct drm_i915_private *dev_priv = engine->i915;
list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
&dev_priv->mm.fence_list);
}
}
i915_vma_move_to_active(vma, req, vma->exec_entry->flags);
eb_export_fence(obj, req, vma->exec_entry->flags);
trace_i915_gem_object_change_domain(obj, old_read, old_write);
}
}
static void
i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
{
/* Unconditionally force add_request to emit a full flush. */
params->engine->gpu_caches_dirty = true;
/* Add a breadcrumb for the completion of the batch buffer */
__i915_add_request(params->request, params->batch_obj, true);
}
static int
i915_reset_gen7_sol_offsets(struct drm_device *dev,
struct drm_i915_gem_request *req)
i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
{
struct intel_engine_cs *engine = req->engine;
struct drm_i915_private *dev_priv = to_i915(dev);
struct intel_ring *ring = req->ring;
int ret, i;
if (!IS_GEN7(dev) || engine != &dev_priv->engine[RCS]) {
if (!IS_GEN7(req->i915) || req->engine->id != RCS) {
DRM_DEBUG("sol reset is gen7/rcs only\n");
return -EINVAL;
}
@ -1155,21 +1271,21 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev,
return ret;
for (i = 0; i < 4; i++) {
intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
intel_ring_emit_reg(engine, GEN7_SO_WRITE_OFFSET(i));
intel_ring_emit(engine, 0);
intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
intel_ring_emit(ring, 0);
}
intel_ring_advance(engine);
intel_ring_advance(ring);
return 0;
}
static struct drm_i915_gem_object*
static struct i915_vma*
i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
struct drm_i915_gem_exec_object2 *shadow_exec_entry,
struct eb_vmas *eb,
struct drm_i915_gem_object *batch_obj,
struct eb_vmas *eb,
u32 batch_start_offset,
u32 batch_len,
bool is_master)
@ -1181,18 +1297,18 @@ i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
PAGE_ALIGN(batch_len));
if (IS_ERR(shadow_batch_obj))
return shadow_batch_obj;
return ERR_CAST(shadow_batch_obj);
ret = i915_parse_cmds(engine,
batch_obj,
shadow_batch_obj,
batch_start_offset,
batch_len,
is_master);
ret = intel_engine_cmd_parser(engine,
batch_obj,
shadow_batch_obj,
batch_start_offset,
batch_len,
is_master);
if (ret)
goto err;
ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
ret = i915_gem_object_ggtt_pin(shadow_batch_obj, NULL, 0, 0, 0);
if (ret)
goto err;
@ -1203,29 +1319,25 @@ i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
vma->exec_entry = shadow_exec_entry;
vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
drm_gem_object_reference(&shadow_batch_obj->base);
i915_gem_object_get(shadow_batch_obj);
list_add_tail(&vma->exec_list, &eb->vmas);
shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
return shadow_batch_obj;
return vma;
err:
i915_gem_object_unpin_pages(shadow_batch_obj);
if (ret == -EACCES) /* unhandled chained batch */
return batch_obj;
return NULL;
else
return ERR_PTR(ret);
}
int
i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
struct drm_i915_gem_execbuffer2 *args,
struct list_head *vmas)
static int
execbuf_submit(struct i915_execbuffer_params *params,
struct drm_i915_gem_execbuffer2 *args,
struct list_head *vmas)
{
struct drm_device *dev = params->dev;
struct intel_engine_cs *engine = params->engine;
struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_private *dev_priv = params->request->i915;
u64 exec_start, exec_len;
int instp_mode;
u32 instp_mask;
@ -1239,34 +1351,31 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
if (ret)
return ret;
WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<<engine->id),
"%s didn't clear reload\n", engine->name);
instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
instp_mask = I915_EXEC_CONSTANTS_MASK;
switch (instp_mode) {
case I915_EXEC_CONSTANTS_REL_GENERAL:
case I915_EXEC_CONSTANTS_ABSOLUTE:
case I915_EXEC_CONSTANTS_REL_SURFACE:
if (instp_mode != 0 && engine != &dev_priv->engine[RCS]) {
if (instp_mode != 0 && params->engine->id != RCS) {
DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
return -EINVAL;
}
if (instp_mode != dev_priv->relative_constants_mode) {
if (INTEL_INFO(dev)->gen < 4) {
if (INTEL_INFO(dev_priv)->gen < 4) {
DRM_DEBUG("no rel constants on pre-gen4\n");
return -EINVAL;
}
if (INTEL_INFO(dev)->gen > 5 &&
if (INTEL_INFO(dev_priv)->gen > 5 &&
instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
return -EINVAL;
}
/* The HW changed the meaning on this bit on gen6 */
if (INTEL_INFO(dev)->gen >= 6)
if (INTEL_INFO(dev_priv)->gen >= 6)
instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
}
break;
@ -1275,37 +1384,39 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
return -EINVAL;
}
if (engine == &dev_priv->engine[RCS] &&
if (params->engine->id == RCS &&
instp_mode != dev_priv->relative_constants_mode) {
struct intel_ring *ring = params->request->ring;
ret = intel_ring_begin(params->request, 4);
if (ret)
return ret;
intel_ring_emit(engine, MI_NOOP);
intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
intel_ring_emit_reg(engine, INSTPM);
intel_ring_emit(engine, instp_mask << 16 | instp_mode);
intel_ring_advance(engine);
intel_ring_emit(ring, MI_NOOP);
intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
intel_ring_emit_reg(ring, INSTPM);
intel_ring_emit(ring, instp_mask << 16 | instp_mode);
intel_ring_advance(ring);
dev_priv->relative_constants_mode = instp_mode;
}
if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
ret = i915_reset_gen7_sol_offsets(dev, params->request);
ret = i915_reset_gen7_sol_offsets(params->request);
if (ret)
return ret;
}
exec_len = args->batch_len;
exec_start = params->batch_obj_vm_offset +
exec_start = params->batch->node.start +
params->args_batch_start_offset;
if (exec_len == 0)
exec_len = params->batch_obj->base.size;
exec_len = params->batch->size;
ret = engine->dispatch_execbuffer(params->request,
exec_start, exec_len,
params->dispatch_flags);
ret = params->engine->emit_bb_start(params->request,
exec_start, exec_len,
params->dispatch_flags);
if (ret)
return ret;
@ -1318,43 +1429,24 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
/**
* Find one BSD ring to dispatch the corresponding BSD command.
* The ring index is returned.
* The engine index is returned.
*/
static unsigned int
gen8_dispatch_bsd_ring(struct drm_i915_private *dev_priv, struct drm_file *file)
gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
struct drm_file *file)
{
struct drm_i915_file_private *file_priv = file->driver_priv;
/* Check whether the file_priv has already selected one ring. */
if ((int)file_priv->bsd_ring < 0) {
if ((int)file_priv->bsd_engine < 0) {
/* If not, use the ping-pong mechanism to select one. */
mutex_lock(&dev_priv->drm.struct_mutex);
file_priv->bsd_ring = dev_priv->mm.bsd_ring_dispatch_index;
dev_priv->mm.bsd_ring_dispatch_index ^= 1;
file_priv->bsd_engine = dev_priv->mm.bsd_engine_dispatch_index;
dev_priv->mm.bsd_engine_dispatch_index ^= 1;
mutex_unlock(&dev_priv->drm.struct_mutex);
}
return file_priv->bsd_ring;
}
static struct drm_i915_gem_object *
eb_get_batch(struct eb_vmas *eb)
{
struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
/*
* SNA is doing fancy tricks with compressing batch buffers, which leads
* to negative relocation deltas. Usually that works out ok since the
* relocate address is still positive, except when the batch is placed
* very low in the GTT. Ensure this doesn't happen.
*
* Note that actual hangs have only been observed on gen7, but for
* paranoia do it everywhere.
*/
if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
return vma->obj;
return file_priv->bsd_engine;
}
#define I915_USER_RINGS (4)
@ -1367,31 +1459,31 @@ static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
[I915_EXEC_VEBOX] = VECS
};
static int
eb_select_ring(struct drm_i915_private *dev_priv,
struct drm_file *file,
struct drm_i915_gem_execbuffer2 *args,
struct intel_engine_cs **ring)
static struct intel_engine_cs *
eb_select_engine(struct drm_i915_private *dev_priv,
struct drm_file *file,
struct drm_i915_gem_execbuffer2 *args)
{
unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
struct intel_engine_cs *engine;
if (user_ring_id > I915_USER_RINGS) {
DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
return -EINVAL;
return NULL;
}
if ((user_ring_id != I915_EXEC_BSD) &&
((args->flags & I915_EXEC_BSD_MASK) != 0)) {
DRM_DEBUG("execbuf with non bsd ring but with invalid "
"bsd dispatch flags: %d\n", (int)(args->flags));
return -EINVAL;
return NULL;
}
if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
bsd_idx = gen8_dispatch_bsd_ring(dev_priv, file);
bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
} else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
bsd_idx <= I915_EXEC_BSD_RING2) {
bsd_idx >>= I915_EXEC_BSD_SHIFT;
@ -1399,20 +1491,20 @@ eb_select_ring(struct drm_i915_private *dev_priv,
} else {
DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
bsd_idx);
return -EINVAL;
return NULL;
}
*ring = &dev_priv->engine[_VCS(bsd_idx)];
engine = &dev_priv->engine[_VCS(bsd_idx)];
} else {
*ring = &dev_priv->engine[user_ring_map[user_ring_id]];
engine = &dev_priv->engine[user_ring_map[user_ring_id]];
}
if (!intel_engine_initialized(*ring)) {
if (!intel_engine_initialized(engine)) {
DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
return -EINVAL;
return NULL;
}
return 0;
return engine;
}
static int
@ -1423,9 +1515,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
{
struct drm_i915_private *dev_priv = to_i915(dev);
struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct drm_i915_gem_request *req = NULL;
struct eb_vmas *eb;
struct drm_i915_gem_object *batch_obj;
struct drm_i915_gem_exec_object2 shadow_exec_entry;
struct intel_engine_cs *engine;
struct i915_gem_context *ctx;
@ -1454,9 +1544,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
if (args->flags & I915_EXEC_IS_PINNED)
dispatch_flags |= I915_DISPATCH_PINNED;
ret = eb_select_ring(dev_priv, file, args, &engine);
if (ret)
return ret;
engine = eb_select_engine(dev_priv, file, args);
if (!engine)
return -EINVAL;
if (args->buffer_count < 1) {
DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
@ -1496,7 +1586,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
goto pre_mutex_err;
}
i915_gem_context_reference(ctx);
i915_gem_context_get(ctx);
if (ctx->ppgtt)
vm = &ctx->ppgtt->base;
@ -1507,7 +1597,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
eb = eb_create(args);
if (eb == NULL) {
i915_gem_context_unreference(ctx);
i915_gem_context_put(ctx);
mutex_unlock(&dev->struct_mutex);
ret = -ENOMEM;
goto pre_mutex_err;
@ -1519,7 +1609,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
goto err;
/* take note of the batch buffer before we might reorder the lists */
batch_obj = eb_get_batch(eb);
params->batch = eb_get_batch(eb);
/* Move the objects en-masse into the GTT, evicting if necessary. */
need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
@ -1543,34 +1633,28 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
}
/* Set the pending read domains for the batch buffer to COMMAND */
if (batch_obj->base.pending_write_domain) {
if (params->batch->obj->base.pending_write_domain) {
DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
ret = -EINVAL;
goto err;
}
params->args_batch_start_offset = args->batch_start_offset;
if (i915_needs_cmd_parser(engine) && args->batch_len) {
struct drm_i915_gem_object *parsed_batch_obj;
if (intel_engine_needs_cmd_parser(engine) && args->batch_len) {
struct i915_vma *vma;
parsed_batch_obj = i915_gem_execbuffer_parse(engine,
&shadow_exec_entry,
eb,
batch_obj,
args->batch_start_offset,
args->batch_len,
drm_is_current_master(file));
if (IS_ERR(parsed_batch_obj)) {
ret = PTR_ERR(parsed_batch_obj);
vma = i915_gem_execbuffer_parse(engine, &shadow_exec_entry,
params->batch->obj,
eb,
args->batch_start_offset,
args->batch_len,
drm_is_current_master(file));
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto err;
}
/*
* parsed_batch_obj == batch_obj means batch not fully parsed:
* Accept, but don't promote to secure.
*/
if (parsed_batch_obj != batch_obj) {
if (vma) {
/*
* Batch parsed and accepted:
*
@ -1582,16 +1666,18 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
*/
dispatch_flags |= I915_DISPATCH_SECURE;
params->args_batch_start_offset = 0;
batch_obj = parsed_batch_obj;
params->batch = vma;
}
}
batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
params->batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
* batch" bit. Hence we need to pin secure batches into the global gtt.
* hsw should have this fixed, but bdw mucks it up again. */
if (dispatch_flags & I915_DISPATCH_SECURE) {
struct drm_i915_gem_object *obj = params->batch->obj;
/*
* So on first glance it looks freaky that we pin the batch here
* outside of the reservation loop. But:
@ -1602,22 +1688,21 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
* fitting due to fragmentation.
* So this is actually safe.
*/
ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
if (ret)
goto err;
params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
} else
params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
params->batch = i915_gem_obj_to_ggtt(obj);
}
/* Allocate a request for this batch buffer nice and early. */
req = i915_gem_request_alloc(engine, ctx);
if (IS_ERR(req)) {
ret = PTR_ERR(req);
params->request = i915_gem_request_alloc(engine, ctx);
if (IS_ERR(params->request)) {
ret = PTR_ERR(params->request);
goto err_batch_unpin;
}
ret = i915_gem_request_add_to_client(req, file);
ret = i915_gem_request_add_to_client(params->request, file);
if (ret)
goto err_request;
@ -1631,13 +1716,11 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
params->file = file;
params->engine = engine;
params->dispatch_flags = dispatch_flags;
params->batch_obj = batch_obj;
params->ctx = ctx;
params->request = req;
ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
ret = execbuf_submit(params, args, &eb->vmas);
err_request:
i915_gem_execbuffer_retire_commands(params);
__i915_add_request(params->request, params->batch->obj, ret == 0);
err_batch_unpin:
/*
@ -1647,11 +1730,10 @@ err_batch_unpin:
* active.
*/
if (dispatch_flags & I915_DISPATCH_SECURE)
i915_gem_object_ggtt_unpin(batch_obj);
i915_vma_unpin(params->batch);
err:
/* the request owns the ref now */
i915_gem_context_unreference(ctx);
i915_gem_context_put(ctx);
eb_destroy(eb);
mutex_unlock(&dev->struct_mutex);

52
drivers/gpu/drm/i915/i915_gem_fence.c
View file

@ -86,20 +86,22 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg,
if (obj) {
u32 size = i915_gem_obj_ggtt_size(obj);
unsigned int tiling = i915_gem_object_get_tiling(obj);
unsigned int stride = i915_gem_object_get_stride(obj);
uint64_t val;
/* Adjust fence size to match tiled area */
if (obj->tiling_mode != I915_TILING_NONE) {
uint32_t row_size = obj->stride *
(obj->tiling_mode == I915_TILING_Y ? 32 : 8);
if (tiling != I915_TILING_NONE) {
uint32_t row_size = stride *
(tiling == I915_TILING_Y ? 32 : 8);
size = (size / row_size) * row_size;
}
val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
0xfffff000) << 32;
val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
if (obj->tiling_mode == I915_TILING_Y)
val |= (uint64_t)((stride / 128) - 1) << fence_pitch_shift;
if (tiling == I915_TILING_Y)
val |= 1 << I965_FENCE_TILING_Y_SHIFT;
val |= I965_FENCE_REG_VALID;
@ -122,6 +124,8 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
if (obj) {
u32 size = i915_gem_obj_ggtt_size(obj);
unsigned int tiling = i915_gem_object_get_tiling(obj);
unsigned int stride = i915_gem_object_get_stride(obj);
int pitch_val;
int tile_width;
@ -131,17 +135,17 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
"object 0x%08llx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
if (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
tile_width = 128;
else
tile_width = 512;
/* Note: pitch better be a power of two tile widths */
pitch_val = obj->stride / tile_width;
pitch_val = stride / tile_width;
pitch_val = ffs(pitch_val) - 1;
val = i915_gem_obj_ggtt_offset(obj);
if (obj->tiling_mode == I915_TILING_Y)
if (tiling == I915_TILING_Y)
val |= 1 << I830_FENCE_TILING_Y_SHIFT;
val |= I915_FENCE_SIZE_BITS(size);
val |= pitch_val << I830_FENCE_PITCH_SHIFT;
@ -161,6 +165,8 @@ static void i830_write_fence_reg(struct drm_device *dev, int reg,
if (obj) {
u32 size = i915_gem_obj_ggtt_size(obj);
unsigned int tiling = i915_gem_object_get_tiling(obj);
unsigned int stride = i915_gem_object_get_stride(obj);
uint32_t pitch_val;
WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
@ -169,11 +175,11 @@ static void i830_write_fence_reg(struct drm_device *dev, int reg,
"object 0x%08llx not 512K or pot-size 0x%08x aligned\n",
i915_gem_obj_ggtt_offset(obj), size);
pitch_val = obj->stride / 128;
pitch_val = stride / 128;
pitch_val = ffs(pitch_val) - 1;
val = i915_gem_obj_ggtt_offset(obj);
if (obj->tiling_mode == I915_TILING_Y)
if (tiling == I915_TILING_Y)
val |= 1 << I830_FENCE_TILING_Y_SHIFT;
val |= I830_FENCE_SIZE_BITS(size);
val |= pitch_val << I830_FENCE_PITCH_SHIFT;
@ -201,9 +207,12 @@ static void i915_gem_write_fence(struct drm_device *dev, int reg,
if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
mb();
WARN(obj && (!obj->stride || !obj->tiling_mode),
WARN(obj &&
(!i915_gem_object_get_stride(obj) ||
!i915_gem_object_get_tiling(obj)),
"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
obj->stride, obj->tiling_mode);
i915_gem_object_get_stride(obj),
i915_gem_object_get_tiling(obj));
if (IS_GEN2(dev))
i830_write_fence_reg(dev, reg, obj);
@ -248,7 +257,7 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
if (obj->tiling_mode)
if (i915_gem_object_is_tiled(obj))
i915_gem_release_mmap(obj);
/* As we do not have an associated fence register, we will force
@ -261,15 +270,8 @@ static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
static int
i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
{
if (obj->last_fenced_req) {
int ret = i915_wait_request(obj->last_fenced_req);
if (ret)
return ret;
i915_gem_request_assign(&obj->last_fenced_req, NULL);
}
return 0;
return i915_gem_active_retire(&obj->last_fence,
&obj->base.dev->struct_mutex);
}
/**
@ -368,7 +370,7 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
{
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = to_i915(dev);
bool enable = obj->tiling_mode != I915_TILING_NONE;
bool enable = i915_gem_object_is_tiled(obj);
struct drm_i915_fence_reg *reg;
int ret;
@ -438,7 +440,7 @@ i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
WARN_ON(!ggtt_vma ||
dev_priv->fence_regs[obj->fence_reg].pin_count >
ggtt_vma->pin_count);
i915_vma_pin_count(ggtt_vma));
dev_priv->fence_regs[obj->fence_reg].pin_count++;
return true;
} else
@ -484,7 +486,7 @@ void i915_gem_restore_fences(struct drm_device *dev)
*/
if (reg->obj) {
i915_gem_object_update_fence(reg->obj, reg,
reg->obj->tiling_mode);
i915_gem_object_get_tiling(reg->obj));
} else {
i915_gem_write_fence(dev, i, NULL);
}

585
drivers/gpu/drm/i915/i915_gem_gtt.c
View file

File diff suppressed because it is too large Load diff

183
drivers/gpu/drm/i915/i915_gem_gtt.h
View file

@ -36,6 +36,8 @@
#include <linux/io-mapping.h>
#include "i915_gem_request.h"
struct drm_i915_file_private;
typedef uint32_t gen6_pte_t;
@ -178,12 +180,32 @@ struct i915_vma {
struct drm_i915_gem_object *obj;
struct i915_address_space *vm;
void __iomem *iomap;
u64 size;
unsigned int flags;
/**
* How many users have pinned this object in GTT space. The following
* users can each hold at most one reference: pwrite/pread, execbuffer
* (objects are not allowed multiple times for the same batchbuffer),
* and the framebuffer code. When switching/pageflipping, the
* framebuffer code has at most two buffers pinned per crtc.
*
* In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
* bits with absolutely no headroom. So use 4 bits.
*/
#define I915_VMA_PIN_MASK 0xf
#define I915_VMA_PIN_OVERFLOW BIT(5)
/** Flags and address space this VMA is bound to */
#define GLOBAL_BIND (1<<0)
#define LOCAL_BIND (1<<1)
unsigned int bound : 4;
bool is_ggtt : 1;
#define I915_VMA_GLOBAL_BIND BIT(6)
#define I915_VMA_LOCAL_BIND BIT(7)
#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
#define I915_VMA_GGTT BIT(8)
#define I915_VMA_CLOSED BIT(9)
unsigned int active;
struct i915_gem_active last_read[I915_NUM_ENGINES];
/**
* Support different GGTT views into the same object.
@ -208,20 +230,46 @@ struct i915_vma {
struct hlist_node exec_node;
unsigned long exec_handle;
struct drm_i915_gem_exec_object2 *exec_entry;
/**
* How many users have pinned this object in GTT space. The following
* users can each hold at most one reference: pwrite/pread, execbuffer
* (objects are not allowed multiple times for the same batchbuffer),
* and the framebuffer code. When switching/pageflipping, the
* framebuffer code has at most two buffers pinned per crtc.
*
* In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
* bits with absolutely no headroom. So use 4 bits. */
unsigned int pin_count:4;
#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
};
static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
{
return vma->flags & I915_VMA_GGTT;
}
static inline bool i915_vma_is_closed(const struct i915_vma *vma)
{
return vma->flags & I915_VMA_CLOSED;
}
static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
{
return vma->active;
}
static inline bool i915_vma_is_active(const struct i915_vma *vma)
{
return i915_vma_get_active(vma);
}
static inline void i915_vma_set_active(struct i915_vma *vma,
unsigned int engine)
{
vma->active |= BIT(engine);
}
static inline void i915_vma_clear_active(struct i915_vma *vma,
unsigned int engine)
{
vma->active &= ~BIT(engine);
}
static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
unsigned int engine)
{
return vma->active & BIT(engine);
}
struct i915_page_dma {
struct page *page;
union {
@ -272,11 +320,20 @@ struct i915_pml4 {
struct i915_address_space {
struct drm_mm mm;
struct drm_device *dev;
/* Every address space belongs to a struct file - except for the global
* GTT that is owned by the driver (and so @file is set to NULL). In
* principle, no information should leak from one context to another
* (or between files/processes etc) unless explicitly shared by the
* owner. Tracking the owner is important in order to free up per-file
* objects along with the file, to aide resource tracking, and to
* assign blame.
*/
struct drm_i915_file_private *file;
struct list_head global_link;
u64 start; /* Start offset always 0 for dri2 */
u64 total; /* size addr space maps (ex. 2GB for ggtt) */
bool is_ggtt;
bool closed;
struct i915_page_scratch *scratch_page;
struct i915_page_table *scratch_pt;
@ -306,6 +363,13 @@ struct i915_address_space {
*/
struct list_head inactive_list;
/**
* List of vma that have been unbound.
*
* A reference is not held on the buffer while on this list.
*/
struct list_head unbound_list;
/* FIXME: Need a more generic return type */
gen6_pte_t (*pte_encode)(dma_addr_t addr,
enum i915_cache_level level,
@ -338,7 +402,7 @@ struct i915_address_space {
u32 flags);
};
#define i915_is_ggtt(V) ((V)->is_ggtt)
#define i915_is_ggtt(V) (!(V)->file)
/* The Graphics Translation Table is the way in which GEN hardware translates a
* Graphics Virtual Address into a Physical Address. In addition to the normal
@ -354,7 +418,6 @@ struct i915_ggtt {
size_t stolen_usable_size; /* Total size minus BIOS reserved */
size_t stolen_reserved_base;
size_t stolen_reserved_size;
size_t size; /* Total size of Global GTT */
u64 mappable_end; /* End offset that we can CPU map */
struct io_mapping *mappable; /* Mapping to our CPU mappable region */
phys_addr_t mappable_base; /* PA of our GMADR */
@ -365,8 +428,6 @@ struct i915_ggtt {
bool do_idle_maps;
int mtrr;
int (*probe)(struct i915_ggtt *ggtt);
};
struct i915_hw_ppgtt {
@ -380,8 +441,6 @@ struct i915_hw_ppgtt {
struct i915_page_directory pd; /* GEN6-7 */
};
struct drm_i915_file_private *file_priv;
gen6_pte_t __iomem *pd_addr;
int (*enable)(struct i915_hw_ppgtt *ppgtt);
@ -521,14 +580,15 @@ i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n)
px_dma(ppgtt->base.scratch_pd);
}
int i915_ggtt_init_hw(struct drm_device *dev);
int i915_ggtt_enable_hw(struct drm_device *dev);
void i915_gem_init_ggtt(struct drm_device *dev);
void i915_ggtt_cleanup_hw(struct drm_device *dev);
int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv);
int i915_ggtt_init_hw(struct drm_i915_private *dev_priv);
int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv);
int i915_gem_init_ggtt(struct drm_i915_private *dev_priv);
void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
int i915_ppgtt_init_hw(struct drm_device *dev);
void i915_ppgtt_release(struct kref *kref);
struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev,
struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
struct drm_i915_file_private *fpriv);
static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
{
@ -562,9 +622,66 @@ i915_ggtt_view_equal(const struct i915_ggtt_view *a,
return true;
}
size_t
i915_ggtt_view_size(struct drm_i915_gem_object *obj,
const struct i915_ggtt_view *view);
/* Flags used by pin/bind&friends. */
#define PIN_NONBLOCK BIT(0)
#define PIN_MAPPABLE BIT(1)
#define PIN_ZONE_4G BIT(2)
#define PIN_MBZ BIT(5) /* I915_VMA_PIN_OVERFLOW */
#define PIN_GLOBAL BIT(6) /* I915_VMA_GLOBAL_BIND */
#define PIN_USER BIT(7) /* I915_VMA_LOCAL_BIND */
#define PIN_UPDATE BIT(8)
#define PIN_HIGH BIT(9)
#define PIN_OFFSET_BIAS BIT(10)
#define PIN_OFFSET_FIXED BIT(11)
#define PIN_OFFSET_MASK (~4095)
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
static inline int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
/* Pin early to prevent the shrinker/eviction logic from destroying
* our vma as we insert and bind.
*/
if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
return 0;
return __i915_vma_do_pin(vma, size, alignment, flags);
}
static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
return vma->flags & I915_VMA_PIN_MASK;
}
static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
{
return i915_vma_pin_count(vma);
}
static inline void __i915_vma_pin(struct i915_vma *vma)
{
vma->flags++;
GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
}
static inline void __i915_vma_unpin(struct i915_vma *vma)
{
GEM_BUG_ON(!i915_vma_is_pinned(vma));
vma->flags--;
}
static inline void i915_vma_unpin(struct i915_vma *vma)
{
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
__i915_vma_unpin(vma);
}
/**
* i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
@ -580,6 +697,7 @@ i915_ggtt_view_size(struct drm_i915_gem_object *obj,
* Returns a valid iomapped pointer or ERR_PTR.
*/
void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
/**
* i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
@ -593,9 +711,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
{
lockdep_assert_held(&vma->vm->dev->struct_mutex);
GEM_BUG_ON(vma->pin_count == 0);
GEM_BUG_ON(vma->iomap == NULL);
vma->pin_count--;
i915_vma_unpin(vma);
}
#endif

128
drivers/gpu/drm/i915/i915_gem_render_state.c
View file

@ -28,10 +28,18 @@
#include "i915_drv.h"
#include "intel_renderstate.h"
struct render_state {
const struct intel_renderstate_rodata *rodata;
struct drm_i915_gem_object *obj;
u64 ggtt_offset;
u32 aux_batch_size;
u32 aux_batch_offset;
};
static const struct intel_renderstate_rodata *
render_state_get_rodata(const int gen)
render_state_get_rodata(const struct drm_i915_gem_request *req)
{
switch (gen) {
switch (INTEL_GEN(req->i915)) {
case 6:
return &gen6_null_state;
case 7:
@ -45,35 +53,6 @@ render_state_get_rodata(const int gen)
return NULL;
}
static int render_state_init(struct render_state *so,
struct drm_i915_private *dev_priv)
{
int ret;
so->gen = INTEL_GEN(dev_priv);
so->rodata = render_state_get_rodata(so->gen);
if (so->rodata == NULL)
return 0;
if (so->rodata->batch_items * 4 > 4096)
return -EINVAL;
so->obj = i915_gem_object_create(&dev_priv->drm, 4096);
if (IS_ERR(so->obj))
return PTR_ERR(so->obj);
ret = i915_gem_obj_ggtt_pin(so->obj, 4096, 0);
if (ret)
goto free_gem;
so->ggtt_offset = i915_gem_obj_ggtt_offset(so->obj);
return 0;
free_gem:
drm_gem_object_unreference(&so->obj->base);
return ret;
}
/*
* Macro to add commands to auxiliary batch.
* This macro only checks for page overflow before inserting the commands,
@ -96,6 +75,7 @@ static int render_state_setup(struct render_state *so)
{
struct drm_device *dev = so->obj->base.dev;
const struct intel_renderstate_rodata *rodata = so->rodata;
const bool has_64bit_reloc = INTEL_GEN(dev) >= 8;
unsigned int i = 0, reloc_index = 0;
struct page *page;
u32 *d;
@ -114,7 +94,7 @@ static int render_state_setup(struct render_state *so)
if (i * 4 == rodata->reloc[reloc_index]) {
u64 r = s + so->ggtt_offset;
s = lower_32_bits(r);
if (so->gen >= 8) {
if (has_64bit_reloc) {
if (i + 1 >= rodata->batch_items ||
rodata->batch[i + 1] != 0) {
ret = -EINVAL;
@ -192,67 +172,55 @@ err_out:
#undef OUT_BATCH
void i915_gem_render_state_fini(struct render_state *so)
{
i915_gem_object_ggtt_unpin(so->obj);
drm_gem_object_unreference(&so->obj->base);
}
int i915_gem_render_state_prepare(struct intel_engine_cs *engine,
struct render_state *so)
{
int ret;
if (WARN_ON(engine->id != RCS))
return -ENOENT;
ret = render_state_init(so, engine->i915);
if (ret)
return ret;
if (so->rodata == NULL)
return 0;
ret = render_state_setup(so);
if (ret) {
i915_gem_render_state_fini(so);
return ret;
}
return 0;
}
int i915_gem_render_state_init(struct drm_i915_gem_request *req)
{
struct render_state so;
int ret;
ret = i915_gem_render_state_prepare(req->engine, &so);
if (ret)
return ret;
if (WARN_ON(req->engine->id != RCS))
return -ENOENT;
if (so.rodata == NULL)
so.rodata = render_state_get_rodata(req);
if (!so.rodata)
return 0;
ret = req->engine->dispatch_execbuffer(req, so.ggtt_offset,
so.rodata->batch_items * 4,
I915_DISPATCH_SECURE);
if (so.rodata->batch_items * 4 > 4096)
return -EINVAL;
so.obj = i915_gem_object_create(&req->i915->drm, 4096);
if (IS_ERR(so.obj))
return PTR_ERR(so.obj);
ret = i915_gem_object_ggtt_pin(so.obj, NULL, 0, 0, 0);
if (ret)
goto out;
goto err_obj;
so.ggtt_offset = i915_gem_obj_ggtt_offset(so.obj);
ret = render_state_setup(&so);
if (ret)
goto err_unpin;
ret = req->engine->emit_bb_start(req, so.ggtt_offset,
so.rodata->batch_items * 4,
I915_DISPATCH_SECURE);
if (ret)
goto err_unpin;
if (so.aux_batch_size > 8) {
ret = req->engine->dispatch_execbuffer(req,
(so.ggtt_offset +
so.aux_batch_offset),
so.aux_batch_size,
I915_DISPATCH_SECURE);
ret = req->engine->emit_bb_start(req,
(so.ggtt_offset +
so.aux_batch_offset),
so.aux_batch_size,
I915_DISPATCH_SECURE);
if (ret)
goto out;
goto err_unpin;
}
i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req);
out:
i915_gem_render_state_fini(&so);
i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req, 0);
err_unpin:
i915_gem_object_ggtt_unpin(so.obj);
err_obj:
i915_gem_object_put(so.obj);
return ret;
}

18
drivers/gpu/drm/i915/i915_gem_render_state.h
View file

@ -26,24 +26,6 @@
#include <linux/types.h>
struct intel_renderstate_rodata {
const u32 *reloc;
const u32 *batch;
const u32 batch_items;
};
struct render_state {
const struct intel_renderstate_rodata *rodata;
struct drm_i915_gem_object *obj;
u64 ggtt_offset;
int gen;
u32 aux_batch_size;
u32 aux_batch_offset;
};
int i915_gem_render_state_init(struct drm_i915_gem_request *req);
void i915_gem_render_state_fini(struct render_state *so);
int i915_gem_render_state_prepare(struct intel_engine_cs *engine,
struct render_state *so);
#endif /* _I915_GEM_RENDER_STATE_H_ */

767
drivers/gpu/drm/i915/i915_gem_request.c Normal file
View file

@ -0,0 +1,767 @@
/*
* Copyright © 2008-2015 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*
*/
#include <linux/prefetch.h>
#include "i915_drv.h"
static const char *i915_fence_get_driver_name(struct fence *fence)
{
return "i915";
}
static const char *i915_fence_get_timeline_name(struct fence *fence)
{
/* Timelines are bound by eviction to a VM. However, since
* we only have a global seqno at the moment, we only have
* a single timeline. Note that each timeline will have
* multiple execution contexts (fence contexts) as we allow
* engines within a single timeline to execute in parallel.
*/
return "global";
}
static bool i915_fence_signaled(struct fence *fence)
{
return i915_gem_request_completed(to_request(fence));
}
static bool i915_fence_enable_signaling(struct fence *fence)
{
if (i915_fence_signaled(fence))
return false;
intel_engine_enable_signaling(to_request(fence));
return true;
}
static signed long i915_fence_wait(struct fence *fence,
bool interruptible,
signed long timeout_jiffies)
{
s64 timeout_ns, *timeout;
int ret;
if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) {
timeout_ns = jiffies_to_nsecs(timeout_jiffies);
timeout = &timeout_ns;
} else {
timeout = NULL;
}
ret = i915_wait_request(to_request(fence),
interruptible, timeout,
NO_WAITBOOST);
if (ret == -ETIME)
return 0;
if (ret < 0)
return ret;
if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT)
timeout_jiffies = nsecs_to_jiffies(timeout_ns);
return timeout_jiffies;
}
static void i915_fence_value_str(struct fence *fence, char *str, int size)
{
snprintf(str, size, "%u", fence->seqno);
}
static void i915_fence_timeline_value_str(struct fence *fence, char *str,
int size)
{
snprintf(str, size, "%u",
intel_engine_get_seqno(to_request(fence)->engine));
}
static void i915_fence_release(struct fence *fence)
{
struct drm_i915_gem_request *req = to_request(fence);
kmem_cache_free(req->i915->requests, req);
}
const struct fence_ops i915_fence_ops = {
.get_driver_name = i915_fence_get_driver_name,
.get_timeline_name = i915_fence_get_timeline_name,
.enable_signaling = i915_fence_enable_signaling,
.signaled = i915_fence_signaled,
.wait = i915_fence_wait,
.release = i915_fence_release,
.fence_value_str = i915_fence_value_str,
.timeline_value_str = i915_fence_timeline_value_str,
};
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
struct drm_file *file)
{
struct drm_i915_private *dev_private;
struct drm_i915_file_private *file_priv;
WARN_ON(!req || !file || req->file_priv);
if (!req || !file)
return -EINVAL;
if (req->file_priv)
return -EINVAL;
dev_private = req->i915;
file_priv = file->driver_priv;
spin_lock(&file_priv->mm.lock);
req->file_priv = file_priv;
list_add_tail(&req->client_list, &file_priv->mm.request_list);
spin_unlock(&file_priv->mm.lock);
req->pid = get_pid(task_pid(current));
return 0;
}
static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
{
struct drm_i915_file_private *file_priv = request->file_priv;
if (!file_priv)
return;
spin_lock(&file_priv->mm.lock);
list_del(&request->client_list);
request->file_priv = NULL;
spin_unlock(&file_priv->mm.lock);
put_pid(request->pid);
request->pid = NULL;
}
void i915_gem_retire_noop(struct i915_gem_active *active,
struct drm_i915_gem_request *request)
{
/* Space left intentionally blank */
}
static void i915_gem_request_retire(struct drm_i915_gem_request *request)
{
struct i915_gem_active *active, *next;
trace_i915_gem_request_retire(request);
list_del(&request->link);
/* We know the GPU must have read the request to have
* sent us the seqno + interrupt, so use the position
* of tail of the request to update the last known position
* of the GPU head.
*
* Note this requires that we are always called in request
* completion order.
*/
list_del(&request->ring_link);
request->ring->last_retired_head = request->postfix;
/* Walk through the active list, calling retire on each. This allows
* objects to track their GPU activity and mark themselves as idle
* when their *last* active request is completed (updating state
* tracking lists for eviction, active references for GEM, etc).
*
* As the ->retire() may free the node, we decouple it first and
* pass along the auxiliary information (to avoid dereferencing
* the node after the callback).
*/
list_for_each_entry_safe(active, next, &request->active_list, link) {
/* In microbenchmarks or focusing upon time inside the kernel,
* we may spend an inordinate amount of time simply handling
* the retirement of requests and processing their callbacks.
* Of which, this loop itself is particularly hot due to the
* cache misses when jumping around the list of i915_gem_active.
* So we try to keep this loop as streamlined as possible and
* also prefetch the next i915_gem_active to try and hide
* the likely cache miss.
*/
prefetchw(next);
INIT_LIST_HEAD(&active->link);
RCU_INIT_POINTER(active->request, NULL);
active->retire(active, request);
}
i915_gem_request_remove_from_client(request);
if (request->previous_context) {
if (i915.enable_execlists)
intel_lr_context_unpin(request->previous_context,
request->engine);
}
i915_gem_context_put(request->ctx);
i915_gem_request_put(request);
}
void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
{
struct intel_engine_cs *engine = req->engine;
struct drm_i915_gem_request *tmp;
lockdep_assert_held(&req->i915->drm.struct_mutex);
GEM_BUG_ON(list_empty(&req->link));
do {
tmp = list_first_entry(&engine->request_list,
typeof(*tmp), link);
i915_gem_request_retire(tmp);
} while (tmp != req);
}
static int i915_gem_check_wedge(unsigned int reset_counter, bool interruptible)
{
if (__i915_terminally_wedged(reset_counter))
return -EIO;
if (__i915_reset_in_progress(reset_counter)) {
/* Non-interruptible callers can't handle -EAGAIN, hence return
* -EIO unconditionally for these.
*/
if (!interruptible)
return -EIO;
return -EAGAIN;
}
return 0;
}
static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
{
struct intel_engine_cs *engine;
int ret;
/* Carefully retire all requests without writing to the rings */
for_each_engine(engine, dev_priv) {
ret = intel_engine_idle(engine, true);
if (ret)
return ret;
}
i915_gem_retire_requests(dev_priv);
/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
while (intel_kick_waiters(dev_priv) ||
intel_kick_signalers(dev_priv))
yield();
}
/* Finally reset hw state */
for_each_engine(engine, dev_priv)
intel_engine_init_seqno(engine, seqno);
return 0;
}
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
struct drm_i915_private *dev_priv = to_i915(dev);
int ret;
if (seqno == 0)
return -EINVAL;
/* HWS page needs to be set less than what we
* will inject to ring
*/
ret = i915_gem_init_seqno(dev_priv, seqno - 1);
if (ret)
return ret;
dev_priv->next_seqno = seqno;
return 0;
}
static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
{
/* reserve 0 for non-seqno */
if (unlikely(dev_priv->next_seqno == 0)) {
int ret;
ret = i915_gem_init_seqno(dev_priv, 0);
if (ret)
return ret;
dev_priv->next_seqno = 1;
}
*seqno = dev_priv->next_seqno++;
return 0;
}
/**
* i915_gem_request_alloc - allocate a request structure
*
* @engine: engine that we wish to issue the request on.
* @ctx: context that the request will be associated with.
* This can be NULL if the request is not directly related to
* any specific user context, in which case this function will
* choose an appropriate context to use.
*
* Returns a pointer to the allocated request if successful,
* or an error code if not.
*/
struct drm_i915_gem_request *
i915_gem_request_alloc(struct intel_engine_cs *engine,
struct i915_gem_context *ctx)
{
struct drm_i915_private *dev_priv = engine->i915;
unsigned int reset_counter = i915_reset_counter(&dev_priv->gpu_error);
struct drm_i915_gem_request *req;
u32 seqno;
int ret;
/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
* EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
* and restart.
*/
ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
if (ret)
return ERR_PTR(ret);
/* Move the oldest request to the slab-cache (if not in use!) */
req = list_first_entry_or_null(&engine->request_list,
typeof(*req), link);
if (req && i915_gem_request_completed(req))
i915_gem_request_retire(req);
req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
if (!req)
return ERR_PTR(-ENOMEM);
ret = i915_gem_get_seqno(dev_priv, &seqno);
if (ret)
goto err;
spin_lock_init(&req->lock);
fence_init(&req->fence,
&i915_fence_ops,
&req->lock,
engine->fence_context,
seqno);
INIT_LIST_HEAD(&req->active_list);
req->i915 = dev_priv;
req->engine = engine;
req->ctx = i915_gem_context_get(ctx);
/*
* Reserve space in the ring buffer for all the commands required to
* eventually emit this request. This is to guarantee that the
* i915_add_request() call can't fail. Note that the reserve may need
* to be redone if the request is not actually submitted straight
* away, e.g. because a GPU scheduler has deferred it.
*/
req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
if (i915.enable_execlists)
ret = intel_logical_ring_alloc_request_extras(req);
else
ret = intel_ring_alloc_request_extras(req);
if (ret)
goto err_ctx;
return req;
err_ctx:
i915_gem_context_put(ctx);
err:
kmem_cache_free(dev_priv->requests, req);
return ERR_PTR(ret);
}
static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
{
struct drm_i915_private *dev_priv = engine->i915;
dev_priv->gt.active_engines |= intel_engine_flag(engine);
if (dev_priv->gt.awake)
return;
intel_runtime_pm_get_noresume(dev_priv);
dev_priv->gt.awake = true;
intel_enable_gt_powersave(dev_priv);
i915_update_gfx_val(dev_priv);
if (INTEL_GEN(dev_priv) >= 6)
gen6_rps_busy(dev_priv);
queue_delayed_work(dev_priv->wq,
&dev_priv->gt.retire_work,
round_jiffies_up_relative(HZ));
}
/*
* NB: This function is not allowed to fail. Doing so would mean the the
* request is not being tracked for completion but the work itself is
* going to happen on the hardware. This would be a Bad Thing(tm).
*/
void __i915_add_request(struct drm_i915_gem_request *request,
struct drm_i915_gem_object *obj,
bool flush_caches)
{
struct intel_engine_cs *engine;
struct intel_ring *ring;
u32 request_start;
u32 reserved_tail;
int ret;
if (WARN_ON(!request))
return;
engine = request->engine;
ring = request->ring;
/*
* To ensure that this call will not fail, space for its emissions
* should already have been reserved in the ring buffer. Let the ring
* know that it is time to use that space up.
*/
request_start = ring->tail;
reserved_tail = request->reserved_space;
request->reserved_space = 0;
/*
* Emit any outstanding flushes - execbuf can fail to emit the flush
* after having emitted the batchbuffer command. Hence we need to fix
* things up similar to emitting the lazy request. The difference here
* is that the flush _must_ happen before the next request, no matter
* what.
*/
if (flush_caches) {
ret = engine->emit_flush(request, EMIT_FLUSH);
/* Not allowed to fail! */
WARN(ret, "engine->emit_flush() failed: %d!\n", ret);
}
trace_i915_gem_request_add(request);
request->head = request_start;
/* Whilst this request exists, batch_obj will be on the
* active_list, and so will hold the active reference. Only when this
* request is retired will the the batch_obj be moved onto the
* inactive_list and lose its active reference. Hence we do not need
* to explicitly hold another reference here.
*/
request->batch_obj = obj;
/* Seal the request and mark it as pending execution. Note that
* we may inspect this state, without holding any locks, during
* hangcheck. Hence we apply the barrier to ensure that we do not
* see a more recent value in the hws than we are tracking.
*/
request->emitted_jiffies = jiffies;
request->previous_seqno = engine->last_submitted_seqno;
engine->last_submitted_seqno = request->fence.seqno;
i915_gem_active_set(&engine->last_request, request);
list_add_tail(&request->link, &engine->request_list);
list_add_tail(&request->ring_link, &ring->request_list);
/* Record the position of the start of the request so that
* should we detect the updated seqno part-way through the
* GPU processing the request, we never over-estimate the
* position of the head.
*/
request->postfix = ring->tail;
/* Not allowed to fail! */
ret = engine->emit_request(request);
WARN(ret, "(%s)->emit_request failed: %d!\n", engine->name, ret);
/* Sanity check that the reserved size was large enough. */
ret = ring->tail - request_start;
if (ret < 0)
ret += ring->size;
WARN_ONCE(ret > reserved_tail,
"Not enough space reserved (%d bytes) "
"for adding the request (%d bytes)\n",
reserved_tail, ret);
i915_gem_mark_busy(engine);
engine->submit_request(request);
}
static unsigned long local_clock_us(unsigned int *cpu)
{
unsigned long t;
/* Cheaply and approximately convert from nanoseconds to microseconds.
* The result and subsequent calculations are also defined in the same
* approximate microseconds units. The principal source of timing
* error here is from the simple truncation.
*
* Note that local_clock() is only defined wrt to the current CPU;
* the comparisons are no longer valid if we switch CPUs. Instead of
* blocking preemption for the entire busywait, we can detect the CPU
* switch and use that as indicator of system load and a reason to
* stop busywaiting, see busywait_stop().
*/
*cpu = get_cpu();
t = local_clock() >> 10;
put_cpu();
return t;
}
static bool busywait_stop(unsigned long timeout, unsigned int cpu)
{
unsigned int this_cpu;
if (time_after(local_clock_us(&this_cpu), timeout))
return true;
return this_cpu != cpu;
}
bool __i915_spin_request(const struct drm_i915_gem_request *req,
int state, unsigned long timeout_us)
{
unsigned int cpu;
/* When waiting for high frequency requests, e.g. during synchronous
* rendering split between the CPU and GPU, the finite amount of time
* required to set up the irq and wait upon it limits the response
* rate. By busywaiting on the request completion for a short while we
* can service the high frequency waits as quick as possible. However,
* if it is a slow request, we want to sleep as quickly as possible.
* The tradeoff between waiting and sleeping is roughly the time it
* takes to sleep on a request, on the order of a microsecond.
*/
timeout_us += local_clock_us(&cpu);
do {
if (i915_gem_request_completed(req))
return true;
if (signal_pending_state(state, current))
break;
if (busywait_stop(timeout_us, cpu))
break;
cpu_relax_lowlatency();
} while (!need_resched());
return false;
}
/**
* i915_wait_request - wait until execution of request has finished
* @req: duh!
* @interruptible: do an interruptible wait (normally yes)
* @timeout: in - how long to wait (NULL forever); out - how much time remaining
* @rps: client to charge for RPS boosting
*
* Note: It is of utmost importance that the passed in seqno and reset_counter
* values have been read by the caller in an smp safe manner. Where read-side
* locks are involved, it is sufficient to read the reset_counter before
* unlocking the lock that protects the seqno. For lockless tricks, the
* reset_counter _must_ be read before, and an appropriate smp_rmb must be
* inserted.
*
* Returns 0 if the request was found within the alloted time. Else returns the
* errno with remaining time filled in timeout argument.
*/
int i915_wait_request(struct drm_i915_gem_request *req,
bool interruptible,
s64 *timeout,
struct intel_rps_client *rps)
{
int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
DEFINE_WAIT(reset);
struct intel_wait wait;
unsigned long timeout_remain;
int ret = 0;
might_sleep();
if (i915_gem_request_completed(req))
return 0;
timeout_remain = MAX_SCHEDULE_TIMEOUT;
if (timeout) {
if (WARN_ON(*timeout < 0))
return -EINVAL;
if (*timeout == 0)
return -ETIME;
/* Record current time in case interrupted, or wedged */
timeout_remain = nsecs_to_jiffies_timeout(*timeout);
*timeout += ktime_get_raw_ns();
}
trace_i915_gem_request_wait_begin(req);
/* This client is about to stall waiting for the GPU. In many cases
* this is undesirable and limits the throughput of the system, as
* many clients cannot continue processing user input/output whilst
* blocked. RPS autotuning may take tens of milliseconds to respond
* to the GPU load and thus incurs additional latency for the client.
* We can circumvent that by promoting the GPU frequency to maximum
* before we wait. This makes the GPU throttle up much more quickly
* (good for benchmarks and user experience, e.g. window animations),
* but at a cost of spending more power processing the workload
* (bad for battery). Not all clients even want their results
* immediately and for them we should just let the GPU select its own
* frequency to maximise efficiency. To prevent a single client from
* forcing the clocks too high for the whole system, we only allow
* each client to waitboost once in a busy period.
*/
if (IS_RPS_CLIENT(rps) && INTEL_GEN(req->i915) >= 6)
gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
/* Optimistic short spin before touching IRQs */
if (i915_spin_request(req, state, 5))
goto complete;
set_current_state(state);
add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
intel_wait_init(&wait, req->fence.seqno);
if (intel_engine_add_wait(req->engine, &wait))
/* In order to check that we haven't missed the interrupt
* as we enabled it, we need to kick ourselves to do a
* coherent check on the seqno before we sleep.
*/
goto wakeup;
for (;;) {
if (signal_pending_state(state, current)) {
ret = -ERESTARTSYS;
break;
}
timeout_remain = io_schedule_timeout(timeout_remain);
if (timeout_remain == 0) {
ret = -ETIME;
break;
}
if (intel_wait_complete(&wait))
break;
set_current_state(state);
wakeup:
/* Carefully check if the request is complete, giving time
* for the seqno to be visible following the interrupt.
* We also have to check in case we are kicked by the GPU
* reset in order to drop the struct_mutex.
*/
if (__i915_request_irq_complete(req))
break;
/* Only spin if we know the GPU is processing this request */
if (i915_spin_request(req, state, 2))
break;
}
remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
intel_engine_remove_wait(req->engine, &wait);
__set_current_state(TASK_RUNNING);
complete:
trace_i915_gem_request_wait_end(req);
if (timeout) {
*timeout -= ktime_get_raw_ns();
if (*timeout < 0)
*timeout = 0;
/*
* Apparently ktime isn't accurate enough and occasionally has a
* bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
* things up to make the test happy. We allow up to 1 jiffy.
*
* This is a regrssion from the timespec->ktime conversion.
*/
if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
*timeout = 0;
}
if (IS_RPS_USER(rps) &&
req->fence.seqno == req->engine->last_submitted_seqno) {
/* The GPU is now idle and this client has stalled.
* Since no other client has submitted a request in the
* meantime, assume that this client is the only one
* supplying work to the GPU but is unable to keep that
* work supplied because it is waiting. Since the GPU is
* then never kept fully busy, RPS autoclocking will
* keep the clocks relatively low, causing further delays.
* Compensate by giving the synchronous client credit for
* a waitboost next time.
*/
spin_lock(&req->i915->rps.client_lock);
list_del_init(&rps->link);
spin_unlock(&req->i915->rps.client_lock);
}
return ret;
}
static void engine_retire_requests(struct intel_engine_cs *engine)
{
struct drm_i915_gem_request *request, *next;
list_for_each_entry_safe(request, next, &engine->request_list, link) {
if (!i915_gem_request_completed(request))
break;
i915_gem_request_retire(request);
}
}
void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
{
struct intel_engine_cs *engine;
lockdep_assert_held(&dev_priv->drm.struct_mutex);
if (dev_priv->gt.active_engines == 0)
return;
GEM_BUG_ON(!dev_priv->gt.awake);
for_each_engine(engine, dev_priv) {
engine_retire_requests(engine);
if (!intel_engine_is_active(engine))
dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
}
if (dev_priv->gt.active_engines == 0)
queue_delayed_work(dev_priv->wq,
&dev_priv->gt.idle_work,
msecs_to_jiffies(100));
}

Some files were not shown because too many files have changed in this diff Show more