# Driver version string which is returned to userspace via an ioctl
-MALI_RELEASE_NAME ?= "r11p0-00rel0"
+MALI_RELEASE_NAME ?= "r12p0-04rel0"
# Paths required for build
KBASE_PATH = $(src)
mali_kbase_smc.c \
mali_kbase_mem_pool.c \
mali_kbase_mem_pool_debugfs.c \
- mali_kbase_tlstream.c
+ mali_kbase_tlstream.c \
+ mali_kbase_strings.c
ifeq ($(MALI_UNIT_TEST),1)
SRC += mali_kbase_tlstream_test.c
SRC += mali_kbase_regs_dump_debugfs.c
endif
+
# Job Scheduler Policy: Completely Fair Scheduler
SRC += mali_kbase_js_policy_cfs.c
# Tell the Linux build system to enable building of our .c files
mali_kbase-y := $(SRC:.c=.o)
+mali_kbase-$(CONFIG_MALI_DMA_FENCE) += mali_kbase_dma_fence.o
+
ifneq ($(wildcard $(src)/internal/Kbuild),)
ifeq ($(MALI_CUSTOMER_RELEASE),0)
-# This include may set MALI_BACKEND_PATH and CONFIG_MALI_BACKEND_REAL
include $(src)/internal/Kbuild
mali_kbase-y += $(INTERNAL:.c=.o)
endif
governor, the frequency of Mali will be dynamically selected from the
available OPPs.
+config MALI_DMA_FENCE
+ bool "DMA_BUF fence support for Mali (EXPERIMENTAL)"
+ depends on MALI_MIDGARD && !KDS
+ default n
+ help
+ Support DMA_BUF fences for Mali.
+
+ This option should only be enabled if KDS is not present and
+ the Linux Kernel has built in support for DMA_BUF fences.
# MALI_EXPERT configuration options
#
-# (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers
endif
-ifneq ($(wildcard $(CURDIR)/internal/Makefile.in),)
-include $(CURDIR)/internal/Makefile.in
-endif
-
ifeq ($(MALI_BUS_LOG), 1)
#Add bus logger symbols
EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers
/*
*
- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID);
KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value);
+
writel(value, kbdev->reg + offset);
+
if (kctx && kctx->jctx.tb)
kbase_device_trace_register_access(kctx, REG_WRITE, offset,
value);
KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID);
KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
+
val = readl(kbdev->reg + offset);
+
dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val);
if (kctx && kctx->jctx.tb)
kbase_device_trace_register_access(kctx, REG_READ, offset, val);
/*
*
- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
if (err)
goto fail_timer;
-/* Currently disabled on the prototype */
#ifdef CONFIG_MALI_DEBUG
#ifndef CONFIG_MALI_NO_MALI
if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
return 0;
fail_job_slot:
-/* Currently disabled on the prototype */
+
#ifdef CONFIG_MALI_DEBUG
#ifndef CONFIG_MALI_NO_MALI
fail_interrupt_test:
#endif /* !CONFIG_MALI_NO_MALI */
#endif /* CONFIG_MALI_DEBUG */
+
kbase_backend_timer_term(kbdev);
fail_timer:
kbase_hwaccess_pm_halt(kbdev);
/*
*
- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define MMU_IRQ_TAG 1
#define GPU_IRQ_TAG 2
-
static void *kbase_tag(void *ptr, u32 tag)
{
return (void *)(((uintptr_t) ptr) | tag);
return (void *)(((uintptr_t) ptr) & ~3);
}
-
-
-
static irqreturn_t kbase_job_irq_handler(int irq, void *data)
{
unsigned long flags;
return IRQ_HANDLED;
}
+
static irq_handler_t kbase_handler_table[] = {
[JOB_IRQ_TAG] = kbase_job_irq_handler,
[MMU_IRQ_TAG] = kbase_mmu_irq_handler,
[GPU_IRQ_TAG] = kbase_gpu_irq_handler,
};
-
#ifdef CONFIG_MALI_DEBUG
#define JOB_IRQ_HANDLER JOB_IRQ_TAG
#define MMU_IRQ_HANDLER MMU_IRQ_TAG
}
}
-#if (defined(MALI_MIPE_ENABLED) && MALI_MIPE_ENABLED) || \
- !defined(MALI_MIPE_ENABLED)
/**
* kbasep_trace_tl_nret_atom_lpu - Call nret_atom_lpu timeline tracepoint
* @kbdev: kbase device
kbase_tlstream_tl_nret_atom_lpu(katom,
&kbdev->gpu_props.props.raw_props.js_features[i]);
}
-#endif
void kbase_job_done(struct kbase_device *kbdev, u32 done)
{
kbase_tlstream_aux_job_softstop(i);
-#if (defined(MALI_MIPE_ENABLED) && MALI_MIPE_ENABLED) || \
- !defined(MALI_MIPE_ENABLED)
kbasep_trace_tl_nret_atom_lpu(
kbdev, i);
-#endif
/* Soft-stopped job - read the value of
* JS<n>_TAIL so that the job chain can
/*
*
- * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
BASE_HW_FEATURE_END
};
+
+
#endif /* _BASE_HWCONFIG_FEATURES_H_ */
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_26,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_TMIX_7940,
BASE_HW_ISSUE_TMIX_8042,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_3964,
GPUCORE_1619,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_11012,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_26,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_26,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_26,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_26,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_26,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3964,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
GPUCORE_1619,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3964,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
GPUCORE_1619,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3964,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
GPUCORE_1619,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3964,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
GPUCORE_1619,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_T76X_3964,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11051,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3979,
BASE_HW_ISSUE_TMIX_7891,
GPUCORE_1619,
BASE_HW_ISSUE_END
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_11054,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_TMIX_7891,
BASE_HW_ISSUE_TMIX_7940,
BASE_HW_ISSUE_END
};
+
+
+
+
#endif /* _BASE_HWCONFIG_ISSUES_H_ */
BASE_MEM_SECURE = (1U << 16), /**< Secure memory */
BASE_MEM_DONT_NEED = (1U << 17), /**< Not needed physical
memory */
-
+ BASE_MEM_IMPORT_SHARED = (1U << 18), /**< Must use shared CPU/GPU zone
+ (SAME_VA zone) but doesn't
+ require the addresses to
+ be the same */
};
/**
*
* Must be kept in sync with the ::base_mem_alloc_flags flags
*/
-#define BASE_MEM_FLAGS_NR_BITS 18
+#define BASE_MEM_FLAGS_NR_BITS 19
/**
* A mask for all output bits, excluding IN/OUT bits.
* If this bit is set then completion of this atom will not cause an event to
* be sent to userspace, whether successful or not; completion events will be
* deferred until an atom completes which does not have this bit set.
+ *
+ * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES.
*/
#define BASE_JD_REQ_EVENT_COALESCE (1U << 5)
* but should instead be part of a NULL jobs inserted into the dependency tree.
* The first pre_dep object must be configured for the external resouces to use,
* the second pre_dep object can be used to create other dependencies.
+ *
+ * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE.
*/
#define BASE_JD_REQ_EXTERNAL_RESOURCES (1U << 8)
struct base_jd_debug_copy_buffer {
u64 address;
u64 size;
+ struct base_external_resource extres;
};
/**
#include <mali_kbase_uku.h>
#include <mali_kbase_linux.h>
+#include "mali_kbase_strings.h"
#include "mali_kbase_pm.h"
#include "mali_kbase_mem_lowlevel.h"
#include "mali_kbase_defs.h"
bool jd_submit_atom(struct kbase_context *kctx,
const struct base_jd_atom_v2 *user_atom,
struct kbase_jd_atom *katom);
+void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom);
void kbase_job_done(struct kbase_device *kbdev, u32 done);
void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value);
u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset);
-
void kbasep_as_do_poke(struct work_struct *work);
/** Returns the name associated with a Mali exception code
kctx->process_mm = NULL;
atomic_set(&kctx->nonmapped_pages, 0);
kctx->slots_pullable = 0;
+ kctx->tgid = current->tgid;
+ kctx->pid = current->pid;
err = kbase_mem_pool_init(&kctx->mem_pool,
kbdev->mem_pool_max_size_default,
#ifdef CONFIG_KDS
INIT_LIST_HEAD(&kctx->waiting_kds_resource);
#endif
+ err = kbase_dma_fence_init(kctx);
+ if (err)
+ goto free_event;
err = kbase_mmu_init(kctx);
if (err)
- goto free_event;
+ goto term_dma_fence;
kctx->pgd = kbase_mmu_alloc_pgd(kctx);
if (!kctx->pgd)
if (!kctx->aliasing_sink_page)
goto no_sink_page;
- kctx->tgid = current->tgid;
- kctx->pid = current->pid;
init_waitqueue_head(&kctx->event_queue);
kctx->cookies = KBASE_COOKIE_MASK;
kbase_gpu_vm_unlock(kctx);
free_mmu:
kbase_mmu_term(kctx);
+term_dma_fence:
+ kbase_dma_fence_term(kctx);
free_event:
kbase_event_cleanup(kctx);
free_jd:
kbase_pm_context_idle(kbdev);
+ kbase_dma_fence_term(kctx);
+
kbase_mmu_term(kctx);
pages = atomic_read(&kctx->used_pages);
#include <mali_kbase_config.h>
-#ifdef CONFIG_MACH_MANTA
-#include <plat/devs.h>
-#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
#include <linux/pm_opp.h>
EXPORT_SYMBOL(shared_kernel_test_data);
#endif /* MALI_UNIT_TEST */
-#define KBASE_DRV_NAME "mali"
/** rk_ext : version of rk_ext on mali_ko, aka. rk_ko_ver. */
#define ROCKCHIP_VERSION (13)
-static const char kbase_drv_name[] = KBASE_DRV_NAME;
-
static int kbase_dev_nr;
static DEFINE_MUTEX(kbase_dev_list_lock);
}
#endif /* !CONFIG_MALI_NO_MALI */
-
/** Show callback for the @c power_policy sysfs file.
*
* This function is called to get the contents of the @c power_policy sysfs
{ .id = GPU_ID_PI_T86X, .name = "Mali-T86x" },
{ .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" },
{ .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT,
- .name = "Mali-TMIx" },
+ .name = "Mali-G71" },
};
const char *product_name = "(Unknown Mali GPU)";
struct kbase_device *kbdev;
static int registers_map(struct kbase_device * const kbdev)
{
+
/* the first memory resource is the physical address of the GPU
* registers */
struct platform_device *pdev = to_platform_device(kbdev->dev);
#endif /* KBASE_GPU_RESET_EN */
-static int kbasep_secure_mode_seq_show(struct seq_file *m, void *p)
-{
- struct kbase_device *kbdev = m->private;
-
- if (!kbdev->secure_mode_support)
- seq_puts(m, "unsupported\n");
- else
- seq_printf(m, "%s\n", kbdev->secure_mode ? "Y" : "N");
-
- return 0;
-}
-
-static int kbasep_secure_mode_debugfs_open(struct inode *in, struct file *file)
-{
- return single_open(file, kbasep_secure_mode_seq_show, in->i_private);
-}
-
-static const struct file_operations kbasep_secure_mode_debugfs_fops = {
- .open = kbasep_secure_mode_debugfs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
-
static int kbase_device_debugfs_init(struct kbase_device *kbdev)
{
struct dentry *debugfs_ctx_defaults_directory;
kbasep_trace_timeline_debugfs_init(kbdev);
#endif /* CONFIG_MALI_TRACE_TIMELINE */
- debugfs_create_file("secure_mode", S_IRUGO,
- kbdev->mali_debugfs_directory, kbdev,
- &kbasep_secure_mode_debugfs_fops);
-
return 0;
out:
}
#endif
-
static struct attribute *kbase_attrs[] = {
#ifdef CONFIG_MALI_DEBUG
&dev_attr_debug_command.attr,
kbdev->inited_subsys &= ~inited_backend_early;
}
-
if (kbdev->inited_subsys & inited_power_control) {
power_control_term(kbdev);
kbdev->inited_subsys &= ~inited_power_control;
}
kbdev->inited_subsys |= inited_power_control;
-
err = kbase_backend_early_init(kbdev);
if (err) {
dev_err(kbdev->dev, "Early backend initialization failed\n");
if (ret)
return ret;
-#ifndef CONFIG_MACH_MANTA
#ifdef CONFIG_MALI_PLATFORM_FAKE
ret = kbase_platform_fake_register();
if (ret)
return ret;
-#endif
#endif
ret = platform_driver_register(&kbase_platform_driver);
-#ifndef CONFIG_MACH_MANTA
#ifdef CONFIG_MALI_PLATFORM_FAKE
if (ret)
kbase_platform_fake_unregister();
-#endif
#endif
return ret;
}
static void __exit kbase_driver_exit(void)
{
platform_driver_unregister(&kbase_platform_driver);
-#ifndef CONFIG_MACH_MANTA
#ifdef CONFIG_MALI_PLATFORM_FAKE
kbase_platform_fake_unregister();
#endif
-#endif
}
module_init(kbase_driver_init);
/*
*
- * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
+#include <linux/spinlock.h>
#include "mali_kbase_debug_job_fault.h"
#ifdef CONFIG_DEBUG_FS
-static bool kbase_is_job_fault_event_pending(struct list_head *event_list)
+static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev)
{
- bool ret;
+ struct list_head *event_list = &kbdev->job_fault_event_list;
+ unsigned long flags;
+ bool ret;
- ret = (!list_empty(event_list));
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
+ ret = !list_empty(event_list);
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
return ret;
}
-static bool kbase_ctx_has_no_event_pending(
- struct kbase_context *kctx, struct list_head *event_list)
+static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx)
{
+ struct kbase_device *kbdev = kctx->kbdev;
+ struct list_head *event_list = &kctx->kbdev->job_fault_event_list;
struct base_job_fault_event *event;
+ unsigned long flags;
- if (list_empty(event_list))
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
+ if (list_empty(event_list)) {
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
return true;
+ }
list_for_each_entry(event, event_list, head) {
- if (event->katom->kctx == kctx)
+ if (event->katom->kctx == kctx) {
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock,
+ flags);
return false;
+ }
}
- return false;
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
+ return true;
}
/* wait until the fault happen and copy the event */
static int kbase_job_fault_event_wait(struct kbase_device *kbdev,
- struct list_head *event_list,
struct base_job_fault_event *event)
{
+ struct list_head *event_list = &kbdev->job_fault_event_list;
struct base_job_fault_event *event_in;
+ unsigned long flags;
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
if (list_empty(event_list)) {
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
if (wait_event_interruptible(kbdev->job_fault_wq,
- kbase_is_job_fault_event_pending(event_list)))
+ kbase_is_job_fault_event_pending(kbdev)))
return -ERESTARTSYS;
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
}
event_in = list_entry(event_list->next,
struct base_job_fault_event, head);
-
event->event_code = event_in->event_code;
event->katom = event_in->katom;
+
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
+
return 0;
}
static void kbase_job_fault_event_cleanup(struct kbase_device *kbdev)
{
struct list_head *event_list = &kbdev->job_fault_event_list;
+ unsigned long flags;
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
while (!list_empty(event_list)) {
-
kbase_job_fault_event_dequeue(kbdev, event_list);
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
wake_up(&kbdev->job_fault_resume_wq);
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
}
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
}
static void kbase_job_fault_resume_worker(struct work_struct *data)
* atoms belong to the same context.
*/
wait_event(kctx->kbdev->job_fault_resume_wq,
- kbase_ctx_has_no_event_pending(kctx,
- &kctx->kbdev->job_fault_event_list));
+ kbase_ctx_has_no_event_pending(kctx));
atomic_set(&kctx->job_fault_count, 0);
kbase_jd_done_worker(&katom->work);
struct kbase_jd_atom *katom, u32 completion_code)
{
struct base_job_fault_event *event;
+ unsigned long flags;
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list,
katom, completion_code);
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
wake_up_interruptible(&kbdev->job_fault_wq);
*/
if (*pos == 0) {
event = kmalloc(sizeof(*event), GFP_KERNEL);
+ if (!event)
+ return NULL;
event->reg_offset = 0;
- if (kbase_job_fault_event_wait(kbdev,
- &kbdev->job_fault_event_list, event)) {
+ if (kbase_job_fault_event_wait(kbdev, event)) {
kfree(event);
return NULL;
}
dev_info(kbdev->dev, "debug job fault seq stop stage 1");
} else {
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
if (!list_empty(&kbdev->job_fault_event_list)) {
kbase_job_fault_event_dequeue(kbdev,
&kbdev->job_fault_event_list);
wake_up(&kbdev->job_fault_resume_wq);
}
+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
dev_info(kbdev->dev, "debug job fault seq stop stage 2");
}
init_waitqueue_head(&(kbdev->job_fault_wq));
init_waitqueue_head(&(kbdev->job_fault_resume_wq));
+ spin_lock_init(&kbdev->job_fault_event_lock);
kbdev->job_fault_resume_workq = alloc_workqueue(
"kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1);
/*
*
- * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
page = pfn_to_page(PFN_DOWN(map->alloc->pages[data->offset]));
mapping = vmap(&page, 1, VM_MAP, prot);
+ if (!mapping)
+ goto out;
for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) {
seq_printf(m, "%016llx:", i + ((map->start_pfn +
int ret;
ret = seq_open(file, &ops);
-
if (ret)
return ret;
mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL);
+ if (!mem_data) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
mem_data->kctx = kctx;
INIT_LIST_HEAD(&mem_data->mapping_list);
continue;
mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
+ if (!mapping) {
+ ret = -ENOMEM;
+ kbase_gpu_vm_unlock(kctx);
+ goto out;
+ }
mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
mapping->start_pfn = reg->start_pfn;
((struct seq_file *)file->private_data)->private = mem_data;
return 0;
+
+out:
+ if (mem_data) {
+ while (!list_empty(&mem_data->mapping_list)) {
+ struct debug_mem_mapping *mapping;
+
+ mapping = list_first_entry(&mem_data->mapping_list,
+ struct debug_mem_mapping, node);
+ kbase_mem_phy_alloc_put(mapping->alloc);
+ list_del(&mapping->node);
+ kfree(mapping);
+ }
+ fput(kctx_file);
+ }
+ seq_release(i, file);
+ return ret;
}
static int debug_mem_release(struct inode *inode, struct file *file)
#include "sync.h"
#endif /* CONFIG_SYNC */
+#include "mali_kbase_dma_fence.h"
+
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
#endif /* CONFIG_DEBUG_FS */
struct sync_fence *fence;
struct sync_fence_waiter sync_waiter;
#endif /* CONFIG_SYNC */
+#ifdef CONFIG_MALI_DMA_FENCE
+ struct {
+ /* This points to the dma-buf fence for this atom. If this is
+ * NULL then there is no fence for this atom and the other
+ * fields related to dma_fence may have invalid data.
+ *
+ * The context and seqno fields contain the details for this
+ * fence.
+ *
+ * This fence is signaled when the katom is completed,
+ * regardless of the event_code of the katom (signal also on
+ * failure).
+ */
+ struct fence *fence;
+ /* The dma-buf fence context number for this atom. A unique
+ * context number is allocated to each katom in the context on
+ * context creation.
+ */
+ unsigned int context;
+ /* The dma-buf fence sequence number for this atom. This is
+ * increased every time this katom uses dma-buf fence.
+ */
+ atomic_t seqno;
+ /* This contains a list of all callbacks set up to wait on
+ * other fences. This atom must be held back from JS until all
+ * these callbacks have been called and dep_count have reached
+ * 0. The initial value of dep_count must be equal to the
+ * number of callbacks on this list.
+ *
+ * This list is protected by jctx.lock. Callbacks are added to
+ * this list when the atom is built and the wait are set up.
+ * All the callbacks then stay on the list until all callbacks
+ * have been called and the atom is queued, or cancelled, and
+ * then all callbacks are taken off the list and freed.
+ */
+ struct list_head callbacks;
+ /* Atomic counter of number of outstandind dma-buf fence
+ * dependencies for this atom. When dep_count reaches 0 the
+ * atom may be queued.
+ *
+ * The special value "-1" may only be set after the count
+ * reaches 0, while holding jctx.lock. This indicates that the
+ * atom has been handled, either queued in JS or cancelled.
+ *
+ * If anyone but the dma-fence worker sets this to -1 they must
+ * ensure that any potentially queued worker must have
+ * completed before allowing the atom to be marked as unused.
+ * This can be done by flushing the fence work queue:
+ * kctx->dma_fence.wq.
+ */
+ atomic_t dep_count;
+ } dma_fence;
+#endif /* CONFIG_MALI_DMA_FENCE */
/* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */
enum base_jd_event_code event_code;
u64 reg_start;
size_t reg_size;
void __iomem *reg;
+
struct {
int irq;
int flags;
} irqs[3];
+
struct clk *clock;
#ifdef CONFIG_REGULATOR
struct regulator *regulator;
atomic_t serving_gpu_irq;
atomic_t serving_mmu_irq;
spinlock_t reg_op_lock;
-#endif /* CONFIG_MALI_NO_MALI */
+#endif /* CONFIG_MALI_NO_MALI */
struct kbase_pm_device_data pm;
struct kbasep_js_device_data js_data;
wait_queue_head_t job_fault_resume_wq;
struct workqueue_struct *job_fault_resume_workq;
struct list_head job_fault_event_list;
+ spinlock_t job_fault_event_lock;
struct kbase_context *kctx_fault;
#if !MALI_CUSTOMER_RELEASE
#ifdef CONFIG_KDS
struct list_head waiting_kds_resource;
#endif
+#ifdef CONFIG_MALI_DMA_FENCE
+ struct {
+ struct list_head waiting_resource;
+ struct workqueue_struct *wq;
+ } dma_fence;
+#endif /* CONFIG_MALI_DMA_FENCE */
/** This is effectively part of the Run Pool, because it only has a valid
* setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in
*
* which is mapped.
* @gpu_addr: The GPU virtual address the resource is
* mapped to.
- * @refcount: Refcount to keep track of the number of
- * active mappings.
*
* External resources can be mapped into multiple contexts as well as the same
* context multiple times.
* information to it as it could be removed under our feet leaving external
* resources pinned.
* This metadata structure binds a single external resource to a single
- * context, ensuring that per context refcount is tracked separately so it can
+ * context, ensuring that per context mapping is tracked separately so it can
* be overridden when needed and abuses by the application (freeing the resource
* multiple times) don't effect the refcount of the physical allocation.
*/
struct list_head ext_res_node;
struct kbase_mem_phy_alloc *alloc;
u64 gpu_addr;
- u64 refcount;
};
enum kbase_reg_access_type {
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_DMA_FENCE as
+ * it will be set there.
+ */
+#include "mali_kbase_dma_fence.h"
+
+#include <linux/atomic.h>
+#include <linux/fence.h>
+#include <linux/list.h>
+#include <linux/lockdep.h>
+#include <linux/mutex.h>
+#include <linux/reservation.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/ww_mutex.h>
+
+#include <mali_kbase.h>
+
+
+/* Spin lock protecting all Mali fences as fence->lock. */
+static DEFINE_SPINLOCK(kbase_dma_fence_lock);
+
+
+static void
+kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom)
+{
+ struct kbase_context *kctx = katom->kctx;
+
+ list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource);
+}
+
+void
+kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom)
+{
+ list_del(&katom->queue);
+}
+
+static const char *
+kbase_dma_fence_get_driver_name(struct fence *fence)
+{
+ return kbase_drv_name;
+}
+
+static const char *
+kbase_dma_fence_get_timeline_name(struct fence *fence)
+{
+ return kbase_timeline_name;
+}
+
+static bool
+kbase_dma_fence_enable_signaling(struct fence *fence)
+{
+ /* If in the future we need to add code here remember to
+ * to get a reference to the fence and release it when signaling
+ * as stated in fence.h
+ */
+ return true;
+}
+
+static void
+kbase_dma_fence_fence_value_str(struct fence *fence, char *str, int size)
+{
+ snprintf(str, size, "%u", fence->seqno);
+}
+
+static const struct fence_ops kbase_dma_fence_ops = {
+ .get_driver_name = kbase_dma_fence_get_driver_name,
+ .get_timeline_name = kbase_dma_fence_get_timeline_name,
+ .enable_signaling = kbase_dma_fence_enable_signaling,
+ /* Use the default wait */
+ .wait = fence_default_wait,
+ .fence_value_str = kbase_dma_fence_fence_value_str,
+};
+
+static struct fence *
+kbase_dma_fence_new(unsigned int context, unsigned int seqno)
+{
+ struct fence *fence;
+
+ fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+ if (!fence)
+ return NULL;
+
+ fence_init(fence,
+ &kbase_dma_fence_ops,
+ &kbase_dma_fence_lock,
+ context,
+ seqno);
+
+ return fence;
+}
+
+static int
+kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info,
+ struct ww_acquire_ctx *ctx)
+{
+ struct reservation_object *content_res = NULL;
+ unsigned int content_res_idx = 0;
+ unsigned int r;
+ int err;
+
+ ww_acquire_init(ctx, &reservation_ww_class);
+
+retry:
+ for (r = 0; r < info->dma_fence_resv_count; r++) {
+ if (info->resv_objs[r] == content_res) {
+ content_res = NULL;
+ continue;
+ }
+
+ err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx);
+ if (err)
+ goto error;
+ }
+
+ ww_acquire_done(ctx);
+ return err;
+
+error:
+ content_res_idx = r;
+
+ /* Unlock the locked one ones */
+ for (r--; r >= 0; r--)
+ ww_mutex_unlock(&info->resv_objs[r]->lock);
+
+ if (content_res)
+ ww_mutex_unlock(&content_res->lock);
+
+ /* If we deadlock try with lock_slow and retry */
+ if (err == -EDEADLK) {
+ content_res = info->resv_objs[content_res_idx];
+ ww_mutex_lock_slow(&content_res->lock, ctx);
+ goto retry;
+ }
+
+ /* If we are here the function failed */
+ ww_acquire_fini(ctx);
+ return err;
+}
+
+static void
+kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info,
+ struct ww_acquire_ctx *ctx)
+{
+ unsigned int r;
+
+ for (r = 0; r < info->dma_fence_resv_count; r++)
+ ww_mutex_unlock(&info->resv_objs[r]->lock);
+ ww_acquire_fini(ctx);
+}
+
+/**
+ * kbase_dma_fence_free_callbacks - Free dma-fence callbacks on a katom
+ * @katom: Pointer to katom
+ *
+ * This function will free all fence callbacks on the katom's list of
+ * callbacks. Callbacks that have not yet been called, because their fence
+ * hasn't yet signaled, will first be removed from the fence.
+ *
+ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held.
+ */
+static void
+kbase_dma_fence_free_callbacks(struct kbase_jd_atom *katom)
+{
+ struct kbase_dma_fence_cb *cb, *tmp;
+
+ lockdep_assert_held(&katom->kctx->jctx.lock);
+
+ /* Clean up and free callbacks. */
+ list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) {
+ bool ret;
+
+ /* Cancel callbacks that hasn't been called yet. */
+ ret = fence_remove_callback(cb->fence, &cb->fence_cb);
+ if (ret) {
+ /* Fence had not signaled, clean up after
+ * canceling.
+ */
+ atomic_dec(&katom->dma_fence.dep_count);
+ }
+
+ fence_put(cb->fence);
+ list_del(&cb->node);
+ kfree(cb);
+ }
+}
+
+/**
+ * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom
+ * @katom: Katom to cancel
+ *
+ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held.
+ */
+static void
+kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom)
+{
+ lockdep_assert_held(&katom->kctx->jctx.lock);
+
+ /* Cancel callbacks and clean up. */
+ kbase_dma_fence_free_callbacks(katom);
+
+ KBASE_DEBUG_ASSERT(atomic_read(&katom->dma_fence.dep_count) == 0);
+
+ /* Mark the atom as handled in case all fences signaled just before
+ * canceling the callbacks and the worker was queued.
+ */
+ atomic_set(&katom->dma_fence.dep_count, -1);
+
+ /* Prevent job_done_nolock from being called twice on an atom when
+ * there is a race between job completion and cancellation.
+ */
+
+ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) {
+ /* Wait was cancelled - zap the atom */
+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+ if (jd_done_nolock(katom, NULL))
+ kbase_js_sched_all(katom->kctx->kbdev);
+ }
+}
+
+/**
+ * kbase_dma_fence_work() - Worker thread called when a fence is signaled
+ * @pwork: work_struct containing a pointer to a katom
+ *
+ * This function will clean and mark all dependencies as satisfied
+ */
+static void
+kbase_dma_fence_work(struct work_struct *pwork)
+{
+ struct kbase_jd_atom *katom;
+ struct kbase_jd_context *ctx;
+
+ katom = container_of(pwork, struct kbase_jd_atom, work);
+ ctx = &katom->kctx->jctx;
+
+ mutex_lock(&ctx->lock);
+ if (atomic_read(&katom->dma_fence.dep_count) != 0)
+ goto out;
+
+ atomic_set(&katom->dma_fence.dep_count, -1);
+
+ /* Remove atom from list of dma-fence waiting atoms. */
+ kbase_dma_fence_waiters_remove(katom);
+ /* Cleanup callbacks. */
+ kbase_dma_fence_free_callbacks(katom);
+ /* Queue atom on GPU. */
+ kbase_jd_dep_clear_locked(katom);
+
+out:
+ mutex_unlock(&ctx->lock);
+}
+
+static int
+kbase_dma_fence_add_callback(struct kbase_jd_atom *katom,
+ struct fence *fence,
+ fence_func_t callback)
+{
+ int err = 0;
+ struct kbase_dma_fence_cb *kbase_fence_cb;
+
+ kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL);
+ if (!kbase_fence_cb)
+ return -ENOMEM;
+
+ fence_get(fence);
+
+ kbase_fence_cb->fence = fence;
+ kbase_fence_cb->katom = katom;
+ INIT_LIST_HEAD(&kbase_fence_cb->node);
+
+ err = fence_add_callback(fence, &kbase_fence_cb->fence_cb, callback);
+ if (err == -ENOENT) {
+ /* Fence signaled, clear the error and return */
+ err = 0;
+ kbase_fence_cb->fence = NULL;
+ fence_put(fence);
+ kfree(kbase_fence_cb);
+ } else if (err) {
+ /* Do nothing, just return the error */
+ fence_put(fence);
+ kfree(kbase_fence_cb);
+ } else {
+ atomic_inc(&katom->dma_fence.dep_count);
+ /* Add callback to katom's list of callbacks */
+ list_add(&katom->dma_fence.callbacks, &kbase_fence_cb->node);
+ }
+
+ return err;
+}
+
+static void
+kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb)
+{
+ struct kbase_dma_fence_cb *kcb = container_of(cb,
+ struct kbase_dma_fence_cb,
+ fence_cb);
+ struct kbase_jd_atom *katom = kcb->katom;
+ struct kbase_context *kctx = katom->kctx;
+
+ /* If the atom is zapped dep_count will be forced to a negative number
+ * preventing this callback from ever scheduling work. Which in turn
+ * would reschedule the atom.
+ */
+ if (atomic_dec_and_test(&katom->dma_fence.dep_count)) {
+ bool ret;
+
+ INIT_WORK(&katom->work, kbase_dma_fence_work);
+ ret = queue_work(kctx->dma_fence.wq, &katom->work);
+ /* Warn if work was already queued, that should not happen. */
+ WARN_ON(!ret);
+ }
+}
+
+static int
+kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom,
+ struct reservation_object *resv,
+ bool exclusive)
+{
+ struct fence *excl_fence = NULL;
+ struct fence **shared_fences = NULL;
+ unsigned int shared_count = 0;
+ int err, i;
+
+ err = reservation_object_get_fences_rcu(resv,
+ &excl_fence,
+ &shared_count,
+ &shared_fences);
+ if (err)
+ return err;
+
+ if (excl_fence) {
+ err = kbase_dma_fence_add_callback(katom,
+ excl_fence,
+ kbase_dma_fence_cb);
+ if (err)
+ goto error;
+ }
+
+ if (exclusive) {
+ for (i = 0; i < shared_count; i++) {
+ err = kbase_dma_fence_add_callback(katom,
+ shared_fences[i],
+ kbase_dma_fence_cb);
+ if (err)
+ goto error;
+ }
+ }
+ kfree(shared_fences);
+
+ return err;
+
+error:
+ /* Cancel and clean up all callbacks that was set up before the error.
+ */
+ kbase_dma_fence_free_callbacks(katom);
+ kfree(shared_fences);
+
+ return err;
+}
+
+void kbase_dma_fence_add_reservation(struct reservation_object *resv,
+ struct kbase_dma_fence_resv_info *info,
+ bool exclusive)
+{
+ unsigned int i;
+
+ for (i = 0; i < info->dma_fence_resv_count; i++) {
+ /* Duplicate resource, ignore */
+ if (info->resv_objs[i] == resv)
+ return;
+ }
+
+ info->resv_objs[info->dma_fence_resv_count] = resv;
+ if (exclusive)
+ set_bit(info->dma_fence_resv_count,
+ info->dma_fence_excl_bitmap);
+ (info->dma_fence_resv_count)++;
+}
+
+int kbase_dma_fence_wait(struct kbase_jd_atom *katom,
+ struct kbase_dma_fence_resv_info *info)
+{
+ int err, i;
+ struct fence *fence;
+ struct ww_acquire_ctx ww_ctx;
+
+ lockdep_assert_held(&katom->kctx->jctx.lock);
+
+ atomic_set(&katom->dma_fence.dep_count, 1);
+ fence = kbase_dma_fence_new(katom->dma_fence.context,
+ atomic_inc_return(&katom->dma_fence.seqno));
+ if (!fence) {
+ err = -ENOMEM;
+ dev_err(katom->kctx->kbdev->dev,
+ "Error %d creating fence.\n", err);
+ return err;
+ }
+
+ katom->dma_fence.fence = fence;
+
+ err = kbase_dma_fence_lock_reservations(info, &ww_ctx);
+ if (err) {
+ dev_err(katom->kctx->kbdev->dev,
+ "Error %d locking reservations.\n", err);
+ return err;
+ }
+
+ for (i = 0; i < info->dma_fence_resv_count; i++) {
+ struct reservation_object *obj = info->resv_objs[i];
+
+ if (!test_bit(i, info->dma_fence_excl_bitmap)) {
+ err = reservation_object_reserve_shared(obj);
+ if (err) {
+ dev_err(katom->kctx->kbdev->dev,
+ "Error %d reserving space for shared fence.\n", err);
+ goto end;
+ }
+
+ err = kbase_dma_fence_add_reservation_callback(katom, obj, false);
+ if (err) {
+ dev_err(katom->kctx->kbdev->dev,
+ "Error %d adding reservation to callback.\n", err);
+ goto end;
+ }
+
+ reservation_object_add_shared_fence(obj, katom->dma_fence.fence);
+ } else {
+ err = kbase_dma_fence_add_reservation_callback(katom, obj, true);
+ if (err) {
+ dev_err(katom->kctx->kbdev->dev,
+ "Error %d adding reservation to callback.\n", err);
+ goto end;
+ }
+
+ reservation_object_add_excl_fence(obj, katom->dma_fence.fence);
+ }
+ }
+
+end:
+ kbase_dma_fence_unlock_reservations(info, &ww_ctx);
+
+ if (!err) {
+ /* Test if the callbacks are already triggered */
+ if (atomic_dec_and_test(&katom->dma_fence.dep_count)) {
+ atomic_set(&katom->dma_fence.dep_count, -1);
+ kbase_dma_fence_free_callbacks(katom);
+ } else {
+ /* Add katom to the list of dma-buf fence waiting atoms
+ * only if it is still waiting.
+ */
+ kbase_dma_fence_waiters_add(katom);
+ }
+ }
+
+ return err;
+}
+
+void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx)
+{
+ struct kbase_jd_atom *katom, *katom_tmp;
+
+ list_for_each_entry_safe(katom, katom_tmp,
+ &kctx->dma_fence.waiting_resource, queue) {
+ kbase_dma_fence_waiters_remove(katom);
+ kbase_dma_fence_cancel_atom(katom);
+ }
+}
+
+void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom)
+{
+ /* Cancel callbacks and clean up. */
+ kbase_dma_fence_free_callbacks(katom);
+}
+
+void kbase_dma_fence_signal(struct kbase_jd_atom *katom)
+{
+ if (!katom->dma_fence.fence)
+ return;
+
+ KBASE_DEBUG_ASSERT(atomic_read(&katom->dma_fence.dep_count) == -1);
+
+ /* Signal the atom's fence. */
+ fence_signal(katom->dma_fence.fence);
+ fence_put(katom->dma_fence.fence);
+ katom->dma_fence.fence = NULL;
+
+ kbase_dma_fence_free_callbacks(katom);
+}
+
+void kbase_dma_fence_term(struct kbase_context *kctx)
+{
+ destroy_workqueue(kctx->dma_fence.wq);
+ kctx->dma_fence.wq = NULL;
+}
+
+int kbase_dma_fence_init(struct kbase_context *kctx)
+{
+ INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource);
+
+ kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d",
+ WQ_UNBOUND, 1, kctx->pid);
+ if (!kctx->dma_fence.wq)
+ return -ENOMEM;
+
+ return 0;
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_DMA_FENCE_H_
+#define _KBASE_DMA_FENCE_H_
+
+#ifdef CONFIG_MALI_DMA_FENCE
+
+#include <linux/fence.h>
+#include <linux/list.h>
+#include <linux/reservation.h>
+
+
+/* Forward declaration from mali_kbase_defs.h */
+struct kbase_jd_atom;
+struct kbase_context;
+
+/**
+ * struct kbase_dma_fence_cb - Mali dma-fence callback data struct
+ * @fence_cb: Callback function
+ * @katom: Pointer to katom that is waiting on this callback
+ * @fence: Pointer to the fence object on which this callback is waiting
+ * @node: List head for linking this callback to the katom
+ */
+struct kbase_dma_fence_cb {
+ struct fence_cb fence_cb;
+ struct kbase_jd_atom *katom;
+ struct fence *fence;
+ struct list_head node;
+};
+
+/**
+ * struct kbase_dma_fence_resv_info - Structure with list of reservation objects
+ * @resv_objs: Array of reservation objects to attach the
+ * new fence to.
+ * @dma_fence_resv_count: Number of reservation objects in the array.
+ * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive.
+ *
+ * This is used by some functions to pass around a collection of data about
+ * reservation objects.
+ */
+struct kbase_dma_fence_resv_info {
+ struct reservation_object **resv_objs;
+ unsigned int dma_fence_resv_count;
+ unsigned long *dma_fence_excl_bitmap;
+};
+
+/**
+ * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs
+ * @resv: Reservation object to add to the array.
+ * @info: Pointer to struct with current reservation info
+ * @exclusive: Boolean indicating if exclusive access is needed
+ *
+ * The function adds a new reservation_object to an existing array of
+ * reservation_objects. At the same time keeps track of which objects require
+ * exclusive access in dma_fence_excl_bitmap.
+ */
+void kbase_dma_fence_add_reservation(struct reservation_object *resv,
+ struct kbase_dma_fence_resv_info *info,
+ bool exclusive);
+
+/**
+ * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs
+ * @katom: Katom with the external dependency.
+ * @info: Pointer to struct with current reservation info
+ *
+ * Return: An error code or 0 if succeeds
+ */
+int kbase_dma_fence_wait(struct kbase_jd_atom *katom,
+ struct kbase_dma_fence_resv_info *info);
+
+/**
+ * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx
+ * @kctx: Pointer to kbase context
+ *
+ * This function will cancel and clean up all katoms on @kctx that is waiting
+ * on dma-buf fences.
+ *
+ * Locking: jctx.lock needs to be held when calling this function.
+ */
+void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx);
+
+/**
+ * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom
+ * @katom: Pointer to katom whose callbacks are to be canceled
+ *
+ * This function cancels all dma-buf fence callbacks on @katom, but does not
+ * cancel the katom itself.
+ *
+ * The caller is responsible for ensuring that jd_done_nolock is called on
+ * @katom.
+ *
+ * Locking: jctx.lock must be held when calling this function.
+ */
+void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom);
+
+/**
+ * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait
+ * @katom: Pointer to katom to signal and clean up
+ *
+ * This function will signal the @katom's fence, if it has one, and clean up
+ * the callback data from the katom's wait on earlier fences.
+ *
+ * Locking: jctx.lock must be held while calling this function.
+ */
+void kbase_dma_fence_signal(struct kbase_jd_atom *katom);
+
+/**
+ * kbase_dma_fence_term() - Terminate Mali dma-fence context
+ * @kctx: kbase context to terminate
+ */
+void kbase_dma_fence_term(struct kbase_context *kctx);
+
+/**
+ * kbase_dma_fence_init() - Initialize Mali dma-fence context
+ * @kctx: kbase context to initialize
+ */
+int kbase_dma_fence_init(struct kbase_context *kctx);
+
+/**
+ * kbase_dma_fence_waiters_remove()- Remove katom from dma-fence wait list
+ * @katom: Pointer to katom to remove from list
+ */
+void kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom);
+
+#else /* CONFIG_MALI_DMA_FENCE */
+/* Dummy functions for when dma-buf fence isn't enabled. */
+
+static inline int kbase_dma_fence_init(struct kbase_context *kctx)
+{
+ return 0;
+}
+
+static inline void kbase_dma_fence_term(struct kbase_context *kctx) {}
+#endif /* CONFIG_MALI_DMA_FENCE */
+#endif
/*
*
- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define MALI_BYTES_PER_COUNTER 4
struct kbase_gator_hwcnt_handles {
- struct kbase_device *kbdev;
- struct kbase_context *kctx;
- u64 hwcnt_gpu_va;
- void *hwcnt_cpu_va;
- struct kbase_vmap_struct hwcnt_map;
+ struct kbase_device *kbdev;
+ struct kbase_vinstr_client *vinstr_cli;
+ void *vinstr_buffer;
+ struct work_struct dump_work;
+ int dump_complete;
+ spinlock_t dump_lock;
};
+static void dump_worker(struct work_struct *work);
+
const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters)
{
- uint32_t gpu_id;
const char * const *hardware_counters;
struct kbase_device *kbdev;
+ uint32_t gpu_id;
+ uint32_t product_id;
+ uint32_t count;
if (!total_counters)
return NULL;
return NULL;
gpu_id = kbdev->gpu_props.props.core_props.product_id;
-
- switch (gpu_id) {
- /* If we are using a Mali-T60x device */
- case GPU_ID_PI_T60X:
- hardware_counters = hardware_counters_mali_t60x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t60x);
- break;
- /* If we are using a Mali-T62x device */
- case GPU_ID_PI_T62X:
- hardware_counters = hardware_counters_mali_t62x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t62x);
- break;
- /* If we are using a Mali-T72x device */
- case GPU_ID_PI_T72X:
- hardware_counters = hardware_counters_mali_t72x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t72x);
- break;
- /* If we are using a Mali-T76x device */
- case GPU_ID_PI_T76X:
- hardware_counters = hardware_counters_mali_t76x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t76x);
- break;
- /* If we are using a Mali-T82x device */
- case GPU_ID_PI_T82X:
- hardware_counters = hardware_counters_mali_t82x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t82x);
- break;
- /* If we are using a Mali-T83x device */
- case GPU_ID_PI_T83X:
- hardware_counters = hardware_counters_mali_t83x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t83x);
- break;
- /* If we are using a Mali-T86x device */
- case GPU_ID_PI_T86X:
- hardware_counters = hardware_counters_mali_t86x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t86x);
- break;
- /* If we are using a Mali-T88x device */
- case GPU_ID_PI_TFRX:
- hardware_counters = hardware_counters_mali_t88x;
- *total_counters = ARRAY_SIZE(hardware_counters_mali_t88x);
- break;
- default:
- hardware_counters = NULL;
- *total_counters = 0;
- dev_err(kbdev->dev, "Unrecognized gpu ID: %u\n", gpu_id);
- break;
+ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID;
+ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT;
+
+ if (GPU_ID_IS_NEW_FORMAT(product_id)) {
+ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) {
+ case GPU_ID2_PRODUCT_TMIX:
+ hardware_counters = hardware_counters_mali_tMIx;
+ count = ARRAY_SIZE(hardware_counters_mali_tMIx);
+ break;
+ default:
+ hardware_counters = NULL;
+ count = 0;
+ dev_err(kbdev->dev, "Unrecognized gpu ID: %u\n",
+ gpu_id);
+ break;
+ }
+ } else {
+ switch (gpu_id) {
+ /* If we are using a Mali-T60x device */
+ case GPU_ID_PI_T60X:
+ hardware_counters = hardware_counters_mali_t60x;
+ count = ARRAY_SIZE(hardware_counters_mali_t60x);
+ break;
+ /* If we are using a Mali-T62x device */
+ case GPU_ID_PI_T62X:
+ hardware_counters = hardware_counters_mali_t62x;
+ count = ARRAY_SIZE(hardware_counters_mali_t62x);
+ break;
+ /* If we are using a Mali-T72x device */
+ case GPU_ID_PI_T72X:
+ hardware_counters = hardware_counters_mali_t72x;
+ count = ARRAY_SIZE(hardware_counters_mali_t72x);
+ break;
+ /* If we are using a Mali-T76x device */
+ case GPU_ID_PI_T76X:
+ hardware_counters = hardware_counters_mali_t76x;
+ count = ARRAY_SIZE(hardware_counters_mali_t76x);
+ break;
+ /* If we are using a Mali-T82x device */
+ case GPU_ID_PI_T82X:
+ hardware_counters = hardware_counters_mali_t82x;
+ count = ARRAY_SIZE(hardware_counters_mali_t82x);
+ break;
+ /* If we are using a Mali-T83x device */
+ case GPU_ID_PI_T83X:
+ hardware_counters = hardware_counters_mali_t83x;
+ count = ARRAY_SIZE(hardware_counters_mali_t83x);
+ break;
+ /* If we are using a Mali-T86x device */
+ case GPU_ID_PI_T86X:
+ hardware_counters = hardware_counters_mali_t86x;
+ count = ARRAY_SIZE(hardware_counters_mali_t86x);
+ break;
+ /* If we are using a Mali-T88x device */
+ case GPU_ID_PI_TFRX:
+ hardware_counters = hardware_counters_mali_t88x;
+ count = ARRAY_SIZE(hardware_counters_mali_t88x);
+ break;
+ default:
+ hardware_counters = NULL;
+ count = 0;
+ dev_err(kbdev->dev, "Unrecognized gpu ID: %u\n",
+ gpu_id);
+ break;
+ }
}
/* Release the kbdev reference. */
kbase_release_device(kbdev);
+ *total_counters = count;
+
/* If we return a string array take a reference on the module (or fail). */
if (hardware_counters && !try_module_get(THIS_MODULE))
return NULL;
struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info)
{
struct kbase_gator_hwcnt_handles *hand;
- struct kbase_uk_hwcnt_setup setup;
- int err;
+ struct kbase_uk_hwcnt_reader_setup setup;
uint32_t dump_size = 0, i = 0;
- struct kbase_va_region *reg;
- u64 flags;
- u64 nr_pages;
- u16 va_alignment = 0;
if (!in_out_info)
return NULL;
if (!hand)
return NULL;
+ INIT_WORK(&hand->dump_work, dump_worker);
+ spin_lock_init(&hand->dump_lock);
+
/* Get the first device */
hand->kbdev = kbase_find_device(-1);
if (!hand->kbdev)
goto free_hand;
- /* Create a kbase_context */
- hand->kctx = kbase_create_context(hand->kbdev, true);
- if (!hand->kctx)
+ dump_size = kbase_vinstr_dump_size(hand->kbdev);
+ hand->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL);
+ if (!hand->vinstr_buffer)
goto release_device;
+ in_out_info->kernel_dump_buffer = hand->vinstr_buffer;
in_out_info->nr_cores = hand->kbdev->gpu_props.num_cores;
in_out_info->nr_core_groups = hand->kbdev->gpu_props.num_core_groups;
in_out_info->nr_core_groups, GFP_KERNEL);
if (!in_out_info->hwc_layout)
- goto destroy_context;
+ goto free_vinstr_buffer;
dump_size = in_out_info->nr_core_groups *
MALI_MAX_NUM_BLOCKS_PER_GROUP *
in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc_bits + nr_l2), GFP_KERNEL);
if (!in_out_info->hwc_layout)
- goto destroy_context;
+ goto free_vinstr_buffer;
dump_size = (2 + nr_sc_bits + nr_l2) * MALI_COUNTERS_PER_BLOCK * MALI_BYTES_PER_COUNTER;
}
in_out_info->nr_hwc_blocks = i;
-
in_out_info->size = dump_size;
- flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR;
- nr_pages = PFN_UP(dump_size);
- reg = kbase_mem_alloc(hand->kctx, nr_pages, nr_pages, 0,
- &flags, &hand->hwcnt_gpu_va, &va_alignment);
- if (!reg)
- goto free_layout;
-
- hand->hwcnt_cpu_va = kbase_vmap(hand->kctx, hand->hwcnt_gpu_va,
- dump_size, &hand->hwcnt_map);
-
- if (!hand->hwcnt_cpu_va)
- goto free_buffer;
-
- in_out_info->kernel_dump_buffer = hand->hwcnt_cpu_va;
- memset(in_out_info->kernel_dump_buffer, 0, nr_pages * PAGE_SIZE);
-
- /*setup.dump_buffer = (uintptr_t)in_out_info->kernel_dump_buffer;*/
- setup.dump_buffer = hand->hwcnt_gpu_va;
setup.jm_bm = in_out_info->bitmask[0];
setup.tiler_bm = in_out_info->bitmask[1];
setup.shader_bm = in_out_info->bitmask[2];
setup.mmu_l2_bm = in_out_info->bitmask[3];
-
- err = kbase_instr_hwcnt_enable(hand->kctx, &setup);
- if (err)
- goto free_unmap;
-
- kbase_instr_hwcnt_clear(hand->kctx);
+ hand->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(hand->kbdev->vinstr_ctx,
+ &setup, hand->vinstr_buffer);
+ if (!hand->vinstr_cli) {
+ dev_err(hand->kbdev->dev, "Failed to register gator with vinstr core");
+ goto free_layout;
+ }
return hand;
-free_unmap:
- kbase_vunmap(hand->kctx, &hand->hwcnt_map);
-
-free_buffer:
- kbase_mem_free(hand->kctx, hand->hwcnt_gpu_va);
-
free_layout:
kfree(in_out_info->hwc_layout);
-destroy_context:
- kbase_destroy_context(hand->kctx);
+free_vinstr_buffer:
+ kfree(hand->vinstr_buffer);
release_device:
kbase_release_device(hand->kbdev);
free_hand:
kfree(hand);
-
return NULL;
}
KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init);
kfree(in_out_info->hwc_layout);
if (opaque_handles) {
- kbase_instr_hwcnt_disable(opaque_handles->kctx);
- kbase_vunmap(opaque_handles->kctx, &opaque_handles->hwcnt_map);
- kbase_mem_free(opaque_handles->kctx, opaque_handles->hwcnt_gpu_va);
- kbase_destroy_context(opaque_handles->kctx);
+ cancel_work_sync(&opaque_handles->dump_work);
+ kbase_vinstr_detach_client(opaque_handles->vinstr_cli);
+ kfree(opaque_handles->vinstr_buffer);
kbase_release_device(opaque_handles->kbdev);
kfree(opaque_handles);
}
}
KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term);
+static void dump_worker(struct work_struct *work)
+{
+ struct kbase_gator_hwcnt_handles *hand;
+
+ hand = container_of(work, struct kbase_gator_hwcnt_handles, dump_work);
+ if (!kbase_vinstr_hwc_dump(hand->vinstr_cli,
+ BASE_HWCNT_READER_EVENT_MANUAL)) {
+ spin_lock_bh(&hand->dump_lock);
+ hand->dump_complete = 1;
+ spin_unlock_bh(&hand->dump_lock);
+ } else {
+ schedule_work(&hand->dump_work);
+ }
+}
+
uint32_t kbase_gator_instr_hwcnt_dump_complete(
struct kbase_gator_hwcnt_handles *opaque_handles,
uint32_t * const success)
{
- bool ret_res, success_res;
if (opaque_handles && success) {
- ret_res = kbase_instr_hwcnt_dump_complete(opaque_handles->kctx,
- &success_res);
- *success = (uint32_t)success_res;
- return (uint32_t)(ret_res != 0);
+ *success = opaque_handles->dump_complete;
+ opaque_handles->dump_complete = 0;
+ return *success;
}
return 0;
}
uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles)
{
if (opaque_handles)
- return (kbase_instr_hwcnt_request_dump(
- opaque_handles->kctx) == 0);
-
+ schedule_work(&opaque_handles->dump_work);
return 0;
}
KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq);
/*
*
- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
"T88x_L2_REPLAY_FULL"
};
+#include "mali_kbase_gator_hwcnt_names_tmix.h"
+
+
+
#endif
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * This header was autogenerated, it should not be edited.
+ */
+
+#ifndef _KBASE_GATOR_HWCNT_NAMES_TMIX_H_
+#define _KBASE_GATOR_HWCNT_NAMES_TMIX_H_
+
+static const char * const hardware_counters_mali_tMIx[] = {
+ /* Performance counters for the Job Manager */
+ "",
+ "",
+ "",
+ "",
+ "TMIx_MESSAGES_SENT",
+ "TMIx_MESSAGES_RECEIVED",
+ "TMIx_GPU_ACTIVE",
+ "TMIx_IRQ_ACTIVE",
+ "TMIx_JS0_JOBS",
+ "TMIx_JS0_TASKS",
+ "TMIx_JS0_ACTIVE",
+ "",
+ "TMIx_JS0_WAIT_READ",
+ "TMIx_JS0_WAIT_ISSUE",
+ "TMIx_JS0_WAIT_DEPEND",
+ "TMIx_JS0_WAIT_FINISH",
+ "TMIx_JS1_JOBS",
+ "TMIx_JS1_TASKS",
+ "TMIx_JS1_ACTIVE",
+ "",
+ "TMIx_JS1_WAIT_READ",
+ "TMIx_JS1_WAIT_ISSUE",
+ "TMIx_JS1_WAIT_DEPEND",
+ "TMIx_JS1_WAIT_FINISH",
+ "TMIx_JS2_JOBS",
+ "TMIx_JS2_TASKS",
+ "TMIx_JS2_ACTIVE",
+ "",
+ "TMIx_JS2_WAIT_READ",
+ "TMIx_JS2_WAIT_ISSUE",
+ "TMIx_JS2_WAIT_DEPEND",
+ "TMIx_JS2_WAIT_FINISH",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /* Performance counters for the Tiler */
+ "",
+ "",
+ "",
+ "",
+ "TMIx_TILER_ACTIVE",
+ "TMIx_JOBS_PROCESSED",
+ "TMIx_TRIANGLES",
+ "TMIx_LINES",
+ "TMIx_POINTS",
+ "TMIx_FRONT_FACING",
+ "TMIx_BACK_FACING",
+ "TMIx_PRIM_VISIBLE",
+ "TMIx_PRIM_CULLED",
+ "TMIx_PRIM_CLIPPED",
+ "TMIx_PRIM_SAT_CULLED",
+ "",
+ "",
+ "TMIx_BUS_READ",
+ "",
+ "TMIx_BUS_WRITE",
+ "TMIx_LOADING_DESC",
+ "TMIx_IDVS_POS_SHAD_REQ",
+ "TMIx_IDVS_POS_SHAD_WAIT",
+ "TMIx_IDVS_POS_SHAD_STALL",
+ "TMIx_IDVS_POS_FIFO_FULL",
+ "TMIx_PREFETCH_STALL",
+ "TMIx_VCACHE_HIT",
+ "TMIx_VCACHE_MISS",
+ "TMIx_VCACHE_LINE_WAIT",
+ "TMIx_VFETCH_POS_READ_WAIT",
+ "TMIx_VFETCH_VERTEX_WAIT",
+ "TMIx_VFETCH_STALL",
+ "TMIx_PRIMASSY_STALL",
+ "TMIx_BBOX_GEN_STALL",
+ "TMIx_IDVS_VBU_HIT",
+ "TMIx_IDVS_VBU_MISS",
+ "TMIx_IDVS_VBU_LINE_DEALLOCATE",
+ "TMIx_IDVS_VAR_SHAD_REQ",
+ "TMIx_IDVS_VAR_SHAD_STALL",
+ "TMIx_BINNER_STALL",
+ "TMIx_ITER_STALL",
+ "TMIx_COMPRESS_MISS",
+ "TMIx_COMPRESS_STALL",
+ "TMIx_PCACHE_HIT",
+ "TMIx_PCACHE_MISS",
+ "TMIx_PCACHE_MISS_STALL",
+ "TMIx_PCACHE_EVICT_STALL",
+ "TMIx_PMGR_PTR_WR_STALL",
+ "TMIx_PMGR_PTR_RD_STALL",
+ "TMIx_PMGR_CMD_WR_STALL",
+ "TMIx_WRBUF_ACTIVE",
+ "TMIx_WRBUF_HIT",
+ "TMIx_WRBUF_MISS",
+ "TMIx_WRBUF_NO_FREE_LINE_STALL",
+ "TMIx_WRBUF_NO_AXI_ID_STALL",
+ "TMIx_WRBUF_AXI_STALL",
+ "",
+ "",
+ "",
+ "TMIx_UTLB_TRANS",
+ "TMIx_UTLB_TRANS_HIT",
+ "TMIx_UTLB_TRANS_STALL",
+ "TMIx_UTLB_TRANS_MISS_DELAY",
+ "TMIx_UTLB_MMU_REQ",
+
+ /* Performance counters for the Shader Core */
+ "",
+ "",
+ "",
+ "",
+ "TMIx_FRAG_ACTIVE",
+ "TMIx_FRAG_PRIMITIVES",
+ "TMIx_FRAG_PRIM_RAST",
+ "TMIx_FRAG_FPK_ACTIVE",
+ "TMIx_FRAG_STARVING",
+ "TMIx_FRAG_WARPS",
+ "TMIx_FRAG_PARTIAL_WARPS",
+ "TMIx_FRAG_QUADS_RAST",
+ "TMIx_FRAG_QUADS_EZS_TEST",
+ "TMIx_FRAG_QUADS_EZS_UPDATE",
+ "TMIx_FRAG_QUADS_EZS_KILL",
+ "TMIx_FRAG_LZS_TEST",
+ "TMIx_FRAG_LZS_KILL",
+ "",
+ "TMIx_FRAG_PTILES",
+ "TMIx_FRAG_TRANS_ELIM",
+ "TMIx_QUAD_FPK_KILLER",
+ "",
+ "TMIx_COMPUTE_ACTIVE",
+ "TMIx_COMPUTE_TASKS",
+ "TMIx_COMPUTE_WARPS",
+ "TMIx_COMPUTE_STARVING",
+ "TMIx_EXEC_CORE_ACTIVE",
+ "TMIx_EXEC_ACTIVE",
+ "TMIx_EXEC_INSTR_COUNT",
+ "TMIx_EXEC_INSTR_DIVERGED",
+ "TMIx_EXEC_INSTR_STARVING",
+ "TMIx_ARITH_INSTR_SINGLE_FMA",
+ "TMIx_ARITH_INSTR_DOUBLE",
+ "TMIx_ARITH_INSTR_MSG",
+ "TMIx_ARITH_INSTR_MSG_ONLY",
+ "TMIx_TEX_INSTR",
+ "TMIx_TEX_INSTR_MIPMAP",
+ "TMIx_TEX_INSTR_COMPRESSED",
+ "TMIx_TEX_INSTR_3D",
+ "TMIx_TEX_INSTR_TRILINEAR",
+ "TMIx_TEX_COORD_ISSUE",
+ "TMIx_TEX_COORD_STALL",
+ "TMIx_TEX_STARVE_CACHE",
+ "TMIx_TEX_STARVE_FILTER",
+ "TMIx_LS_MEM_READ_FULL",
+ "TMIx_LS_MEM_READ_SHORT",
+ "TMIx_LS_MEM_WRITE_FULL",
+ "TMIx_LS_MEM_WRITE_SHORT",
+ "TMIx_LS_MEM_ATOMIC",
+ "TMIx_VARY_INSTR",
+ "",
+ "",
+ "TMIx_ATTR_INSTR",
+ "TMIx_ARITH_INSTR_FP_MUL",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /* Performance counters for the Memory System */
+ "",
+ "",
+ "",
+ "",
+ "TMIx_MMU_REQUESTS",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "TMIx_L2_RD_MSG_IN",
+ "TMIx_L2_RD_MSG_IN_STALL",
+ "TMIx_L2_WR_MSG_IN",
+ "TMIx_L2_WR_MSG_IN_STALL",
+ "TMIx_L2_SNP_MSG_IN",
+ "TMIx_L2_SNP_MSG_IN_STALL",
+ "TMIx_L2_RD_MSG_OUT",
+ "TMIx_L2_RD_MSG_OUT_STALL",
+ "TMIx_L2_WR_MSG_OUT",
+ "TMIx_L2_ANY_LOOKUP",
+ "TMIx_L2_READ_LOOKUP",
+ "TMIx_L2_WRITE_LOOKUP",
+ "TMIx_L2_EXT_SNOOP_LOOKUP",
+ "TMIx_L2_EXT_READ",
+ "TMIx_L2_EXT_READ_NOSNP",
+ "TMIx_L2_EXT_READ_UNIQUE",
+ "TMIx_L2_EXT_READ_BEATS",
+ "TMIx_L2_EXT_AR_STALL",
+ "TMIx_L2_EXT_AR_CNT_Q1",
+ "TMIx_L2_EXT_AR_CNT_Q2",
+ "TMIx_L2_EXT_AR_CNT_Q3",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "TMIx_L2_EXT_WRITE",
+ "TMIx_L2_EXT_WRITE_NOSNP_FULL",
+ "TMIx_L2_EXT_WRITE_NOSNP_PTL",
+ "TMIx_L2_EXT_WRITE_SNP_FULL",
+ "TMIx_L2_EXT_WRITE_SNP_PTL",
+ "TMIx_L2_EXT_WRITE_BEATS",
+ "TMIx_L2_EXT_W_STALL",
+ "TMIx_L2_EXT_AW_CNT_Q1",
+ "TMIx_L2_EXT_AW_CNT_Q2",
+ "TMIx_L2_EXT_AW_CNT_Q3",
+ "TMIx_L2_EXT_SNOOP",
+ "TMIx_L2_EXT_SNOOP_STALL",
+ "TMIx_L2_EXT_SNOOP_RESP_CLEAN",
+ "TMIx_L2_EXT_SNOOP_RESP_DATA",
+ "TMIx_L2_EXT_SNOOP_INTERNAL",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+};
+
+#endif /* _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ */
/*
*
- * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_tlstream.h>
+#include "mali_kbase_dma_fence.h"
+
#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
return kbasep_js_add_job(kctx, katom);
}
+#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE)
+void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom)
+{
+ struct kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(katom);
+ kbdev = katom->kctx->kbdev;
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ /* Check whether the atom's other dependencies were already met. If
+ * katom is a GPU atom then the job scheduler may be able to represent
+ * the dependencies, hence we may attempt to submit it before they are
+ * met. Other atoms must have had both dependencies resolved.
+ */
+ if (IS_GPU_ATOM(katom) ||
+ (!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
+ !kbase_jd_katom_dep_atom(&katom->dep[1]))) {
+ /* katom dep complete, attempt to run it */
+ bool resched = false;
+
+ resched = jd_run_atom(katom);
+
+ if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
+ /* The atom has already finished */
+ resched |= jd_done_nolock(katom, NULL);
+ }
+
+ if (resched)
+ kbase_js_sched_all(kbdev);
+ }
+}
+#endif
+
#ifdef CONFIG_KDS
/* Add the katom to the kds waiting list.
{
struct kbase_jd_atom *katom;
struct kbase_jd_context *ctx;
- struct kbase_device *kbdev;
katom = (struct kbase_jd_atom *)callback_parameter;
KBASE_DEBUG_ASSERT(katom);
+
ctx = &katom->kctx->jctx;
- kbdev = katom->kctx->kbdev;
- KBASE_DEBUG_ASSERT(kbdev);
+ /* If KDS resource has already been satisfied (e.g. due to zapping)
+ * do nothing.
+ */
mutex_lock(&ctx->lock);
-
- /* KDS resource has already been satisfied (e.g. due to zapping) */
- if (katom->kds_dep_satisfied)
- goto out;
-
- /* This atom's KDS dependency has now been met */
- katom->kds_dep_satisfied = true;
-
- /* Check whether the atom's other dependencies were already met. If
- * katom is a GPU atom then the job scheduler may be able to represent
- * the dependencies, hence we may attempt to submit it before they are
- * met. Other atoms must have had both dependencies resolved */
- if (IS_GPU_ATOM(katom) ||
- (!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
- !kbase_jd_katom_dep_atom(&katom->dep[1]))) {
- /* katom dep complete, attempt to run it */
- bool resched = false;
-
- resched = jd_run_atom(katom);
-
- if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
- /* The atom has already finished */
- resched |= jd_done_nolock(katom, NULL);
- }
-
- if (resched)
- kbase_js_sched_all(kbdev);
+ if (!katom->kds_dep_satisfied) {
+ katom->kds_dep_satisfied = true;
+ kbase_jd_dep_clear_locked(katom);
}
- out:
mutex_unlock(&ctx->lock);
}
kds_resource_set_release_sync(&katom->kds_rset);
}
#endif /* CONFIG_KDS */
+
+#ifdef CONFIG_MALI_DMA_FENCE
+ /* Flush dma-fence workqueue to ensure that any callbacks that may have
+ * been queued are done before continuing.
+ * Any successfully completed atom would have had all it's callbacks
+ * completed before the atom was run, so only flush for failed atoms.
+ */
+ if (katom->event_code != BASE_JD_EVENT_DONE)
+ flush_workqueue(katom->kctx->dma_fence.wq);
+#endif /* CONFIG_MALI_DMA_FENCE */
}
static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom)
katom->kds_dep_satisfied = true;
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ kbase_dma_fence_signal(katom);
+#endif /* CONFIG_MALI_DMA_FENCE */
+
kbase_gpu_vm_lock(katom->kctx);
/* only roll back if extres is non-NULL */
if (katom->extres) {
struct kds_resource **kds_resources = NULL;
unsigned long *kds_access_bitmap = NULL;
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ struct kbase_dma_fence_resv_info info = {
+ .dma_fence_resv_count = 0,
+ };
+#endif
struct base_external_resource *input_extres;
KBASE_DEBUG_ASSERT(katom);
KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL);
- if (NULL == kds_resources) {
+ if (!kds_resources) {
err_ret_val = -ENOMEM;
goto early_err_out;
}
KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
- kds_access_bitmap = kzalloc(sizeof(unsigned long) * ((katom->nr_extres + BITS_PER_LONG - 1) / BITS_PER_LONG), GFP_KERNEL);
-
- if (NULL == kds_access_bitmap) {
+ kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres),
+ sizeof(unsigned long),
+ GFP_KERNEL);
+ if (!kds_access_bitmap) {
err_ret_val = -ENOMEM;
goto early_err_out;
}
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ info.resv_objs = kmalloc_array(katom->nr_extres,
+ sizeof(struct reservation_object *),
+ GFP_KERNEL);
+ if (!info.resv_objs) {
+ err_ret_val = -ENOMEM;
+ goto early_err_out;
+ }
+
+ info.dma_fence_excl_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres),
+ sizeof(unsigned long),
+ GFP_KERNEL);
+ if (!info.dma_fence_excl_bitmap) {
+ err_ret_val = -ENOMEM;
+ goto early_err_out;
+ }
+#endif /* CONFIG_MALI_DMA_FENCE */
+
/* Take the processes mmap lock */
down_read(¤t->mm->mmap_sem);
goto failed_loop;
}
+#ifdef CONFIG_MALI_DMA_FENCE
+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) {
+ struct reservation_object *resv;
+
+ resv = reg->gpu_alloc->imported.umm.dma_buf->resv;
+ if (resv)
+ kbase_dma_fence_add_reservation(resv, &info,
+ exclusive);
+ }
+#endif /* CONFIG_MALI_DMA_FENCE */
+
/* finish with updating out array with the data we found */
/* NOTE: It is important that this is the last thing we do (or
* at least not before the first write) as we overwrite elements
kfree(kds_access_bitmap);
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ if (info.dma_fence_resv_count) {
+ int ret;
+
+ ret = kbase_dma_fence_wait(katom, &info);
+ if (ret < 0)
+ goto failed_dma_fence_setup;
+ }
+
+ kfree(info.resv_objs);
+ kfree(info.dma_fence_excl_bitmap);
+#endif /* CONFIG_MALI_DMA_FENCE */
+
/* all done OK */
return 0;
/* error handling section */
+#ifdef CONFIG_MALI_DMA_FENCE
+failed_dma_fence_setup:
#ifdef CONFIG_KDS
- failed_kds_setup:
+ /* If we are here, dma_fence setup failed but KDS didn't.
+ * Revert KDS setup if any.
+ */
+ if (kds_res_count) {
+ mutex_unlock(&katom->kctx->jctx.lock);
+ kds_resource_set_release_sync(&katom->kds_rset);
+ mutex_lock(&katom->kctx->jctx.lock);
+
+ kbase_jd_kds_waiters_remove(katom);
+ katom->kds_dep_satisfied = true;
+ }
+#endif /* CONFIG_KDS */
+#endif /* CONFIG_MALI_DMA_FENCE */
+#ifdef CONFIG_KDS
+failed_kds_setup:
+#endif
+#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE)
/* Lock the processes mmap lock */
down_read(¤t->mm->mmap_sem);
/* lock before we unmap */
kbase_gpu_vm_lock(katom->kctx);
-#endif /* CONFIG_KDS */
+#endif
failed_loop:
/* undo the loop work */
kfree(kds_resources);
kfree(kds_access_bitmap);
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ kfree(info.resv_objs);
+ kfree(info.dma_fence_excl_bitmap);
+#endif
return err_ret_val;
}
}
#endif
+#ifdef CONFIG_MALI_DMA_FENCE
+ kbase_dma_fence_cancel_callbacks(dep_atom);
+#endif
+
dep_atom->event_code = katom->event_code;
KBASE_DEBUG_ASSERT(dep_atom->status !=
KBASE_JD_ATOM_STATE_UNUSED);
}
}
if (!kbase_jd_katom_dep_atom(&dep_atom->dep[other_d])) {
+ bool dep_satisfied = true;
+#ifdef CONFIG_MALI_DMA_FENCE
+ int dep_count;
+
+ dep_count = atomic_read(&dep_atom->dma_fence.dep_count);
+ if (likely(dep_count == -1)) {
+ dep_satisfied = true;
+ } else if (dep_count == 0) {
+ /*
+ * All fences for this atom has signaled, but
+ * the worker that will queue the atom has not
+ * yet run.
+ *
+ * Mark the atom as handled by setting
+ * dep_count to -1 so that the worker doesn't
+ * queue the atom again.
+ */
+ atomic_set(&dep_atom->dma_fence.dep_count, -1);
+ /*
+ * Remove the atom from the list of dma-fence
+ * waiting atoms.
+ */
+ kbase_dma_fence_waiters_remove(dep_atom);
+ dep_satisfied = true;
+ } else {
+ dep_satisfied = false;
+ }
+#endif /* CONFIG_MALI_DMA_FENCE */
+
#ifdef CONFIG_KDS
- if (dep_atom->kds_dep_satisfied)
+ dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied;
#endif
+
+ if (dep_satisfied)
list_add_tail(&dep_atom->dep_item[0], out_list);
}
}
katom->kds_dep_satisfied = true;
katom->kds_rset = NULL;
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ atomic_set(&katom->dma_fence.dep_count, -1);
+#endif
/* Don't do anything if there is a mess up with dependencies.
This is done in a separate cycle to check both the dependencies at ones, otherwise
kbase_tlstream_tl_ret_atom_ctx(katom, kctx);
for (i = 0; i < 2; i++)
if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type(
- &katom->dep[i]))
+ &katom->dep[i])) {
kbase_tlstream_tl_dep_atom_atom(
(void *)kbase_jd_katom_dep_atom(
&katom->dep[i]),
(void *)katom);
+ } else if (BASE_JD_DEP_TYPE_INVALID !=
+ user_atom->pre_dep[i].dependency_type) {
+ /* Resolved dependency. */
+ int dep_atom_number =
+ user_atom->pre_dep[i].atom_id;
+ struct kbase_jd_atom *dep_atom =
+ &jctx->atoms[dep_atom_number];
+
+ kbase_tlstream_tl_rdep_atom_atom(
+ (void *)dep_atom,
+ (void *)katom);
+ }
/* Reject atoms with job chain = NULL, as these cause issues with soft-stop */
if (!katom->jc && (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
goto out;
}
+ /* Reject atoms with invalid core requirements */
+ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) &&
+ (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) {
+ dev_warn(kctx->kbdev->dev,
+ "Rejecting atom with invalid core requirements");
+ katom->event_code = BASE_JD_EVENT_JOB_INVALID;
+ katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE;
+ ret = jd_done_nolock(katom, NULL);
+ goto out;
+ }
+
/* For invalid priority, be most lenient and choose the default */
sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
}
#endif /* CONFIG_KDS */
+#ifdef CONFIG_MALI_DMA_FENCE
+ if (atomic_read(&katom->dma_fence.dep_count) != -1) {
+ ret = false;
+ goto out;
+ }
+#endif /* CONFIG_MALI_DMA_FENCE */
+
if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE)
== BASE_JD_REQ_SOFT_REPLAY) {
if (kbase_replay_process(katom))
}
#endif
+#ifdef CONFIG_MALI_DMA_FENCE
+ kbase_dma_fence_cancel_all_atoms(kctx);
+#endif
+
mutex_unlock(&kctx->jctx.lock);
+#ifdef CONFIG_MALI_DMA_FENCE
+ /* Flush dma-fence workqueue to ensure that any callbacks that may have
+ * been queued are done before continuing.
+ */
+ flush_workqueue(kctx->dma_fence.wq);
+#endif
+
kbase_jm_wait_for_zero_jobs(kctx);
}
KBASE_DEBUG_ASSERT(kctx);
- kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", 0, 1);
+ kctx->jctx.job_done_wq = alloc_workqueue("mali_jd",
+ WQ_HIGHPRI | WQ_UNBOUND, 1);
if (NULL == kctx->jctx.job_done_wq) {
mali_err = -ENOMEM;
goto out1;
/* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */
kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;
+
+#ifdef CONFIG_MALI_DMA_FENCE
+ kctx->jctx.atoms[i].dma_fence.context = fence_context_alloc(1);
+ atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0);
+ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks);
+#endif
}
mutex_init(&kctx->jctx.lock);
WARN_ON(!kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = false;
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
+ timer_sync = true;
}
if (kctx->as_nr != KBASEP_AS_NR_INVALID &&
WARN_ON(!kctx->ctx_runnable_ref);
kctx->ctx_runnable_ref = false;
atomic_dec(&kbdev->js_data.nr_contexts_runnable);
+ timer_sync = true;
}
}
WARN_ON(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_LL);
* list should it be there.
*/
kbase_sticky_resource_release(reg->kctx, NULL,
- reg->start_pfn << PAGE_SHIFT, true);
+ reg->start_pfn << PAGE_SHIFT);
kbase_mem_phy_alloc_put(reg->cpu_alloc);
kbase_mem_phy_alloc_put(reg->gpu_alloc);
size_t nr_pages_requested)
{
int new_page_count __maybe_unused;
+ size_t old_page_count = alloc->nents;
- KBASE_DEBUG_ASSERT(alloc);
KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE);
KBASE_DEBUG_ASSERT(alloc->imported.kctx);
kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested);
if (kbase_mem_pool_alloc_pages(&alloc->imported.kctx->mem_pool,
- nr_pages_requested, alloc->pages + alloc->nents) != 0)
+ nr_pages_requested, alloc->pages + old_page_count) != 0)
goto no_alloc;
+ /*
+ * Request a zone cache update, this scans only the new pages an
+ * appends their information to the zone cache. if the update
+ * fails then clear the cache so we fall-back to doing things
+ * page by page.
+ */
+ if (kbase_zone_cache_update(alloc, old_page_count) != 0)
+ kbase_zone_cache_clear(alloc);
+
kbase_tlstream_aux_pagesalloc(
(u32)alloc->imported.kctx->id,
(u64)new_page_count);
syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
+ /*
+ * Clear the zone cache, we don't expect JIT allocations to be
+ * shrunk in parts so there is no point trying to optimize for that
+ * by scanning for the changes caused by freeing this memory and
+ * updating the existing cache entries.
+ */
+ kbase_zone_cache_clear(alloc);
+
kbase_mem_pool_free_pages(&kctx->mem_pool,
nr_pages_to_free,
start_free,
break;
#endif
case KBASE_MEM_TYPE_IMPORTED_USER_BUF:
+ if (alloc->imported.user_buf.mm)
+ mmdrop(alloc->imported.user_buf.mm);
kfree(alloc->imported.user_buf.pages);
break;
case KBASE_MEM_TYPE_TB:{
if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
return false;
+ /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */
+ if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED)
+ return false;
+
return true;
}
} else {
/* No suitable JIT allocation was found so create a new one */
u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD |
- BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF;
+ BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF |
+ BASE_MEM_COHERENT_LOCAL;
u64 gpu_addr;
u16 alignment;
long i;
int err = -ENOMEM;
unsigned long address;
- struct task_struct *owner;
+ struct mm_struct *mm;
struct device *dev;
unsigned long offset;
unsigned long local_size;
alloc = reg->gpu_alloc;
pa = kbase_get_gpu_phy_pages(reg);
address = alloc->imported.user_buf.address;
- owner = alloc->imported.user_buf.owner;
+ mm = alloc->imported.user_buf.mm;
KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF);
pages = alloc->imported.user_buf.pages;
- pinned_pages = get_user_pages(owner, owner->mm,
+ pinned_pages = get_user_pages(NULL, mm,
address,
alloc->imported.user_buf.nr_pages,
reg->flags & KBASE_REG_GPU_WR,
/* decide what needs to happen for this resource */
switch (reg->gpu_alloc->type) {
case BASE_MEM_IMPORT_TYPE_USER_BUFFER: {
- if (reg->gpu_alloc->imported.user_buf.owner->mm != locked_mm)
+ if (reg->gpu_alloc->imported.user_buf.mm != locked_mm)
goto exit;
reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++;
lockdep_assert_held(&kctx->reg_lock);
/*
- * Walk the per context externel resource metadata list for the
+ * Walk the per context external resource metadata list for the
* metadata which matches the region which is being acquired.
*/
list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) {
goto fail_map;
meta->gpu_addr = reg->start_pfn << PAGE_SHIFT;
- meta->refcount = 1;
list_add(&meta->ext_res_node, &kctx->ext_res_meta_head);
- } else {
- if (meta->refcount == UINT_MAX)
- goto failed;
-
- meta->refcount++;
}
return meta;
}
bool kbase_sticky_resource_release(struct kbase_context *kctx,
- struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr, bool force)
+ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr)
{
struct kbase_ctx_ext_res_meta *walker;
+ struct kbase_va_region *reg;
lockdep_assert_held(&kctx->reg_lock);
/* Search of the metadata if one isn't provided. */
if (!meta) {
/*
- * Walk the per context externel resource metadata list for the
+ * Walk the per context external resource metadata list for the
* metadata which matches the region which is being released.
*/
list_for_each_entry(walker, &kctx->ext_res_meta_head,
if (!meta)
return false;
- meta->refcount--;
- if ((meta->refcount == 0) || force) {
- /*
- * Last reference to the metadata, drop the physical memory
- * reference and free the metadata.
- */
- struct kbase_va_region *reg;
-
- reg = kbase_region_tracker_find_region_enclosing_address(
- kctx,
- meta->gpu_addr);
+ /* Drop the physical memory reference and free the metadata. */
+ reg = kbase_region_tracker_find_region_enclosing_address(
+ kctx,
+ meta->gpu_addr);
- kbase_unmap_external_resource(kctx, reg, meta->alloc);
- list_del(&meta->ext_res_node);
- kfree(meta);
- }
+ kbase_unmap_external_resource(kctx, reg, meta->alloc);
+ list_del(&meta->ext_res_node);
+ kfree(meta);
return true;
}
walker = list_first_entry(&kctx->ext_res_meta_head,
struct kbase_ctx_ext_res_meta, ext_res_node);
- kbase_sticky_resource_release(kctx, walker, 0, true);
+ kbase_sticky_resource_release(kctx, walker, 0);
}
}
unsigned long properties;
+ struct list_head zone_cache;
+
/* member in union valid based on @a type */
union {
#ifdef CONFIG_UMP
unsigned long nr_pages;
struct page **pages;
unsigned int current_mapping_usage_count;
- struct task_struct *owner;
+ struct mm_struct *mm;
dma_addr_t *dma_addrs;
} user_buf;
} imported;
alloc->pages = (void *)(alloc + 1);
INIT_LIST_HEAD(&alloc->mappings);
alloc->type = type;
+ INIT_LIST_HEAD(&alloc->zone_cache);
if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF)
alloc->imported.user_buf.dma_addrs =
* @kctx: kbase context.
* @meta: Binding metadata.
* @gpu_addr: GPU address of the external resource.
- * @force: If the release is being forced.
*
* If meta is NULL then gpu_addr will be used to scan the metadata list and
* find the matching metadata (if any), otherwise the provided meta will be
* used and gpu_addr will be ignored.
*
- * If force is true then the refcount in the metadata is ignored and the
- * resource will be forced freed.
- *
* Return: True if the release found the metadata and the reference was dropped.
*/
bool kbase_sticky_resource_release(struct kbase_context *kctx,
- struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr, bool force);
+ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr);
/**
* kbase_sticky_resource_term - Terminate sticky resource management.
* @kctx: kbase context
*/
void kbase_sticky_resource_term(struct kbase_context *kctx);
+
+/**
+ * kbase_zone_cache_update - Update the memory zone cache after new pages have
+ * been added.
+ * @alloc: The physical memory allocation to build the cache for.
+ * @start_offset: Offset to where the new pages start.
+ *
+ * Updates an existing memory zone cache, updating the counters for the
+ * various zones.
+ * If the memory allocation doesn't already have a zone cache assume that
+ * one isn't created and thus don't do anything.
+ *
+ * Return: Zero cache was updated, negative error code on error.
+ */
+int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc,
+ size_t start_offset);
+
+/**
+ * kbase_zone_cache_build - Build the memory zone cache.
+ * @alloc: The physical memory allocation to build the cache for.
+ *
+ * Create a new zone cache for the provided physical memory allocation if
+ * one doesn't already exist, if one does exist then just return.
+ *
+ * Return: Zero if the zone cache was created, negative error code on error.
+ */
+int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc);
+
+/**
+ * kbase_zone_cache_clear - Clear the memory zone cache.
+ * @alloc: The physical memory allocation to clear the cache on.
+ */
+void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc);
+
#endif /* _KBASE_MEM_H_ */
#include <mali_kbase_mem_linux.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_hwaccess_time.h>
-
-#if defined(CONFIG_MALI_MIPE_ENABLED)
#include <mali_kbase_tlstream.h>
-#endif
static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
static const struct vm_operations_struct kbase_vm_ops;
unregister_shrinker(&kctx->reclaim);
}
+struct kbase_mem_zone_cache_entry {
+ /* List head used to link the cache entry to the memory allocation. */
+ struct list_head zone_node;
+ /* The zone the cacheline is for. */
+ struct zone *zone;
+ /* The number of pages in the allocation which belong to this zone. */
+ u64 count;
+};
+
+static bool kbase_zone_cache_builder(struct kbase_mem_phy_alloc *alloc,
+ size_t start_offset)
+{
+ struct kbase_mem_zone_cache_entry *cache = NULL;
+ size_t i;
+ int ret = 0;
+
+ for (i = start_offset; i < alloc->nents; i++) {
+ struct page *p = phys_to_page(alloc->pages[i]);
+ struct zone *zone = page_zone(p);
+ bool create = true;
+
+ if (cache && (cache->zone == zone)) {
+ /*
+ * Fast path check as most of the time adjacent
+ * pages come from the same zone.
+ */
+ create = false;
+ } else {
+ /*
+ * Slow path check, walk all the cache entries to see
+ * if we already know about this zone.
+ */
+ list_for_each_entry(cache, &alloc->zone_cache, zone_node) {
+ if (cache->zone == zone) {
+ create = false;
+ break;
+ }
+ }
+ }
+
+ /* This zone wasn't found in the cache, create an entry for it */
+ if (create) {
+ cache = kmalloc(sizeof(*cache), GFP_KERNEL);
+ if (!cache) {
+ ret = -ENOMEM;
+ goto bail;
+ }
+ cache->zone = zone;
+ cache->count = 0;
+ list_add(&cache->zone_node, &alloc->zone_cache);
+ }
+
+ cache->count++;
+ }
+ return 0;
+
+bail:
+ return ret;
+}
+
+int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc,
+ size_t start_offset)
+{
+ /*
+ * Bail if the zone cache is empty, only update the cache if it
+ * existed in the first place.
+ */
+ if (list_empty(&alloc->zone_cache))
+ return 0;
+
+ return kbase_zone_cache_builder(alloc, start_offset);
+}
+
+int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc)
+{
+ /* Bail if the zone cache already exists */
+ if (!list_empty(&alloc->zone_cache))
+ return 0;
+
+ return kbase_zone_cache_builder(alloc, 0);
+}
+
+void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc)
+{
+ struct kbase_mem_zone_cache_entry *walker;
+
+ while(!list_empty(&alloc->zone_cache)){
+ walker = list_first_entry(&alloc->zone_cache,
+ struct kbase_mem_zone_cache_entry, zone_node);
+ list_del(&walker->zone_node);
+ kfree(walker);
+ }
+}
+
/**
* kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable.
* @alloc: The physical allocation
static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc)
{
struct kbase_context *kctx = alloc->imported.kctx;
+ struct kbase_mem_zone_cache_entry *zone_cache;
int __maybe_unused new_page_count;
- int i;
+ int err;
- for (i = 0; i < alloc->nents; i++) {
- struct page *p = phys_to_page(alloc->pages[i]);
+ /* Attempt to build a zone cache of tracking */
+ err = kbase_zone_cache_build(alloc);
+ if (err == 0) {
+ /* Bulk update all the zones */
+ list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
+ zone_page_state_add(zone_cache->count,
+ zone_cache->zone, NR_SLAB_RECLAIMABLE);
+ }
+ } else {
+ /* Fall-back to page by page updates */
+ int i;
+
+ for (i = 0; i < alloc->nents; i++) {
+ struct page *p = phys_to_page(alloc->pages[i]);
+ struct zone *zone = page_zone(p);
- zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE);
+ zone_page_state_add(1, zone, NR_SLAB_RECLAIMABLE);
+ }
}
kbase_process_page_usage_dec(kctx, alloc->nents);
&kctx->used_pages);
kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages);
-#if defined(CONFIG_MALI_MIPE_ENABLED)
kbase_tlstream_aux_pagesalloc(
(u32)kctx->id,
(u64)new_page_count);
-#endif
}
/**
void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc)
{
struct kbase_context *kctx = alloc->imported.kctx;
+ struct kbase_mem_zone_cache_entry *zone_cache;
int __maybe_unused new_page_count;
- int i;
+ int err;
new_page_count = kbase_atomic_add_pages(alloc->nents,
&kctx->used_pages);
* then remove it from the reclaimable accounting. */
kbase_process_page_usage_inc(kctx, alloc->nents);
- for (i = 0; i < alloc->nents; i++) {
- struct page *p = phys_to_page(alloc->pages[i]);
+ /* Attempt to build a zone cache of tracking */
+ err = kbase_zone_cache_build(alloc);
+ if (err == 0) {
+ /* Bulk update all the zones */
+ list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) {
+ zone_page_state_add(-zone_cache->count,
+ zone_cache->zone, NR_SLAB_RECLAIMABLE);
+ }
+ } else {
+ /* Fall-back to page by page updates */
+ int i;
+
+ for (i = 0; i < alloc->nents; i++) {
+ struct page *p = phys_to_page(alloc->pages[i]);
+ struct zone *zone = page_zone(p);
- zone_page_state_add(-1, page_zone(p), NR_SLAB_RECLAIMABLE);
+ zone_page_state_add(-1, zone, NR_SLAB_RECLAIMABLE);
+ }
}
-#if defined(CONFIG_MALI_MIPE_ENABLED)
kbase_tlstream_aux_pagesalloc(
(u32)kctx->id,
(u64)new_page_count);
-#endif
}
int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc)
struct kbase_va_region *reg;
struct dma_buf *dma_buf;
struct dma_buf_attachment *dma_attachment;
+ bool shared_zone = false;
dma_buf = dma_buf_get(fd);
if (IS_ERR_OR_NULL(dma_buf))
/* ignore SAME_VA */
*flags &= ~BASE_MEM_SAME_VA;
+ if (*flags & BASE_MEM_IMPORT_SHARED)
+ shared_zone = true;
+
#ifdef CONFIG_64BIT
if (!kctx->is_compat) {
- /* 64-bit tasks must MMAP anyway, but not expose this address to clients */
+ /*
+ * 64-bit tasks require us to reserve VA on the CPU that we use
+ * on the GPU.
+ */
+ shared_zone = true;
+ }
+#endif
+
+ if (shared_zone) {
*flags |= BASE_MEM_NEED_MMAP;
reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA);
} else {
-#else
- if (1) {
-#endif
reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
}
struct kbase_va_region *reg;
long faulted_pages;
int zone = KBASE_REG_ZONE_CUSTOM_VA;
+ bool shared_zone = false;
*va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) -
PFN_DOWN(address);
/* SAME_VA generally not supported with imported memory (no known use cases) */
*flags &= ~BASE_MEM_SAME_VA;
+ if (*flags & BASE_MEM_IMPORT_SHARED)
+ shared_zone = true;
+
#ifdef CONFIG_64BIT
if (!kctx->is_compat) {
- /* 64-bit tasks must MMAP anyway, but not expose this address to
- * clients */
+ /*
+ * 64-bit tasks require us to reserve VA on the CPU that we use
+ * on the GPU.
+ */
+ shared_zone = true;
+ }
+#endif
+
+ if (shared_zone) {
*flags |= BASE_MEM_NEED_MMAP;
zone = KBASE_REG_ZONE_SAME_VA;
}
-#endif
+
reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone);
if (!reg)
reg->gpu_alloc->imported.user_buf.nr_pages = faulted_pages;
reg->gpu_alloc->imported.user_buf.pages = kmalloc_array(faulted_pages,
sizeof(struct page *), GFP_KERNEL);
- reg->gpu_alloc->imported.user_buf.owner = current;
+ reg->gpu_alloc->imported.user_buf.mm = current->mm;
+ atomic_inc(¤t->mm->mm_count);
if (!reg->gpu_alloc->imported.user_buf.pages)
goto no_page_array;
goto out_unlock;
}
/* can't grow regions which are ephemeral */
- if (reg->flags & BASE_MEM_DONT_NEED) {
+ if (reg->flags & KBASE_REG_DONT_NEED) {
*failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
goto out_unlock;
}
for (i = 0; i < nr_to_grow && !kbase_mem_pool_is_full(pool); i++) {
p = kbase_mem_pool_alloc_page(pool);
+ if (!p)
+ break;
kbase_mem_pool_add(pool, p);
}
/*
*
- * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014, 2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/platform_device.h>
#include <linux/string.h>
-#ifdef CONFIG_MACH_MANTA
-#include <plat/devs.h>
-#endif
/*
* This file is included only for type definitions and functions belonging to
linux_resources[0].start = io_resources->io_memory_region.start;
linux_resources[0].end = io_resources->io_memory_region.end;
linux_resources[0].flags = IORESOURCE_MEM;
+
linux_resources[1].start = io_resources->job_irq_number;
linux_resources[1].end = io_resources->job_irq_number;
linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL;
f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER;
/* Sanity check core requirements*/
- if ((t_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
+ if (unlikely((t_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
~BASE_JD_REQ_COHERENT_GROUP) != BASE_JD_REQ_T ||
(f_atom->core_req & BASEP_JD_REQ_ATOM_TYPE &
- ~BASE_JD_REQ_COHERENT_GROUP) != BASE_JD_REQ_FS ||
+ ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC) != BASE_JD_REQ_FS ||
t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES ||
- f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
- dev_err(kctx->kbdev->dev, "Invalid core requirements\n");
+ f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)) {
+
+ int t_atom_type = t_atom->core_req & BASEP_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP;
+ int f_atom_type = f_atom->core_req & BASEP_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC;
+ int t_has_ex_res = t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES;
+ int f_has_ex_res = f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES;
+
+ if (t_atom_type != BASE_JD_REQ_T) {
+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom not a tiler job. Was: 0x%x\n Expected: 0x%x",
+ t_atom_type, BASE_JD_REQ_T);
+ }
+ if (f_atom_type != BASE_JD_REQ_FS) {
+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom not a fragment shader. Was 0x%x Expected: 0x%x\n",
+ f_atom_type, BASE_JD_REQ_FS);
+ }
+ if (t_has_ex_res) {
+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom has external resources.\n");
+ }
+ if (f_has_ex_res) {
+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom has external resources.\n");
+ }
+
goto out;
}
#include <mali_kbase.h>
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+#include <linux/dma-buf.h>
+#include <asm/cacheflush.h>
+#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */
#include <linux/dma-mapping.h>
#ifdef CONFIG_SYNC
#include "sync.h"
}
struct kbase_debug_copy_buffer {
- u64 size;
+ size_t size;
struct page **pages;
int nr_pages;
- u64 offset;
+ size_t offset;
+ /*To find memory region*/
+ u64 gpu_addr;
+
+ struct page **extres_pages;
+ int nr_extres_pages;
};
+static inline void free_user_buffer(struct kbase_debug_copy_buffer *buffer)
+{
+ struct page **pages = buffer->extres_pages;
+ int nr_pages = buffer->nr_extres_pages;
+
+ if (pages) {
+ int i;
+
+ for (i = 0; i < nr_pages; i++) {
+ struct page *pg = pages[i];
+
+ if (pg)
+ put_page(pg);
+ }
+ kfree(pages);
+ }
+}
+
static void kbase_debug_copy_finish(struct kbase_jd_atom *katom)
{
struct kbase_debug_copy_buffer *buffers =
if (!buffers)
return;
+ kbase_gpu_vm_lock(katom->kctx);
for (i = 0; i < nr; i++) {
int p;
+ struct kbase_va_region *reg;
+
+ reg = kbase_region_tracker_find_region_enclosing_address(
+ katom->kctx, buffers[i].gpu_addr);
if (!buffers[i].pages)
break;
put_page(pg);
}
kfree(buffers[i].pages);
+ if (reg && reg->gpu_alloc) {
+ switch (reg->gpu_alloc->type) {
+ case BASE_MEM_IMPORT_TYPE_USER_BUFFER:
+ {
+ free_user_buffer(&buffers[i]);
+ break;
+ }
+ default:
+ /* Nothing to be done. */
+ break;
+ }
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
+ }
}
+ kbase_gpu_vm_unlock(katom->kctx);
kfree(buffers);
katom->jc = 0;
if (!user_structs)
return -EINVAL;
- buffers = kmalloc_array(nr, sizeof(*buffers), GFP_KERNEL);
+ buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL);
if (!buffers) {
ret = -ENOMEM;
katom->jc = 0;
goto out_cleanup;
}
- if (copy_from_user(user_buffers, user_structs,
- sizeof(*user_buffers)*nr)) {
- ret = -EINVAL;
+ ret = copy_from_user(user_buffers, user_structs,
+ sizeof(*user_buffers)*nr);
+ if (ret)
goto out_cleanup;
- }
- down_read(¤t->mm->mmap_sem);
for (i = 0; i < nr; i++) {
u64 addr = user_buffers[i].address;
u64 page_addr = addr & PAGE_MASK;
u64 last_page_addr = end_page_addr & PAGE_MASK;
int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1;
int pinned_pages;
+ struct kbase_va_region *reg;
+ struct base_external_resource user_extres;
- if (!user_buffers[i].address) {
- memset(&buffers[i], 0,
- sizeof(struct kbase_debug_copy_buffer));
+ if (!addr)
continue;
- }
buffers[i].nr_pages = nr_pages;
buffers[i].offset = addr & ~PAGE_MASK;
+ if (buffers[i].offset >= PAGE_SIZE) {
+ ret = -EINVAL;
+ goto out_cleanup;
+ }
buffers[i].size = user_buffers[i].size;
buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *),
GFP_KERNEL);
if (!buffers[i].pages) {
ret = -ENOMEM;
- goto out_unlock;
+ goto out_cleanup;
}
- pinned_pages = get_user_pages(current, current->mm, page_addr,
+ pinned_pages = get_user_pages_fast(page_addr,
nr_pages,
1, /* Write */
- 0, /* No force */
- buffers[i].pages,
- NULL);
+ buffers[i].pages);
if (pinned_pages < 0) {
ret = pinned_pages;
- goto out_unlock;
+ goto out_cleanup;
}
if (pinned_pages != nr_pages) {
ret = -EINVAL;
+ goto out_cleanup;
+ }
+
+ user_extres = user_buffers[i].extres;
+ if (user_extres.ext_resource == 0ULL) {
+ ret = -EINVAL;
+ goto out_cleanup;
+ }
+
+ buffers[i].gpu_addr = user_extres.ext_resource &
+ ~BASE_EXT_RES_ACCESS_EXCLUSIVE;
+ kbase_gpu_vm_lock(katom->kctx);
+ reg = kbase_region_tracker_find_region_enclosing_address(
+ katom->kctx, buffers[i].gpu_addr);
+
+ if (NULL == reg || NULL == reg->cpu_alloc ||
+ (reg->flags & KBASE_REG_FREE)) {
+ ret = -EINVAL;
goto out_unlock;
}
- }
- up_read(¤t->mm->mmap_sem);
+ kbase_mem_phy_alloc_get(reg->gpu_alloc);
+ buffers[i].nr_extres_pages = reg->nr_pages;
+ if (reg->nr_pages*PAGE_SIZE != buffers[i].size)
+ dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n");
+
+ switch (reg->gpu_alloc->type) {
+ case BASE_MEM_IMPORT_TYPE_USER_BUFFER:
+ {
+ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc;
+ unsigned long nr_pages =
+ alloc->imported.user_buf.nr_pages;
+
+ if (alloc->imported.user_buf.mm != current->mm) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+ buffers[i].extres_pages = kcalloc(nr_pages,
+ sizeof(struct page *), GFP_KERNEL);
+ if (!buffers[i].extres_pages) {
+ ret = -ENOMEM;
+ goto out_unlock;
+ }
+
+ ret = get_user_pages_fast(
+ alloc->imported.user_buf.address,
+ nr_pages, 0,
+ buffers[i].extres_pages);
+ if (ret != nr_pages)
+ goto out_unlock;
+ ret = 0;
+ break;
+ }
+ case BASE_MEM_IMPORT_TYPE_UMP:
+ {
+ dev_warn(katom->kctx->kbdev->dev,
+ "UMP is not supported for debug_copy jobs\n");
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+ default:
+ /* Nothing to be done. */
+ break;
+ }
+ kbase_gpu_vm_unlock(katom->kctx);
+ }
kfree(user_buffers);
return ret;
out_unlock:
- up_read(¤t->mm->mmap_sem);
+ kbase_gpu_vm_unlock(katom->kctx);
out_cleanup:
kfree(buffers);
/* Frees allocated memory for kbase_debug_copy_job struct, including
* members, and sets jc to 0 */
kbase_debug_copy_finish(katom);
-
return ret;
}
-static int kbase_debug_copy(struct kbase_jd_atom *katom)
+static void kbase_mem_copy_from_extres_page(struct kbase_context *kctx,
+ void *extres_page, struct page **pages, unsigned int nr_pages,
+ unsigned int *target_page_nr, size_t offset, size_t *to_copy)
+{
+ void *target_page = kmap(pages[*target_page_nr]);
+ size_t chunk = PAGE_SIZE-offset;
+
+ if (!target_page) {
+ *target_page_nr += 1;
+ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job.");
+ return;
+ }
+
+ chunk = min(chunk, *to_copy);
+
+ memcpy(target_page + offset, extres_page, chunk);
+ *to_copy -= chunk;
+
+ kunmap(pages[*target_page_nr]);
+
+ *target_page_nr += 1;
+ if (*target_page_nr >= nr_pages)
+ return;
+
+ target_page = kmap(pages[*target_page_nr]);
+ if (!target_page) {
+ *target_page_nr += 1;
+ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job.");
+ return;
+ }
+
+ KBASE_DEBUG_ASSERT(target_page);
+
+ chunk = min(offset, *to_copy);
+ memcpy(target_page, extres_page + PAGE_SIZE-offset, chunk);
+ *to_copy -= chunk;
+
+ kunmap(pages[*target_page_nr]);
+}
+
+static int kbase_mem_copy_from_extres(struct kbase_context *kctx,
+ struct kbase_debug_copy_buffer *buf_data)
{
- struct kbase_debug_copy_buffer *buffers =
- (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc;
unsigned int i;
+ unsigned int target_page_nr = 0;
+ struct kbase_va_region *reg;
+ struct page **pages = buf_data->pages;
+ u64 offset = buf_data->offset;
+ size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE;
+ size_t to_copy = min(extres_size, buf_data->size);
+ int ret = 0;
- for (i = 0; i < katom->nr_extres; i++) {
- u64 offset = buffers[i].offset;
- u64 buffer_space = buffers[i].size;
- int p;
+ KBASE_DEBUG_ASSERT(pages != NULL);
- for (p = 0; p < buffers[i].nr_pages; p++) {
- struct page *pg = buffers[i].pages[p];
- void *kpage = kmap(pg);
- u64 page_space = PAGE_SIZE-offset;
- u64 space;
+ kbase_gpu_vm_lock(kctx);
+ reg = kbase_region_tracker_find_region_enclosing_address(
+ kctx, buf_data->gpu_addr);
- if (page_space <= buffer_space)
- space = page_space;
- else
- space = buffer_space;
+ if (!reg) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
- /* Temporary - GPUCORE-1843 covers the implementation
- * of the actual copying. */
- memset(kpage+offset, 0x4B, space);
+ switch (reg->gpu_alloc->type) {
+ case BASE_MEM_IMPORT_TYPE_USER_BUFFER:
+ {
+ for (i = 0; i < buf_data->nr_extres_pages; i++) {
+ struct page *pg = buf_data->extres_pages[i];
+ void *extres_page = kmap(pg);
- if (!PageReserved(pg))
- SetPageDirty(pg);
+ if (extres_page)
+ kbase_mem_copy_from_extres_page(kctx,
+ extres_page, pages,
+ buf_data->nr_pages,
+ &target_page_nr,
+ offset, &to_copy);
kunmap(pg);
- offset = 0;
- buffer_space -= space;
+ if (target_page_nr >= buf_data->nr_pages)
+ break;
}
+ break;
+ }
+ break;
+#ifdef CONFIG_DMA_SHARED_BUFFER
+ case BASE_MEM_IMPORT_TYPE_UMM: {
+ struct dma_buf *dma_buf = reg->gpu_alloc->imported.umm.dma_buf;
+
+ KBASE_DEBUG_ASSERT(dma_buf != NULL);
+
+ ret = dma_buf_begin_cpu_access(dma_buf, 0,
+ buf_data->nr_extres_pages*PAGE_SIZE,
+ DMA_FROM_DEVICE);
+ if (ret)
+ goto out_unlock;
+
+ for (i = 0; i < buf_data->nr_extres_pages; i++) {
+
+ void *extres_page = dma_buf_kmap(dma_buf, i);
+
+ if (extres_page)
+ kbase_mem_copy_from_extres_page(kctx,
+ extres_page, pages,
+ buf_data->nr_pages,
+ &target_page_nr,
+ offset, &to_copy);
+
+ dma_buf_kunmap(dma_buf, i, extres_page);
+ if (target_page_nr >= buf_data->nr_pages)
+ break;
+ }
+ dma_buf_end_cpu_access(dma_buf, 0,
+ buf_data->nr_extres_pages*PAGE_SIZE,
+ DMA_FROM_DEVICE);
+ break;
+ }
+#endif
+ default:
+ ret = -EINVAL;
+ }
+out_unlock:
+ kbase_gpu_vm_unlock(kctx);
+ return ret;
+
+}
+
+static int kbase_debug_copy(struct kbase_jd_atom *katom)
+{
+ struct kbase_debug_copy_buffer *buffers =
+ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc;
+ unsigned int i;
+
+ for (i = 0; i < katom->nr_extres; i++) {
+ int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]);
+
+ if (res)
+ return res;
}
return 0;
{
__user void *data = (__user void *)(uintptr_t) katom->jc;
struct base_jit_alloc_info *info;
- struct kbase_context *kctx = katom->kctx;
int ret;
/* Fail the job if there is no info structure */
goto free_info;
}
- /* If the ID is zero or is in use then fail the job */
- if ((info->id == 0) || (kctx->jit_alloc[info->id])) {
+ /* If the ID is zero then fail the job */
+ if (info->id == 0) {
ret = -EINVAL;
goto free_info;
}
- /* Set the jit_alloc to a non-zero value so we know the ID is in use */
- kctx->jit_alloc[info->id] = (struct kbase_va_region *) -1;
-
/* Sanity check that the PA fits within the VA */
if (info->va_pages < info->commit_pages) {
ret = -EINVAL;
info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc;
+ /* The JIT ID is still in use so fail the allocation */
+ if (kctx->jit_alloc[info->id]) {
+ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED;
+ return;
+ }
+
+ /*
+ * Mark the allocation so we know it's in use even if the
+ * allocation itself fails.
+ */
+ kctx->jit_alloc[info->id] = (struct kbase_va_region *) -1;
+
/* Create a JIT allocation */
reg = kbase_jit_allocate(kctx, info);
if (!reg) {
goto failed_loop;
} else
if (!kbase_sticky_resource_release(katom->kctx, NULL,
- gpu_addr, false))
+ gpu_addr))
failed = true;
}
gpu_addr = ext_res->ext_res[i].ext_resource &
~BASE_EXT_RES_ACCESS_EXCLUSIVE;
- kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr,
- false);
+ kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr);
}
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
kbasep_soft_event_update(katom, BASE_JD_SOFT_EVENT_RESET);
break;
case BASE_JD_REQ_SOFT_DEBUG_COPY:
- return kbase_debug_copy(katom);
+ {
+ int res = kbase_debug_copy(katom);
+
+ if (res)
+ katom->event_code = BASE_JD_EVENT_JOB_INVALID;
+ break;
+ }
case BASE_JD_REQ_SOFT_JIT_ALLOC:
kbase_jit_allocate_process(katom);
break;
--- /dev/null
+ /*
+ *
+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+#include "mali_kbase_strings.h"
+
+#define KBASE_DRV_NAME "mali"
+#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline"
+
+const char kbase_drv_name[] = KBASE_DRV_NAME;
+const char kbase_timeline_name[] = KBASE_TIMELINE_NAME;
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+extern const char kbase_drv_name[];
+extern const char kbase_timeline_name[];
KBASE_TL_RET_ATOM_AS,
KBASE_TL_NRET_ATOM_AS,
KBASE_TL_DEP_ATOM_ATOM,
+ KBASE_TL_NDEP_ATOM_ATOM,
+ KBASE_TL_RDEP_ATOM_ATOM,
KBASE_TL_ATTRIB_ATOM_CONFIG,
KBASE_TL_ATTRIB_AS_CONFIG,
"@pp",
"atom1,atom2"
},
+ {
+ KBASE_TL_NDEP_ATOM_ATOM,
+ __stringify(KBASE_TL_NDEP_ATOM_ATOM),
+ "atom2 no longer depends on atom1",
+ "@pp",
+ "atom1,atom2"
+ },
+ {
+ KBASE_TL_RDEP_ATOM_ATOM,
+ __stringify(KBASE_TL_RDEP_ATOM_ATOM),
+ "resolved dependecy of atom2 depending on atom1",
+ "@pp",
+ "atom1,atom2"
+ },
{
KBASE_TL_ATTRIB_ATOM_CONFIG,
__stringify(KBASE_TL_ATTRIB_ATOM_CONFIG),
kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}
+void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2)
+{
+ const u32 msg_id = KBASE_TL_NDEP_ATOM_ATOM;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom1, sizeof(atom1));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom2, sizeof(atom2));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2)
+{
+ const u32 msg_id = KBASE_TL_RDEP_ATOM_ATOM;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom1, sizeof(atom1));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom2, sizeof(atom2));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu)
{
const u32 msg_id = KBASE_TL_NRET_ATOM_LPU;
void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as);
void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as);
void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2);
+void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2);
+void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2);
void __kbase_tlstream_tl_attrib_atom_config(
void *atom, u64 jd, u64 affinity, u32 config);
void __kbase_tlstream_tl_attrib_as_config(
#define kbase_tlstream_tl_dep_atom_atom(atom1, atom2) \
__TRACE_IF_ENABLED(tl_dep_atom_atom, atom1, atom2)
+/**
+ * kbase_tlstream_tl_ndep_atom_atom - dependency between atoms resolved
+ * @atom1: name of the child atom object
+ * @atom2: name of the parent atom object that depended on child atom
+ *
+ * Function emits a timeline message informing that parent atom execution
+ * dependency on child atom has been resolved.
+ */
+#define kbase_tlstream_tl_ndep_atom_atom(atom1, atom2) \
+ __TRACE_IF_ENABLED(tl_ndep_atom_atom, atom1, atom2)
+
+/**
+ * kbase_tlstream_tl_rdep_atom_atom - information about already resolved dependency between atoms
+ * @atom1: name of the child atom object
+ * @atom2: name of the parent atom object that depended on child atom
+ *
+ * Function emits a timeline message informing that parent atom execution
+ * dependency on child atom has been resolved.
+ */
+#define kbase_tlstream_tl_rdep_atom_atom(atom1, atom2) \
+ __TRACE_IF_ENABLED(tl_rdep_atom_atom, atom1, atom2)
+
/**
* kbase_tlstream_tl_attrib_atom_config - atom job slot attributes
* @atom: name of the atom object
/*
*
- * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
js, _producerof_atom_number_completed); \
} while (0)
-
/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a
* certin caller */
#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \
#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP()
-
static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx,
struct kbase_jd_atom *katom, int js)
{
struct kbase_vinstr_client *cli;
KBASE_DEBUG_ASSERT(vinstr_ctx);
- KBASE_DEBUG_ASSERT(buffer_count >= 0);
- KBASE_DEBUG_ASSERT(buffer_count <= MAX_BUFFER_COUNT);
- KBASE_DEBUG_ASSERT(!(buffer_count & (buffer_count - 1)));
+
+ if (buffer_count > MAX_BUFFER_COUNT
+ || (buffer_count & (buffer_count - 1)))
+ return NULL;
cli = kzalloc(sizeof(*cli), GFP_KERNEL);
if (!cli)
/* Allocate required number of dumping buffers. */
cli->dump_buffers = (char *)__get_free_pages(
- GFP_KERNEL,
+ GFP_KERNEL | __GFP_ZERO,
get_order(cli->dump_size * cli->buffer_count));
if (!cli->dump_buffers)
goto error;
struct vm_area_struct *vma)
{
struct kbase_vinstr_client *cli;
- size_t size;
+ unsigned long size, addr, pfn, offset;
+ unsigned long vm_size = vma->vm_end - vma->vm_start;
KBASE_DEBUG_ASSERT(filp);
KBASE_DEBUG_ASSERT(vma);
KBASE_DEBUG_ASSERT(cli);
size = cli->buffer_count * cli->dump_size;
- if (vma->vm_end - vma->vm_start > size)
- return -ENOMEM;
+
+ if (vma->vm_pgoff > (size >> PAGE_SHIFT))
+ return -EINVAL;
+ if (vm_size > size)
+ return -EINVAL;
+
+ offset = vma->vm_pgoff << PAGE_SHIFT;
+ if ((vm_size + offset) > size)
+ return -EINVAL;
+
+ addr = __pa((unsigned long)cli->dump_buffers + offset);
+ pfn = addr >> PAGE_SHIFT;
return remap_pfn_range(
vma,
vma->vm_start,
- __pa((unsigned long)cli->dump_buffers) >> PAGE_SHIFT,
- size,
+ pfn,
+ vm_size,
vma->vm_page_prot);
}
/* End TILER_CONFIG register */
-
#endif /* _MIDGARD_REGMAP_H_ */
/*
*
- * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
__entry->count)
);
-
#endif /* _MALI_TIMELINE_H */
#undef TRACE_INCLUDE_PATH
/*
*
- * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
Glob('#kernel/drivers/gpu/arm/midgard/K*'))
]
-kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/internal/*/*.c'),
- Glob('#kernel/drivers/gpu/arm/midgard/internal/*/*/*.c')]
+kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/internal/*/*.c')]
if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1':
kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')]
if int(env['ump']) == 1:
env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/ump.ko')
-if Glob('internal/sconsfrag'):
- execfile('internal/sconsfrag')
- get_internal(env)
-
env.KernelObjTarget('kbase', cmd)
env.AppendUnique(BASE=['cutils_linked_list'])
projects / firefly-linux-kernel-4.4.55.git / commitdiff